def __init__(self, **kwargs): self.cols = OrderedDict() self.col_order = OrderedDict() if 'reader_settings' in kwargs: self.reader_settings = kwargs['reader_settings'] else: self.reader_settings = {}
def __init__(self, items=[], default=None, required=False): OrderedDict.__init__(self) ResourceArgument.__init__(self, default=default, required=required, expectedType=None) # hack to make ordering work... would be *much* nicer if python # supported an option for order-preserving **kwargs ala: # http://groups.google.com/group/python-ideas/browse_thread/thread/f3663e5b1f4fe7d4 for d in items: self[d[0]] = d[1]
def _parse_blocks(self, lines): class Blocks: def __init__(self): self.imports = [] self.assignments = [] self.defs = [] self.set_up = [] self.tests = [] def add(self, target, accum): try: a = getattr(self, target) a.extend(accum) except AttributeError: pass blocks = Blocks() def begins_with(name): return lambda begin, line: begin == name def exact(name): return lambda begin, line: line.rstrip() == name patterns = OrderedDict([ (begins_with('def'), 'defs'), (begins_with('import'), 'imports'), (begins_with('from'), 'imports'), (exact('set up'), 'set_up'), (lambda begin, line: begin and '=' in line and begin != ' ', 'assignments'), (lambda begin, line: begin != ' ' and begin != '' and begin != '\n', 'tests'), ]) def match(line): begin = line.split(' ')[0] for k, v in patterns.items(): if k(begin, line): return v def accumulate(line): matched = match(line) if matched: blocks.add(accumulate.target, accumulate.items) accumulate.target = matched accumulate.items = [] accumulate.items.append(line) accumulate.target = '' accumulate.items = [] map(accumulate, lines) blocks.add(accumulate.target, accumulate.items) return blocks
def convert(line): s = OrderedDict([ ('==', 'Equal'), ('!=', 'NotEqual'), ('!~=', 'NotAlmostEqual'), ('~=', 'AlmostEqual'), (lambda line: bool(line.count('>') > 1), partial(_convert, '>', 'Greater')), (lambda line: bool(line.count('<') > 1), partial(_convert, '<', 'Less')), ('>=', 'GreaterEqual'), ('<=', 'LessEqual'), ('>', 'Greater'), ('<', 'Less'), ('raises', lambda line: ['with self.assertRaises(' + line.split('raises')[1].strip() + '):', ' ' + line.split('raises')[0].strip()]), (' is not instanceof ', 'NotIsInstance'), (' is instanceof ', 'IsInstance'), ('for ', lambda line: line), (' not in ', 'NotIn'), (' in ', 'In'), (' is not None', 'IsNotNone'), (' is None', 'IsNone'), (' is not ', 'IsNot'), (' is ', 'Is'), ]) def to_lambda(i): def to_code(k, v, line): l, op, r = line.rpartition(k) params = ', '.join(map(strip, (l, r))).rstrip().rstrip(',') return 'self.assert' + v + '(' + params + ')' k, v = i new_v = v if callable(v) else partial(to_code, k, v) del s[k] if callable(k): s[k] = new_v else: s[lambda line: k in line] = new_v map(to_lambda, s.items()) matches = filter(lambda k: k(line), s.keys()) return s[matches[0]](line) if len(matches) else line
def reset(self): super(Kitchen, self).reset() if hasattr(self, '_running') and self._running: raise Fail("Error can't reset Kitchen while it is running") self._sourced_recipes = set() self._included_recipes = OrderedDict() self.cookbooks = AttributeDict() self._cookbookPaths = [ ] self._running = False
def __init__(self, drop_p: float = 0.2, hidden_dim: int = 512, z_dim: int = None, num_classes: int = None, data_dir='', nsynth_class='', dataset='', train_transform=None, batch_size=128): """ Attaches a MLP for finetuning using the standard self-supervised protocol. Example:: from pl_bolts.callbacks.self_supervised import SSLOnlineEvaluator # your model must have 2 attributes model = Model() model.z_dim = ... # the representation dim model.num_classes = ... # the num of classes in the model Args: drop_p: (0.2) dropout probability hidden_dim: (1024) the hidden dimension for the finetune MLP """ super().__init__() self.hidden_dim = hidden_dim self.drop_p = drop_p self.optimizer = None self.z_dim = z_dim self.num_classes = num_classes self.output = OrderedDict() self.batch_size = batch_size if 'esc' in dataset: data_path = os.path.join(data_dir, dataset + ".lmdb") self.train_dataset = NormalLMDB_ESC(data_path, transform=train_transform, aug_transform=None, spec_transform=None, folds=[1, 2, 3, 4]) elif 'nsynth' in dataset: data_path = os.path.join(data_dir, "train_" + dataset + ".lmdb") self.train_dataset = NormalLMDBG_NSYNTH(data_path, transform=train_transform, aug_transform=None, spec_transform=None, target=nsynth_class, perc=1) else: data_path = os.path.join(data_dir, "train_" + dataset + ".lmdb") self.train_dataset = NormalLMDBG(data_path, transform=train_transform, aug_transform=None, spec_transform=None, perc=1)
def __init__(self, ruleset_ids=None, var_list=None, test_ids=None): self.ruleset_ids = [] self.test_ids = [] self.fact_state = OrderedDict() self.rec_nodes = [] self.debug = False self.vars_tested = set() # restore state if ruleset_ids: for rsid in ruleset_ids: self.ruleset_ids.append(rsid) if var_list: self.add_vars(var_list, FACT_ASSERTED) if test_ids: for var_id in test_ids: self.test_ids.append(var_id)
def load_settings(self): ''' load settings from file or user input ''' self._get_user_settings() self._get_initial_settings() while True: # make sure selected settings are ok self.print_settings(self.settings_names, self.settings) settings_ok = raw_input('\nAre these settings ok? (y/n)> ') if not len(settings_ok): continue if settings_ok[0].lower() == 'y': break # clear current settings and prompt user again self.settings = OrderedDict() self.get_settings_from_user() if not self._using_prev_settings: while True: # prompt user to save new settings for future use save = raw_input('\nSave settings? (y/n)> ') if not len(save): continue if save[0].lower() == 'y': # write configs to file try: self.write_configs(self.settings_file, self.settings) except IOError: print "Error writing to config file %s" % self.settings_file break
def __init__(self, settings_file=None, user_settings_file=None): self.settings = OrderedDict() self.settings_file = settings_file self.user_settings = OrderedDict() self.user_settings_file = user_settings_file
class Introspector(object): def __init__(self, **kwargs): self.cols = OrderedDict() self.col_order = OrderedDict() if 'reader_settings' in kwargs: self.reader_settings = kwargs['reader_settings'] else: self.reader_settings = {} def from_stream(self, stream, limit=None): for row_count, row in enumerate(OrderedDictReader(stream, **self.reader_settings)): if limit is None or row_count < limit: self.introspect_row(row) else: break return self def introspect_row(self, row): for (order, (key, val)) in enumerate(row.items()): self.set_col(key, val, order) def ordered_cols(self): '''Create a dictionary that brings together cols and col_order.''' return dict([(key, (self.cols[key], self.col_order[key])) for key in self.cols.keys()]) def set_col(self, key, value, order): if key in self.cols: cast = self.cols[key] else: cast = None # If we haven't done an initial cast, start at most restrictive and move down. if cast == None: if self.passes_int(value): cast = "INT" elif self.passes_float(value): cast = "FLOAT" elif self.passes_char(value): cast = "CHAR" else: cast = "TEXT" # If we've already cast this column, see if the cast can remain. elif cast == "INT": if self.passes_int(value): cast = "INT" elif self.passes_float(value): cast = "FLOAT" elif self.passes_char(value): cast = "CHAR" else: cast = "TEXT" elif cast == "FLOAT": if self.passes_float(value): cast = "FLOAT" elif self.passes_char(value): cast = "CHAR" else: cast = "TEXT" elif cast == "CHAR": if not self.passes_char(value): cast = "TEXT" else: cast = "TEXT" self.cols[key] = cast self.col_order[key] = order def passes_char(self, value): if isinstance(value, basestring): if len(value) <= 100: return True return False def passes_int(self, value, null=True): if re.match('\d{1,3},?\d{,3}?', value): value = value.replace(",", "") if null and value is None: return True try: return str(int(value)) == str(value) except ValueError: return False def passes_float(self, value, null=True): if re.match('\d{1,3},?\d{,3}?', value): value = value.replace(",", "") if null and value is None: return True try: float(value) return True except ValueError: return False
class Kitchen(Environment): def __init__(self): self._running = False super(Kitchen, self).__init__() def reset(self): super(Kitchen, self).reset() if hasattr(self, '_running') and self._running: raise Fail("Error can't reset Kitchen while it is running") self._sourced_recipes = set() self._included_recipes = OrderedDict() self.cookbooks = AttributeDict() self._cookbookPaths = [ ] self._running = False def run(self): recipes = "" for k, v in self._included_recipes.iteritems(): name = k (cookbook, recipe) = v recipes += "%s : %s(%s); " % (name, cookbook, recipe) utils.log("Kitchen is up and running...") utils.log(" recipes (%d): %s" % (len(self._included_recipes), recipes)) utils.log(" cookbook_path (%d): %s" % (len(self._cookbookPaths), str(self._cookbookPaths))) self._running = True self._preRun() super(Kitchen, self).run() self._postRun() self._running = False utils.log("Kitchen finished processing all recipes successfully!") def includeRecipe(self, *args): for name in args: if name in self._included_recipes: continue try: cookbook, recipe = name.split('.') except ValueError: cookbook, recipe = name, "default" try: cookbook = self.cookbooks[cookbook] except KeyError: try: self._loadCookbook(cookbook) cookbook = self.cookbooks[cookbook] except ImportError: utils.log("Error: unable to find cookbook for recipe '%s'" % name) raise self._included_recipes[name] = (cookbook, recipe) if self._running: self._sourceRecipe(cookbook, recipe) def addCookbookPath(self, *args): for origPath in args: path = utils.resolvePath(origPath) #print "%s vs %s" % (origPath, path) self._cookbookPaths.append((origPath, path)) def _registerCookbook(self, cookbook): #utils.log("Registering cookbook %s" % (cookbook, )) self.updateConfig(dict((k, v.get('default')) for k, v in cookbook.config.items()), False) self.cookbooks[cookbook.name] = cookbook def _loadCookbook(self, *args, **kwargs): for name in args: cookbook = None for origpath, path in reversed(self._cookbookPaths): fullpath = os.path.join(path, name) if not os.path.exists(fullpath): continue cookbook = Cookbook.loadFromPath(name, fullpath) break if not cookbook: raise ImportError("Cookbook '%s' not found" % name) self._registerCookbook(cookbook) def _sourceRecipe(self, cookbook, recipe): name = "%s.%s" % (cookbook.name, recipe) if name in self._sourced_recipes: return utils.log("Sourcing recipe '%s' in cookbook '%s'" % (recipe, cookbook.name)) self._sourced_recipes.add(name) cookbook.loader(self) rc = cookbook.getRecipe(recipe) globs = { 'env': self } with self: exec compile(rc, name, 'exec') in globs def _preRun(self): for name in self._included_recipes: (cookbook, recipe) = self._included_recipes[name] self._sourceRecipe(cookbook, recipe) def _postRun(self): pass def __getstate__(self): state = super(Kitchen, self).__getstate__() state.update( cookbookPaths = [ x[0] for x in self.cookbookPaths ], includedRecipes = self.includedRecipes, ) return state def __setstate__(self, state): super(Kitchen, self).__setstate__(state) for path in state['cookbookPaths']: self.addCookbookPath(path) for recipe in state['includedRecipes']: self.includeRecipe(recipe)
class Engine(object): def __init__(self, ruleset_ids=None, var_list=None, test_ids=None): self.ruleset_ids = [] self.test_ids = [] self.fact_state = OrderedDict() self.rec_nodes = [] self.debug = False self.vars_tested = set() # restore state if ruleset_ids: for rsid in ruleset_ids: self.ruleset_ids.append(rsid) if var_list: self.add_vars(var_list, FACT_ASSERTED) if test_ids: for var_id in test_ids: self.test_ids.append(var_id) # list of ruleset to use def get_rulesets(self): return self.ruleset_ids # return list of variables which has been # tested/established/need to be tested def get_vars(self, need_state=False): var_list = [] for key, state in self.fact_state.items(): vid,value = self.decode_fact_key(key) if need_state: var_list.append((vid,value,state)) else: var_list.append((vid,value)) return var_list def get_tests(self): return self.test_ids def gen_fact_key(self, var_id, value): return '%d:%s' % (var_id, value) def decode_fact_key(self, key): idstr,value = key.split(':',1) var_id = int(idstr) return var_id,value def create_vnode(self, var_id, value): key = self.gen_fact_key(var_id, value) if key in self.fact_state: state = NODE_PASSED elif var_id in self.vars_tested: state = NODE_TESTED else: state = NODE_UNTESTED return FactNode(VAR_NODE, var_id, value, state) def get_unique_rnodes(self): # first get unique list of recommends nodes rdict = {} for rnode in self.rec_nodes: if rnode.node_id in rdict: # replace only if node has higher rank onode = rdict[rnode.node_id] if rnode.value > onode.value: rdict[rnode.node_id] = rnode else: rdict[rnode.node_id] = rnode return rdict.values() def get_questions(self): questions = [] for variable in Variable.objects.filter(id__in=self.test_ids): if variable.ask: questions.append(variable) return questions # return list of recommendation for rules that have fired # TODO fix views to use only get_reasons call def get_recommends(self): rnodes = self.get_unique_rnodes() recommends = [] for rnode in rnodes: recommend = Recommend.objects.get(pk=rnode.node_id) # FIXME - this is a hack recommend.rank = rnode.value recommends.append(recommend) return sorted(recommends, key=lambda recommend: recommend.rank, reverse=True) def get_answers(self): answers = [] for key, state in self.fact_state.items(): if state == FACT_ANSWERED: vid,value = self.decode_fact_key(key) # TODO should we include questions not answered if value: answers.append((vid,value)) return answers # reverse climb the tree to the top node (we use node_set to prevent loops) def next_premises(self, search_premises, qa_list, node_set): if len(search_premises) == 0: return tnode_list = [] for tnode in reversed(search_premises[0].get_nodes()): if tnode not in node_set: try: var = Variable.objects.get(pk=tnode.node_id) except Variable.DoesNotExist: continue # FIXME this really should be the fact name text = var.prompt if len(var.prompt) > 0 else var.name qa = (text, tnode.value) qa_list.append(qa) tnode_list.append(tnode) node_set.add(tnode) for tnode in tnode_list: self.next_premises(tnode.get_premises(), qa_list, node_set) def get_reasons(self): # first get unique list of recommends nodes rnode_list = self.get_unique_rnodes() # now build a list of the reasons that go with them reasons = [] for rnode in rnode_list: qa_list = [] node_set = set() self.next_premises(rnode.get_premises(), qa_list, node_set) recommend = Recommend.objects.get(pk=rnode.node_id) reasons.append(Reason( recommend.id, recommend.name, recommend.text, rnode.value, qa_list)) return sorted(reasons, key=lambda reason: reason.rank, reverse=True) # asserted fact = variable that has been assinged a value def add_var(self, var_id, value, state): # record that we've seen this variable self.vars_tested.add(var_id) # update fact state key = self.gen_fact_key(var_id, value) if key not in self.fact_state: self.fact_state[key] = state else: # ASSERTED < ANSWERED < INFERRED curr_state = self.fact_state[key] if state < curr_state: self.fact_state[key] = state def add_vars(self, facts, default_state): for item in facts: state = item[2] if len(item) > 2 else default_state self.add_var(item[0], item[1], state) def get_groups(self, tree, rule): group_list = [] parser = PremiseParser() # parse ast plist = [] for premise in rule.rulepremise_set.all(): plist.append(premise) try: root = parser.parse(plist) root = wff_dnf(root) except PremiseException as e: if self.debug: print e, plist return group_list # now get each or group or_list = [] grab_or_nodes(root, or_list) for or_group in or_list: premise_group = FactGroup() pnode_list = [] flatten_node(or_group, pnode_list) for pnode in pnode_list: if pnode.ptype != PTYPE_VAR: if self.debug: print '!!PremiseNode not a var', pnode continue variable_id = pnode.left value = pnode.right tnode = tree.get_fact(variable_id, value) if not tnode: tnode = self.create_vnode(variable_id, value) tree.add_fact(tnode) premise_group.add_node(tnode) group_list.append(premise_group) return group_list def build_tree(self, ruleset): tree = FactTree() for rule in ruleset.rule_set.all(): group_list = self.get_groups(tree, rule) """ premise_group = FactGroup() for premise in rule.rulepremise_set.all(): node = tree.get_fact(premise.variable_id, premise.value) if not node: node = self.create_vnode(premise.variable_id, premise.value) tree.add_fact(node) premise_group.add_node(node) """ for premise_group in group_list: conclusion_nodes = [] for conclusion in rule.ruleconclusion_set.all(): node = tree.get_fact(conclusion.variable_id, conclusion.value) if not node: node = self.create_vnode(conclusion.variable_id, conclusion.value) tree.add_fact(node) node.add_premise(premise_group) conclusion_nodes.append(node) recommend_nodes = [] for rrecommend in rule.rulerecommend_set.all(): node = tree.get_rec(rrecommend.recommend_id, rrecommend.rank) if not node: node = FactNode(REC_NODE, rrecommend.recommend_id, rrecommend.rank, NODE_UNTESTED) tree.add_rec(node) node.add_premise(premise_group) recommend_nodes.append(node) premise_group.add_children(conclusion_nodes) premise_group.add_children(recommend_nodes) tree.add_group(premise_group) return tree def forward_chain(self, tree): # now look for rules that have fired test_groups = tree.get_groups() new_facts = True while new_facts: new_facts = False next_test = [] for group in test_groups: if group.all_passed(): for child in group.get_children(): child.set_state(NODE_PASSED) if child.get_type() == VAR_NODE: self.add_var(child.node_id, child.value, FACT_INFERRED) new_facts = True else: next_test.append(group) test_groups = next_test return test_groups def get_first(self, premise_list, node_set): if self.debug: print 'get_first premise', [ str(x) for x in premise_list] for premise in premise_list: num_loop = 0 for node in premise.get_nodes(): if self.debug: print 'node', node if node in node_set: if self.debug: print '+++++++LOOP++++++++' num_loop += 1 continue node_set.add(node) if node.check_state(NODE_UNTESTED): leafp = self.get_first(node.get_premises(), node_set) if leafp: return leafp if num_loop == 0: return premise return None def find_backchains(self, node, node_set): if self.debug: print 'find_backchains', node backchains = [] if node in node_set: if self.debug: print '+++++++LOOP++++++++' #print 'Node', node, 'premises', [ str(x) for x in node_set ] return True node_set.add(node) #print 'Node', node, 'premises', [ str(x) for x in node.get_premises() ] for premise in node.get_premises(): if self.debug: print 'premise', premise untested = [] num_tested = 0 for pnode in premise.get_nodes(): if pnode.check_state(NODE_UNTESTED): untested.append(pnode) elif pnode.check_state(NODE_TESTED): num_tested = num_tested + 1 if num_tested == 0 and len(untested) > 0: num_backchain = 0 for pnode in untested: if self.find_backchains(pnode, node_set): num_backchain += 1 if self.debug: print 'num_backchain', num_backchain, len(untested) if num_backchain == len(untested): if self.debug: print 'backchain', premise backchains.append(premise) if self.debug: print 'Node', node, 'backchains', [ str(x) for x in backchains ] premsies = node.get_premises() node.set_premises(backchains) return len(premsies) == 0 or len(backchains) > 0 def find_goals(self, tree, unfired): test_premises = [] for rec_node in tree.get_goals(): if rec_node.check_state(NODE_PASSED): if self.debug: print 'Found goal!', rec_node self.rec_nodes.append(rec_node) else: if self.find_backchains(rec_node, set()): premises = rec_node.get_premises() test_premises.append(self.get_first(premises, set())) test_ids = [] for premise in test_premises: for node in premise.get_nodes(): if node.state == NODE_UNTESTED: try: variable = Variable.objects.get(pk=node.node_id) except Variable.DoesNotExist: continue if variable.ask: test_ids.append(node.node_id) if len(test_ids) > 0: self.test_ids.append(test_ids[0]) # given some answers update facts base and check if rules have fired # we use forward chaining here # answers = list of (var_id,value) tuples def next_state(self, answers=None): # first add asserted facts if answers: if self.debug: print 'Got answers', answers self.add_vars(answers, FACT_ANSWERED) # now add variables for which we did not get answers for var_id in self.test_ids: if var_id not in self.vars_tested: self.add_var(var_id, '', FACT_ANSWERED) # and reset testable node list self.test_ids = [] # reset rule list self.fire_ids = [] # this really needs to be fixed self.rec_nodes = [] for ruleset in RuleSet.objects.filter(id__in=self.ruleset_ids): tree = self.build_tree(ruleset) unfired = self.forward_chain(tree) self.find_goals(tree, unfired)