def __init__(self, **kwargs):
self.cols = OrderedDict()
self.col_order = OrderedDict()
if 'reader_settings' in kwargs:
self.reader_settings = kwargs['reader_settings']
else:
self.reader_settings = {}
def __init__(self, items=[], default=None, required=False):
OrderedDict.__init__(self)
ResourceArgument.__init__(self,
default=default,
required=required,
expectedType=None)
# hack to make ordering work... would be *much* nicer if python
# supported an option for order-preserving **kwargs ala:
# http://groups.google.com/group/python-ideas/browse_thread/thread/f3663e5b1f4fe7d4
for d in items:
self[d[0]] = d[1]
def _parse_blocks(self, lines):
class Blocks:
def __init__(self):
self.imports = []
self.assignments = []
self.defs = []
self.set_up = []
self.tests = []
def add(self, target, accum):
try:
a = getattr(self, target)
a.extend(accum)
except AttributeError:
pass
blocks = Blocks()
def begins_with(name):
return lambda begin, line: begin == name
def exact(name):
return lambda begin, line: line.rstrip() == name
patterns = OrderedDict([
(begins_with('def'), 'defs'),
(begins_with('import'), 'imports'),
(begins_with('from'), 'imports'),
(exact('set up'), 'set_up'),
(lambda begin, line: begin and '=' in line and begin != ' ',
'assignments'),
(lambda begin, line: begin != ' ' and begin != '' and begin !=
'\n', 'tests'),
])
def match(line):
begin = line.split(' ')[0]
for k, v in patterns.items():
if k(begin, line):
return v
def accumulate(line):
matched = match(line)
if matched:
blocks.add(accumulate.target, accumulate.items)
accumulate.target = matched
accumulate.items = []
accumulate.items.append(line)
accumulate.target = ''
accumulate.items = []
map(accumulate, lines)
blocks.add(accumulate.target, accumulate.items)
return blocks
def convert(line):
s = OrderedDict([
('==', 'Equal'),
('!=', 'NotEqual'),
('!~=', 'NotAlmostEqual'),
('~=', 'AlmostEqual'),
(lambda line: bool(line.count('>') > 1), partial(_convert, '>', 'Greater')),
(lambda line: bool(line.count('<') > 1), partial(_convert, '<', 'Less')),
('>=', 'GreaterEqual'),
('<=', 'LessEqual'),
('>', 'Greater'),
('<', 'Less'),
('raises', lambda line: ['with self.assertRaises(' + line.split('raises')[1].strip() + '):', ' ' + line.split('raises')[0].strip()]),
(' is not instanceof ', 'NotIsInstance'),
(' is instanceof ', 'IsInstance'),
('for ', lambda line: line),
(' not in ', 'NotIn'),
(' in ', 'In'),
(' is not None', 'IsNotNone'),
(' is None', 'IsNone'),
(' is not ', 'IsNot'),
(' is ', 'Is'),
])
def to_lambda(i):
def to_code(k, v, line):
l, op, r = line.rpartition(k)
params = ', '.join(map(strip, (l, r))).rstrip().rstrip(',')
return 'self.assert' + v + '(' + params + ')'
k, v = i
new_v = v if callable(v) else partial(to_code, k, v)
del s[k]
if callable(k):
s[k] = new_v
else:
s[lambda line: k in line] = new_v
map(to_lambda, s.items())
matches = filter(lambda k: k(line), s.keys())
return s[matches[0]](line) if len(matches) else line
def reset(self):
super(Kitchen, self).reset()
if hasattr(self, '_running') and self._running:
raise Fail("Error can't reset Kitchen while it is running")
self._sourced_recipes = set()
self._included_recipes = OrderedDict()
self.cookbooks = AttributeDict()
self._cookbookPaths = [ ]
self._running = False
def __init__(self,
drop_p: float = 0.2,
hidden_dim: int = 512,
z_dim: int = None,
num_classes: int = None,
data_dir='',
nsynth_class='',
dataset='',
train_transform=None,
batch_size=128):
"""
Attaches a MLP for finetuning using the standard self-supervised protocol.
Example::
from pl_bolts.callbacks.self_supervised import SSLOnlineEvaluator
# your model must have 2 attributes
model = Model()
model.z_dim = ... # the representation dim
model.num_classes = ... # the num of classes in the model
Args:
drop_p: (0.2) dropout probability
hidden_dim: (1024) the hidden dimension for the finetune MLP
"""
super().__init__()
self.hidden_dim = hidden_dim
self.drop_p = drop_p
self.optimizer = None
self.z_dim = z_dim
self.num_classes = num_classes
self.output = OrderedDict()
self.batch_size = batch_size
if 'esc' in dataset:
data_path = os.path.join(data_dir, dataset + ".lmdb")
self.train_dataset = NormalLMDB_ESC(data_path,
transform=train_transform,
aug_transform=None,
spec_transform=None,
folds=[1, 2, 3, 4])
elif 'nsynth' in dataset:
data_path = os.path.join(data_dir, "train_" + dataset + ".lmdb")
self.train_dataset = NormalLMDBG_NSYNTH(data_path,
transform=train_transform,
aug_transform=None,
spec_transform=None,
target=nsynth_class,
perc=1)
else:
data_path = os.path.join(data_dir, "train_" + dataset + ".lmdb")
self.train_dataset = NormalLMDBG(data_path,
transform=train_transform,
aug_transform=None,
spec_transform=None,
perc=1)
def __init__(self, ruleset_ids=None, var_list=None, test_ids=None):
self.ruleset_ids = []
self.test_ids = []
self.fact_state = OrderedDict()
self.rec_nodes = []
self.debug = False
self.vars_tested = set()
# restore state
if ruleset_ids:
for rsid in ruleset_ids:
self.ruleset_ids.append(rsid)
if var_list:
self.add_vars(var_list, FACT_ASSERTED)
if test_ids:
for var_id in test_ids:
self.test_ids.append(var_id)
def load_settings(self):
'''
load settings from file or user input
'''
self._get_user_settings()
self._get_initial_settings()
while True:
# make sure selected settings are ok
self.print_settings(self.settings_names, self.settings)
settings_ok = raw_input('\nAre these settings ok? (y/n)> ')
if not len(settings_ok):
continue
if settings_ok[0].lower() == 'y':
break
# clear current settings and prompt user again
self.settings = OrderedDict()
self.get_settings_from_user()
if not self._using_prev_settings:
while True:
# prompt user to save new settings for future use
save = raw_input('\nSave settings? (y/n)> ')
if not len(save):
continue
if save[0].lower() == 'y':
# write configs to file
try:
self.write_configs(self.settings_file, self.settings)
except IOError:
print "Error writing to config file %s" % self.settings_file
break
def __init__(self, settings_file=None, user_settings_file=None):
self.settings = OrderedDict()
self.settings_file = settings_file
self.user_settings = OrderedDict()
self.user_settings_file = user_settings_file
class Introspector(object):
def __init__(self, **kwargs):
self.cols = OrderedDict()
self.col_order = OrderedDict()
if 'reader_settings' in kwargs:
self.reader_settings = kwargs['reader_settings']
else:
self.reader_settings = {}
def from_stream(self, stream, limit=None):
for row_count, row in enumerate(OrderedDictReader(stream, **self.reader_settings)):
if limit is None or row_count < limit:
self.introspect_row(row)
else:
break
return self
def introspect_row(self, row):
for (order, (key, val)) in enumerate(row.items()):
self.set_col(key, val, order)
def ordered_cols(self):
'''Create a dictionary that brings together cols and col_order.'''
return dict([(key, (self.cols[key], self.col_order[key])) for key in self.cols.keys()])
def set_col(self, key, value, order):
if key in self.cols:
cast = self.cols[key]
else:
cast = None
# If we haven't done an initial cast, start at most restrictive and move down.
if cast == None:
if self.passes_int(value):
cast = "INT"
elif self.passes_float(value):
cast = "FLOAT"
elif self.passes_char(value):
cast = "CHAR"
else:
cast = "TEXT"
# If we've already cast this column, see if the cast can remain.
elif cast == "INT":
if self.passes_int(value):
cast = "INT"
elif self.passes_float(value):
cast = "FLOAT"
elif self.passes_char(value):
cast = "CHAR"
else:
cast = "TEXT"
elif cast == "FLOAT":
if self.passes_float(value):
cast = "FLOAT"
elif self.passes_char(value):
cast = "CHAR"
else:
cast = "TEXT"
elif cast == "CHAR":
if not self.passes_char(value):
cast = "TEXT"
else:
cast = "TEXT"
self.cols[key] = cast
self.col_order[key] = order
def passes_char(self, value):
if isinstance(value, basestring):
if len(value) <= 100:
return True
return False
def passes_int(self, value, null=True):
if re.match('\d{1,3},?\d{,3}?', value):
value = value.replace(",", "")
if null and value is None:
return True
try:
return str(int(value)) == str(value)
except ValueError:
return False
def passes_float(self, value, null=True):
if re.match('\d{1,3},?\d{,3}?', value):
value = value.replace(",", "")
if null and value is None:
return True
try:
float(value)
return True
except ValueError:
return False
class Kitchen(Environment):
def __init__(self):
self._running = False
super(Kitchen, self).__init__()
def reset(self):
super(Kitchen, self).reset()
if hasattr(self, '_running') and self._running:
raise Fail("Error can't reset Kitchen while it is running")
self._sourced_recipes = set()
self._included_recipes = OrderedDict()
self.cookbooks = AttributeDict()
self._cookbookPaths = [ ]
self._running = False
def run(self):
recipes = ""
for k, v in self._included_recipes.iteritems():
name = k
(cookbook, recipe) = v
recipes += "%s : %s(%s); " % (name, cookbook, recipe)
utils.log("Kitchen is up and running...")
utils.log(" recipes (%d): %s" % (len(self._included_recipes), recipes))
utils.log(" cookbook_path (%d): %s" % (len(self._cookbookPaths), str(self._cookbookPaths)))
self._running = True
self._preRun()
super(Kitchen, self).run()
self._postRun()
self._running = False
utils.log("Kitchen finished processing all recipes successfully!")
def includeRecipe(self, *args):
for name in args:
if name in self._included_recipes:
continue
try:
cookbook, recipe = name.split('.')
except ValueError:
cookbook, recipe = name, "default"
try:
cookbook = self.cookbooks[cookbook]
except KeyError:
try:
self._loadCookbook(cookbook)
cookbook = self.cookbooks[cookbook]
except ImportError:
utils.log("Error: unable to find cookbook for recipe '%s'" % name)
raise
self._included_recipes[name] = (cookbook, recipe)
if self._running:
self._sourceRecipe(cookbook, recipe)
def addCookbookPath(self, *args):
for origPath in args:
path = utils.resolvePath(origPath)
#print "%s vs %s" % (origPath, path)
self._cookbookPaths.append((origPath, path))
def _registerCookbook(self, cookbook):
#utils.log("Registering cookbook %s" % (cookbook, ))
self.updateConfig(dict((k, v.get('default')) for k, v in cookbook.config.items()), False)
self.cookbooks[cookbook.name] = cookbook
def _loadCookbook(self, *args, **kwargs):
for name in args:
cookbook = None
for origpath, path in reversed(self._cookbookPaths):
fullpath = os.path.join(path, name)
if not os.path.exists(fullpath):
continue
cookbook = Cookbook.loadFromPath(name, fullpath)
break
if not cookbook:
raise ImportError("Cookbook '%s' not found" % name)
self._registerCookbook(cookbook)
def _sourceRecipe(self, cookbook, recipe):
name = "%s.%s" % (cookbook.name, recipe)
if name in self._sourced_recipes:
return
utils.log("Sourcing recipe '%s' in cookbook '%s'" % (recipe, cookbook.name))
self._sourced_recipes.add(name)
cookbook.loader(self)
rc = cookbook.getRecipe(recipe)
globs = { 'env': self }
with self:
exec compile(rc, name, 'exec') in globs
def _preRun(self):
for name in self._included_recipes:
(cookbook, recipe) = self._included_recipes[name]
self._sourceRecipe(cookbook, recipe)
def _postRun(self):
pass
def __getstate__(self):
state = super(Kitchen, self).__getstate__()
state.update(
cookbookPaths = [ x[0] for x in self.cookbookPaths ],
includedRecipes = self.includedRecipes,
)
return state
def __setstate__(self, state):
super(Kitchen, self).__setstate__(state)
for path in state['cookbookPaths']:
self.addCookbookPath(path)
for recipe in state['includedRecipes']:
self.includeRecipe(recipe)
class Engine(object):
def __init__(self, ruleset_ids=None, var_list=None, test_ids=None):
self.ruleset_ids = []
self.test_ids = []
self.fact_state = OrderedDict()
self.rec_nodes = []
self.debug = False
self.vars_tested = set()
# restore state
if ruleset_ids:
for rsid in ruleset_ids:
self.ruleset_ids.append(rsid)
if var_list:
self.add_vars(var_list, FACT_ASSERTED)
if test_ids:
for var_id in test_ids:
self.test_ids.append(var_id)
# list of ruleset to use
def get_rulesets(self):
return self.ruleset_ids
# return list of variables which has been
# tested/established/need to be tested
def get_vars(self, need_state=False):
var_list = []
for key, state in self.fact_state.items():
vid,value = self.decode_fact_key(key)
if need_state:
var_list.append((vid,value,state))
else:
var_list.append((vid,value))
return var_list
def get_tests(self):
return self.test_ids
def gen_fact_key(self, var_id, value):
return '%d:%s' % (var_id, value)
def decode_fact_key(self, key):
idstr,value = key.split(':',1)
var_id = int(idstr)
return var_id,value
def create_vnode(self, var_id, value):
key = self.gen_fact_key(var_id, value)
if key in self.fact_state:
state = NODE_PASSED
elif var_id in self.vars_tested:
state = NODE_TESTED
else:
state = NODE_UNTESTED
return FactNode(VAR_NODE, var_id, value, state)
def get_unique_rnodes(self):
# first get unique list of recommends nodes
rdict = {}
for rnode in self.rec_nodes:
if rnode.node_id in rdict:
# replace only if node has higher rank
onode = rdict[rnode.node_id]
if rnode.value > onode.value:
rdict[rnode.node_id] = rnode
else:
rdict[rnode.node_id] = rnode
return rdict.values()
def get_questions(self):
questions = []
for variable in Variable.objects.filter(id__in=self.test_ids):
if variable.ask:
questions.append(variable)
return questions
# return list of recommendation for rules that have fired
# TODO fix views to use only get_reasons call
def get_recommends(self):
rnodes = self.get_unique_rnodes()
recommends = []
for rnode in rnodes:
recommend = Recommend.objects.get(pk=rnode.node_id)
# FIXME - this is a hack
recommend.rank = rnode.value
recommends.append(recommend)
return sorted(recommends, key=lambda recommend: recommend.rank, reverse=True)
def get_answers(self):
answers = []
for key, state in self.fact_state.items():
if state == FACT_ANSWERED:
vid,value = self.decode_fact_key(key)
# TODO should we include questions not answered
if value:
answers.append((vid,value))
return answers
# reverse climb the tree to the top node (we use node_set to prevent loops)
def next_premises(self, search_premises, qa_list, node_set):
if len(search_premises) == 0:
return
tnode_list = []
for tnode in reversed(search_premises[0].get_nodes()):
if tnode not in node_set:
try:
var = Variable.objects.get(pk=tnode.node_id)
except Variable.DoesNotExist:
continue
# FIXME this really should be the fact name
text = var.prompt if len(var.prompt) > 0 else var.name
qa = (text, tnode.value)
qa_list.append(qa)
tnode_list.append(tnode)
node_set.add(tnode)
for tnode in tnode_list:
self.next_premises(tnode.get_premises(), qa_list, node_set)
def get_reasons(self):
# first get unique list of recommends nodes
rnode_list = self.get_unique_rnodes()
# now build a list of the reasons that go with them
reasons = []
for rnode in rnode_list:
qa_list = []
node_set = set()
self.next_premises(rnode.get_premises(), qa_list, node_set)
recommend = Recommend.objects.get(pk=rnode.node_id)
reasons.append(Reason(
recommend.id,
recommend.name,
recommend.text,
rnode.value,
qa_list))
return sorted(reasons, key=lambda reason: reason.rank, reverse=True)
# asserted fact = variable that has been assinged a value
def add_var(self, var_id, value, state):
# record that we've seen this variable
self.vars_tested.add(var_id)
# update fact state
key = self.gen_fact_key(var_id, value)
if key not in self.fact_state:
self.fact_state[key] = state
else:
# ASSERTED < ANSWERED < INFERRED
curr_state = self.fact_state[key]
if state < curr_state:
self.fact_state[key] = state
def add_vars(self, facts, default_state):
for item in facts:
state = item[2] if len(item) > 2 else default_state
self.add_var(item[0], item[1], state)
def get_groups(self, tree, rule):
group_list = []
parser = PremiseParser()
# parse ast
plist = []
for premise in rule.rulepremise_set.all():
plist.append(premise)
try:
root = parser.parse(plist)
root = wff_dnf(root)
except PremiseException as e:
if self.debug: print e, plist
return group_list
# now get each or group
or_list = []
grab_or_nodes(root, or_list)
for or_group in or_list:
premise_group = FactGroup()
pnode_list = []
flatten_node(or_group, pnode_list)
for pnode in pnode_list:
if pnode.ptype != PTYPE_VAR:
if self.debug:
print '!!PremiseNode not a var', pnode
continue
variable_id = pnode.left
value = pnode.right
tnode = tree.get_fact(variable_id, value)
if not tnode:
tnode = self.create_vnode(variable_id, value)
tree.add_fact(tnode)
premise_group.add_node(tnode)
group_list.append(premise_group)
return group_list
def build_tree(self, ruleset):
tree = FactTree()
for rule in ruleset.rule_set.all():
group_list = self.get_groups(tree, rule)
"""
premise_group = FactGroup()
for premise in rule.rulepremise_set.all():
node = tree.get_fact(premise.variable_id, premise.value)
if not node:
node = self.create_vnode(premise.variable_id, premise.value)
tree.add_fact(node)
premise_group.add_node(node)
"""
for premise_group in group_list:
conclusion_nodes = []
for conclusion in rule.ruleconclusion_set.all():
node = tree.get_fact(conclusion.variable_id, conclusion.value)
if not node:
node = self.create_vnode(conclusion.variable_id, conclusion.value)
tree.add_fact(node)
node.add_premise(premise_group)
conclusion_nodes.append(node)
recommend_nodes = []
for rrecommend in rule.rulerecommend_set.all():
node = tree.get_rec(rrecommend.recommend_id, rrecommend.rank)
if not node:
node = FactNode(REC_NODE,
rrecommend.recommend_id,
rrecommend.rank,
NODE_UNTESTED)
tree.add_rec(node)
node.add_premise(premise_group)
recommend_nodes.append(node)
premise_group.add_children(conclusion_nodes)
premise_group.add_children(recommend_nodes)
tree.add_group(premise_group)
return tree
def forward_chain(self, tree):
# now look for rules that have fired
test_groups = tree.get_groups()
new_facts = True
while new_facts:
new_facts = False
next_test = []
for group in test_groups:
if group.all_passed():
for child in group.get_children():
child.set_state(NODE_PASSED)
if child.get_type() == VAR_NODE:
self.add_var(child.node_id,
child.value,
FACT_INFERRED)
new_facts = True
else:
next_test.append(group)
test_groups = next_test
return test_groups
def get_first(self, premise_list, node_set):
if self.debug:
print 'get_first premise', [ str(x) for x in premise_list]
for premise in premise_list:
num_loop = 0
for node in premise.get_nodes():
if self.debug: print 'node', node
if node in node_set:
if self.debug: print '+++++++LOOP++++++++'
num_loop += 1
continue
node_set.add(node)
if node.check_state(NODE_UNTESTED):
leafp = self.get_first(node.get_premises(), node_set)
if leafp:
return leafp
if num_loop == 0:
return premise
return None
def find_backchains(self, node, node_set):
if self.debug: print 'find_backchains', node
backchains = []
if node in node_set:
if self.debug: print '+++++++LOOP++++++++'
#print 'Node', node, 'premises', [ str(x) for x in node_set ]
return True
node_set.add(node)
#print 'Node', node, 'premises', [ str(x) for x in node.get_premises() ]
for premise in node.get_premises():
if self.debug: print 'premise', premise
untested = []
num_tested = 0
for pnode in premise.get_nodes():
if pnode.check_state(NODE_UNTESTED):
untested.append(pnode)
elif pnode.check_state(NODE_TESTED):
num_tested = num_tested + 1
if num_tested == 0 and len(untested) > 0:
num_backchain = 0
for pnode in untested:
if self.find_backchains(pnode, node_set):
num_backchain += 1
if self.debug: print 'num_backchain', num_backchain, len(untested)
if num_backchain == len(untested):
if self.debug: print 'backchain', premise
backchains.append(premise)
if self.debug:
print 'Node', node, 'backchains', [ str(x) for x in backchains ]
premsies = node.get_premises()
node.set_premises(backchains)
return len(premsies) == 0 or len(backchains) > 0
def find_goals(self, tree, unfired):
test_premises = []
for rec_node in tree.get_goals():
if rec_node.check_state(NODE_PASSED):
if self.debug: print 'Found goal!', rec_node
self.rec_nodes.append(rec_node)
else:
if self.find_backchains(rec_node, set()):
premises = rec_node.get_premises()
test_premises.append(self.get_first(premises, set()))
test_ids = []
for premise in test_premises:
for node in premise.get_nodes():
if node.state == NODE_UNTESTED:
try:
variable = Variable.objects.get(pk=node.node_id)
except Variable.DoesNotExist:
continue
if variable.ask:
test_ids.append(node.node_id)
if len(test_ids) > 0:
self.test_ids.append(test_ids[0])
# given some answers update facts base and check if rules have fired
# we use forward chaining here
# answers = list of (var_id,value) tuples
def next_state(self, answers=None):
# first add asserted facts
if answers:
if self.debug: print 'Got answers', answers
self.add_vars(answers, FACT_ANSWERED)
# now add variables for which we did not get answers
for var_id in self.test_ids:
if var_id not in self.vars_tested:
self.add_var(var_id, '', FACT_ANSWERED)
# and reset testable node list
self.test_ids = []
# reset rule list
self.fire_ids = []
# this really needs to be fixed
self.rec_nodes = []
for ruleset in RuleSet.objects.filter(id__in=self.ruleset_ids):
tree = self.build_tree(ruleset)
unfired = self.forward_chain(tree)
self.find_goals(tree, unfired)
