def parse_expressions(self, items, variables, cond_context):
processes = []
for k, v in items:
if isinstance(v, (bool, int, float)):
process = Assignment(v)
elif isinstance(v, basestring):
expr = parse(v, variables, cond_context)
if not isinstance(expr, Process):
if k is None:
process = Compute(expr)
else:
process = Assignment(expr)
else:
process = expr
elif isinstance(v, list):
# v is a procedure
# it should be a list of dict (assignment) or string (action)
group_expressions = []
for element in v:
if isinstance(element, dict):
group_expressions.append(element.items()[0])
else:
group_expressions.append((None, element))
group_predictors = \
self.collect_predictors(group_expressions)
group_context = variables.copy()
group_context.update((name, Variable(name))
for name in group_predictors)
sub_processes = self.parse_expressions(group_expressions,
group_context,
cond_context)
process = ProcessGroup(k, sub_processes)
elif isinstance(v, dict):
warnings.warn("Using the 'predictor' keyword is deprecated. "
"If you need several processes to "
"write to the same variable, you should "
"rather use procedures.",
UserDeprecationWarning)
expr = parse(v['expr'], variables, cond_context)
process = Assignment(expr)
process.predictor = v['predictor']
else:
raise Exception("unknown expression type for %s: %s"
% (k, type(v)))
processes.append((k, process))
return processes
def load(self, fpath):
from exprparser import parse
with open(os.path.join(config.input_directory, fpath), "rb") as f:
reader = csv.reader(f)
lines = skip_comment_cells(strip_rows(reader))
header = lines.next()
self.expressions = [parse(s, autovariables=True) for s in header]
table = []
for line in lines:
if any(value == "" for value in line):
raise Exception("empty cell found in %s" % fpath)
table.append([eval(value) for value in line])
ndim = len(header)
unique_last_d, dupe_last_d = unique_duplicate(table.pop(0))
if dupe_last_d:
print(
"Duplicate column header value(s) (for '%s') in '%s': %s"
% (header[-1], fpath, ", ".join(str(v) for v in dupe_last_d))
)
raise Exception(
"bad alignment data in '%s': found %d " "duplicate column header value(s)" % (fpath, len(dupe_last_d))
)
# strip the ndim-1 first columns
headers = [[line.pop(0) for line in table] for _ in range(ndim - 1)]
possible_values = [list(unique(values)) for values in headers]
if ndim > 1:
# having duplicate values is normal when there are more than 2
# dimensions but we need to test whether there are duplicates of
# combinations.
dupe_combos = list(duplicates(zip(*headers)))
if dupe_combos:
print("Duplicate row header value(s) in '%s':" % fpath)
print(PrettyTable(dupe_combos))
raise Exception(
"bad alignment data in '%s': found %d " "duplicate row header value(s)" % (fpath, len(dupe_combos))
)
possible_values.append(unique_last_d)
self.possible_values = possible_values
self.probabilities = list(chain.from_iterable(table))
num_possible_values = prod(len(values) for values in possible_values)
if len(self.probabilities) != num_possible_values:
raise Exception(
"incoherent alignment data in '%s': %d data cells "
"found while it should be %d based on the number "
"of possible values in headers (%s)"
% (
fpath,
len(self.probabilities),
num_possible_values,
" * ".join(str(len(values)) for values in possible_values),
)
)
def all_variables(self, globals_def):
from links import PrefixingLink
variables = self.global_variables(globals_def).copy()
variables.update(self.variables)
cond_context = self.conditional_context
macros = dict((k, parse(v, variables, cond_context))
for k, v in self.macro_strings.iteritems())
variables.update(macros)
variables['other'] = PrefixingLink(macros, self.links, '__other_')
return variables
def parse_expressions(self, items, variables, cond_context):
processes = []
for k, v in items:
if isinstance(v, (bool, int, float)):
process = Assignment(v)
elif isinstance(v, basestring):
expr = parse(v, variables, cond_context)
if not isinstance(expr, Process):
if k is None:
process = Compute(expr)
else:
process = Assignment(expr)
else:
process = expr
elif isinstance(v, list):
# v should be a list of dict
group_expressions = []
for element in v:
if isinstance(element, dict):
group_expressions.append(element.items()[0])
else:
group_expressions.append((None, element))
group_predictors = \
self.collect_predictors(group_expressions)
group_context = variables.copy()
group_context.update((name, Variable(name))
for name in group_predictors)
sub_processes = self.parse_expressions(group_expressions,
group_context,
cond_context)
process = ProcessGroup(k, sub_processes)
elif isinstance(v, dict):
expr = parse(v['expr'], variables, cond_context)
process = Assignment(expr)
process.predictor = v['predictor']
else:
raise Exception("unknown expression type for %s: %s"
% (k, type(v)))
processes.append((k, process))
return processes
def all_symbols(self, globals_def):
from links import PrefixingLink
symbols = global_variables(globals_def).copy()
symbols = WarnOverrideDict(symbols)
symbols.update(self.variables)
cond_context = self.conditional_context
macros = dict((k, parse(v, symbols, cond_context))
for k, v in self.macro_strings.iteritems())
symbols.update(macros)
symbols['other'] = PrefixingLink(macros, self.links, '__other_')
symbols.update(self.methods)
return symbols
def parse_expr(self, k, v, variables, cond_context):
if isinstance(v, (bool, int, float)):
return Assignment(v)
elif isinstance(v, basestring):
expr = parse(v, variables, cond_context)
if isinstance(expr, Process):
return expr
else:
if k is None:
return Compute(expr)
else:
return Assignment(expr)
else:
# lets be explicit about it
return None
def execute(self, s):
entity = self.entity
if entity is None:
raise Exception(entity_required)
period = self.period
if period is None:
raise Exception(period_required)
cond_context = entity.conditional_context
variables = entity.all_variables(self.globals_def)
# add all currently defined temp_variables because otherwise
# local variables (defined within a procedure) wouldn't be available
variables.update((name, Variable(name)) for name in entity.temp_variables.keys())
expr = parse(s, variables, cond_context, interactive=True)
ctx = EntityContext(entity, {"period": period, "nan": np.nan, "__globals__": self.globals_data})
if isinstance(expr, Process):
expr.run(ctx)
print "done."
else:
return expr_eval(expr, ctx)
def execute(self, s):
entity = self.entity
if entity is None:
raise Exception(entity_required)
period = self.period
if period is None:
raise Exception(period_required)
variables = entity.variables
# add all currently defined temp_variables because otherwise
# local variables (defined within a procedure) wouldn't be available
variables.update((name, Variable(name))
for name in entity.temp_variables.keys())
cond_context = entity.conditional_context
expr = parse(s, variables, cond_context)
#FIXME: add globals
ctx = EntityContext(entity, {'period': period, 'nan': np.nan})
if isinstance(expr, Process):
expr.run(ctx)
print "done."
else:
return expr_eval(expr, ctx)
def parse_expressions(self, items, context, cond_context):
"""
items -- a list of tuples (name, process_string)
context -- a dict of all symbols available in the scope
cond_context --
"""
processes = []
for k, v in items:
if k == 'while':
if not isinstance(v, dict):
raise ValueError("while is a reserved keyword")
cond = parse(v['cond'], context, cond_context)
assert isinstance(cond, Expr)
code = self.parse_process_group("while:code", v['code'],
context, cond_context,
purge=False)
process = While(cond, code)
else:
process = self.parse_expr(k, v, context, cond_context)
if process is None:
if self.ismethod(v):
if isinstance(v, list):
# v should be a list of dicts (assignments) or
# strings (actions)
argnames, code_def, result_def = [], v, None
else:
assert isinstance(v, dict)
args_def = v.get('args', '')
argnames = [a.strip()
for a in args_def.split(',')
if a != '']
code_def = v.get('code', [])
result_def = v.get('return')
method_context = context.copy()
method_context.update((name, Variable(name))
for name in argnames)
code = self.parse_process_group("func:code", code_def,
method_context,
cond_context,
purge=False)
#TODO: use code.predictors instead (but it currently
# fails for some reason)
group_expressions = [elem.items()[0] if isinstance(elem, dict)
else (None, elem)
for elem in code_def]
group_predictors = \
self.collect_predictors(group_expressions)
method_context.update((name, Variable(name))
for name in group_predictors)
result = parse(result_def, method_context,
cond_context)
assert result is None or isinstance(result, Expr)
process = Function(argnames, code, result)
elif isinstance(v, dict) and 'predictor' in v:
raise ValueError("Using the 'predictor' keyword is "
"not supported anymore. "
"If you need several processes to "
"write to the same variable, you "
"should rather use procedures.")
else:
raise Exception("unknown expression type for %s: %s"
% (k, type(v)))
processes.append((k, process))
return processes
def parse_processes(self, globals):
from properties import Assignment, Compute, Process, ProcessGroup
from links import PrefixingLink
variables = dict((name, SubscriptableVariable(name, type_))
for name, type_ in globals)
variables.update(self.variables)
cond_context = self.conditional_context
macros = dict((k, parse(v, variables, cond_context))
for k, v in self.macro_strings.iteritems())
variables['other'] = PrefixingLink(macros, self.links, '__other_')
variables.update(macros)
def parse_expressions(items, variables):
processes = []
for k, v in items:
if isinstance(v, (bool, int, float)):
process = Assignment(v)
elif isinstance(v, basestring):
expr = parse(v, variables, cond_context)
if not isinstance(expr, Process):
if k is None:
process = Compute(expr)
else:
process = Assignment(expr)
else:
process = expr
elif isinstance(v, list):
# v should be a list of dict
group_expressions = []
for element in v:
if isinstance(element, dict):
group_expressions.append(element.items()[0])
else:
group_expressions.append((None, element))
group_predictors = \
self.collect_predictors(group_expressions)
group_context = variables.copy()
group_context.update((name, Variable(name))
for name in group_predictors)
sub_processes = parse_expressions(group_expressions,
group_context)
process = ProcessGroup(k, sub_processes)
elif isinstance(v, dict):
expr = parse(v['expr'], variables, cond_context)
process = Assignment(expr)
process.predictor = v['predictor']
else:
raise Exception("unknown expression type for %s: %s"
% (k, type(v)))
processes.append((k, process))
return processes
processes = dict(parse_expressions(self.process_strings.iteritems(),
variables))
fnames = set(self.period_individual_fnames)
def attach_processes(items):
for k, v in items:
if isinstance(v, ProcessGroup):
v.entity = self
attach_processes(v.subprocesses)
elif isinstance(v, Assignment):
predictor = v.predictor if v.predictor is not None else k
if predictor in fnames:
kind = 'period_individual'
else:
kind = None
v.attach(k, self, kind)
else:
v.attach(k, self)
attach_processes(processes.iteritems())
self.processes = processes
def assertEvalEqual(self, s, result):
e = parse(s, autovariables=True)
self.assertArrayEqual(e.evaluate(self.context), result)
def test_one2many_from_entity_context(self):
person = entity_registry['person']
context = EntityContext(person, {'period': 2002})
e = parse("countlink(children)", person.links, autovariables=True)
self.assertArrayEqual(e.evaluate(context), [2, 0, 1, 0, 0])
def test_many2one_from_entity_context(self):
person = entity_registry['person']
context = EntityContext(person, {'period': 2002})
self.assertEqual(context_length(context), 5)
e = parse("mother.age", person.links, autovariables=True)
self.assertArrayEqual(e.evaluate(context), [-1, 55, 55, -1, 22])
