def compile(self, context):
scope = self.scope
eval_lam_name = self.eval_lam.name
from norm.models import Variable
if self.outputs is not None and len(self.outputs) > 0:
for v in self.eval_lam.variables:
orig_name = self.outputs.get(v.name, None)
if orig_name == v.name:
self.outputs[v.name] = self.VARIABLE_SEPARATOR.join(
[eval_lam_name, v.name])
from norm.engine import QuantifiedLambda
if isinstance(self.scope, QuantifiedLambda):
i = -1
for i, col in enumerate(self.scope.cols):
if col == orig_name:
break
if i >= 0:
self.scope.cols[i] = self.outputs[v.name]
else:
self.outputs = OrderedDict(
(v.name, self.VARIABLE_SEPARATOR.join([eval_lam_name, v.name]))
for v in self.eval_lam.variables)
output_keys = set(self.outputs.keys())
variables = [Variable(v.name, v.type_) for v in scope.variables] + \
[Variable(self.outputs[v.name], v.type_) for v in self.eval_lam.variables
if v.name in output_keys]
self.lam = context.temp_lambda(variables)
self.lam.cloned_from = scope
return self
def __init__(self):
from norm.models import lambdas
formatter = Variable(self.VAR_FORMATTER, lambdas.String)
variables = Variable(self.VAR_VARIABLES, lambdas.Any)
output = Variable(self.VAR_OUTPUT, lambdas.String)
super().__init__(
name='format',
description=
'Format the strings with given inputs, the semantic is the same as Python format',
variables=[formatter, variables, output])
def __init__(self):
from norm.models import lambdas
super().__init__(name='extract',
description='Extract patterns from a string'
'"(2014).*(6)".extract("2014-06") --> (2014, 6)/None'
'"2014.*6".extract(s, fillna=False) --> True/False',
variables=[
Variable(self.VAR_PATTERN, lambdas.String),
Variable(self.VAR_STRING, lambdas.String),
Variable(self.VAR_FILLNA, lambdas.Any),
Variable(self.VAR_OUTPUT, lambdas.Any)
])
def __init__(self):
from norm.models import lambdas
super().__init__(
name='read',
description=
'Read data from files [.csv, .tsv, .parq, .jsonl], default to csv file'
'your_lambda.read("path_to_the_file.csv", (sep="\t", skiprows=3))'
'read(your_lambda, "path_to_the_file.csv", (sep="\t", skiprows=3))',
variables=[
Variable(self.VAR_PATH, lambdas.String),
Variable(self.VAR_PARM, lambdas.Any),
Variable(self.VAR_EXT, lambdas.String),
Variable(self.VAR_OUTPUT, lambdas.Any)
])
def compile(self, context):
if self.output_projection is not None:
from norm.models.norm import Variable
from norm.models import lambdas
self.lam = context.temp_lambda(
[Variable(self.output_projection, lambdas.Integer)])
return self
def compile(self, context):
from norm.models import Variable, Lambda
self.lam = context.temp_lambda(
[Variable(Lambda.VAR_OUTPUT, self.eval_lam.output_type)])
if self.output_projection is None and self.projection is not None:
self.output_projection = self.projection.variables[0].name
return self
def __init__(self, type_):
"""
:param type_: the intern type of the list
:type type_: Lambda
"""
variable = Variable(self.INTERN, type_)
super().__init__(name='List[{}]'.format(type_.signature),
description='A list of a certain type',
variables=[variable])
def compile(self, context):
assert (self.projection is not None)
if len(self.projection.variables) > 1:
return MultipleConstantAssignmentExpr(
self.constant, self.projection).compile(context)
from norm.models.norm import Variable
self.variable_name = self.projection.variables[0].name
if context.scope is not None:
self.eval_lam = context.scope
variables = self.eval_lam.variables
self.lam = context.temp_lambda(
variables + [Variable(self.variable_name, self.var_type)])
else:
self.eval_lam = None
self.lam = context.temp_lambda(
[Variable(self.variable_name, self.var_type)])
return self
def compile(self, context):
self.eval_lam = self.column_variable.lam
from norm.models.norm import Variable
from norm.engine import QuantifiedLambda
if isinstance(self.column_variable.lam, QuantifiedLambda):
variables = self.column_variable.lam.get_grouped_variables()
else:
variables = []
variables.append(Variable(self.var_name, self.var_type))
self.lam = context.temp_lambda(variables=variables)
self.lam.cloned_from = self.eval_lam
return self
def compile(self, context):
from norm.models import Lambda, Variable
self._append_projection()
if len(self.outputs) > 0:
if Lambda.VAR_OID in self.outputs.keys():
from norm.models import lambdas
variables = [
Variable(self.outputs[Lambda.VAR_OID], lambdas.Integer)
]
else:
variables = []
variables += [
Variable(self.outputs[v.name], v.type_)
for v in self.eval_lam.variables
if v.name in self.outputs.keys()
]
else:
variables = self.eval_lam.variables
self.lam = context.temp_lambda(variables)
return self
def __init__(self, namespace, name, description, code):
"""
Python function, inputs are wrapped into one variable, outputs are one variable too.
:param namespace: the namespace
:type namespace: str
:param name: the name of the function
:type name: str
:param description: description
:type description: str
:param code: the code of the Python implementation
:type code: str
"""
from norm.models import lambdas
super().__init__(namespace=namespace,
name=name,
description=description,
variables=[Variable(self.VAR_INPUTS, lambdas.Any),
Variable(self.VAR_OUTPUT, lambdas.Any)])
self.status = Status.READY
self.code = dedent(code)
self._load_func()
self.atomic = True
def compile(self, context):
assert (isinstance(self.constant, ListConstant))
assert (isinstance(self.constant.value[0], tuple))
assert (len(self.var_names) == len(self.var_types))
assert (self.eval_lam is None
or all(vn not in self.eval_lam for vn in self.var_names))
from norm.models.norm import Variable
vs = [Variable(n, t) for n, t in zip(self.var_names, self.var_types)]
if context.scope is not None:
self.eval_lam = context.scope
variables = self.eval_lam.variables
self.lam = context.temp_lambda(variables + vs)
else:
self.eval_lam = None
self.lam = context.temp_lambda(vs)
return self
def compile(self, context):
if self.variable is None:
from norm.models.norm import Lambda
assert (context.scope is not None)
assert (isinstance(context.scope, Lambda))
lam = context.scope
# constructing new objects
assert (self.check_assignment_arguments())
self.inputs = self.build_assignment_inputs(lam)
self.outputs = None
is_to_add_data = True
else:
lam = self.variable.lam
if lam is None:
# Might need to be combined with previous context
return self
if self.check_assignment_arguments():
self.inputs = self.build_assignment_inputs(lam)
is_to_add_data = not lam.atomic
else:
self.inputs = self.build_conditional_inputs()
is_to_add_data = False
self.outputs = self.build_outputs(lam)
is_to_add_data &= len(self.outputs) == 0
if self.equality_scope is not None and len(self.equalities) > 0:
return JoinEqualityEvaluationExpr(lam, self.equality_scope, self.equalities, self.inputs, self.outputs)\
.compile(context)
if is_to_add_data:
return AddDataEvaluationExpr(lam, self.inputs, self.variable
is not None).compile(context)
elif isinstance(self.inputs, dict):
if self.projection is not None and self.projection.num == 1:
output_projection = self.projection.variables[0].name
else:
output_projection = None
if lam.atomic:
return AtomicEvaluationExpr(lam, self.inputs,
output_projection).compile(context)
elif len(self.inputs) == 0:
if isinstance(self.variable, ColumnVariable) or isinstance(
self.variable, UnquoteVariable):
self.variable.output_projection = output_projection
return self.variable
elif self.projection is not None and len(self.outputs) == 1:
return RetrieveAllDataExpr(
lam, output_projection).compile(context)
else:
return RetrievePartialDataExpr(
lam, self.outputs).compile(context)
self.eval_lam = lam
from norm.models import Variable
if len(self.outputs) > 0:
variables = [
Variable(self.outputs[v.name], v.type_)
for v in self.eval_lam.variables
if v.name in self.outputs.keys()
]
else:
variables = self.eval_lam.variables
self.lam = context.temp_lambda(variables)
self.lam.cloned_from = self.eval_lam
return self