def update(self):
TemplateNodeMixin.update(self)
input_meta = self.get_input_meta()
meta_inports = self.template_meta_setup().inports
if self.INPUT_PORT_MODEL_NAME in input_meta:
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
meta_inports[self.INPUT_PORT_NAME] = required_cols
else:
if self.INPUT_PORT_NAME in input_meta:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
else:
col_from_inport = {}
enums = [col for col in col_from_inport.keys()]
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
else:
included_colums = [
col for col in enums if col not in self.conf['columns']
]
for col in included_colums:
if col in col_from_inport:
meta_inports[
self.INPUT_PORT_NAME][col] = col_from_inport[col]
else:
meta_inports[self.INPUT_PORT_NAME][col] = None
self.template_meta_setup(in_ports=meta_inports, out_ports=None)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
required = {
'month': 'int16',
'year': 'int16',
'sample_id': 'int64',
}
meta_inports[self.COV_IN] = required
meta_inports[self.ORDER_IN] = required
json = {}
input_meta = self.get_input_meta()
if self.COV_IN in input_meta:
assets = int(math.sqrt(len(input_meta[self.COV_IN]) - 3))
for i in range(assets):
json[i] = 'float64'
elif self.ORDER_IN in input_meta:
assets = len(input_meta[self.ORDER_IN]) - 3
for i in range(assets):
json[i] = 'float64'
json.update(required)
meta_outports[self.OUTPUT_PORT_NAME] = json
self.template_meta_setup(in_ports=meta_inports,
out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
required = {
'year': 'int16',
'month': 'int16',
'sample_id': 'int64',
}
name = self.conf.get("name", "feature")
input_meta = self.get_input_meta()
if self.FEATURE_DF not in input_meta:
col_from_inport = required.copy()
else:
col_from_inport = input_meta[self.FEATURE_DF].copy()
meta_inports[self.SIGNAL_DF] = required
meta_inports[self.FEATURE_DF] = required
# additional ports
cols = {
name+"_mean": "float64",
name+"_std": "float64"
}
col_from_inport.update(cols)
meta_outports[self.OUTPUT_PORT_NAME] = col_from_inport
self.template_meta_setup(
in_ports=meta_inports,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
sub_dict = {
'year': 'int16',
'month': 'int16',
'sample_id': 'int64',
}
required = {
}
required.update(sub_dict)
meta_inports[self.INPUT_PORT_NAME] = required
json_drawdown = {}
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME in input_meta:
assets = len(input_meta[self.INPUT_PORT_NAME]) - 3
for i in range(assets):
json_drawdown[i] = 'float64'
json_drawdown.update(sub_dict)
meta_outports[self.OUTPUT_PORT_NAME] = json_drawdown
self.template_meta_setup(
in_ports=meta_inports,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
cols_required = {}
input_meta = self.get_input_meta()
if (self.INPUT_PORT_LEFT_NAME in input_meta
and self.INPUT_PORT_RIGHT_NAME in input_meta):
col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
col_from_left_inport.update(col_from_right_inport)
meta_outports[self.OUTPUT_PORT_NAME] = col_from_left_inport
elif self.INPUT_PORT_LEFT_NAME in input_meta:
col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
meta_outports[self.OUTPUT_PORT_NAME] = col_from_left_inport
elif self.INPUT_PORT_RIGHT_NAME in input_meta:
col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
meta_outports[self.OUTPUT_PORT_NAME] = col_from_right_inport
else:
meta_outports[self.OUTPUT_PORT_NAME] = {}
meta_inports[self.INPUT_PORT_RIGHT_NAME] = cols_required
meta_inports[self.INPUT_PORT_LEFT_NAME] = cols_required
self.template_meta_setup(
in_ports=meta_inports,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
input_meta = self.get_input_meta()
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
required = meta_inports[self.INPUT_PORT_NAME]
if self.INPUT_PORT_NAME not in input_meta:
col_from_inport = {'sample_id': 'int64'}
col_ref = {}
else:
col_from_inport = {'sample_id': 'int64'}
col_ref = input_meta[self.INPUT_PORT_NAME].copy()
for key in col_ref.keys():
if key in required:
continue
new_key = key + "_mean"
col_from_inport[new_key] = col_ref[key]
for key in col_ref.keys():
if key in required:
continue
new_key = key + "_std"
col_from_inport[new_key] = col_ref[key]
meta_outports[self.OUTPUT_PORT_NAME] = col_from_inport
self.template_meta_setup(in_ports=None, out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
name = self._find_asset_name()
asset_retension = {"asset_name": name}
meta_outports = self.template_meta_setup().outports
meta_outports[self.OUTPUT_ASSET_NAME][
MetaDataSchema.META_DATA] = asset_retension
self.template_meta_setup(
in_ports=None,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
input_meta = self.get_input_meta()
predict = self.conf.get('prediction', 'predict')
pred_contribs: bool = self.conf.get('pred_contribs', False)
meta_inports = self.template_meta_setup().inports
meta_outports = self.template_meta_setup().outports
if (self.INPUT_PORT_NAME in input_meta
and self.INPUT_PORT_MODEL_NAME in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[
self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
if not pred_contribs:
col_from_inport[predict] = None # the type is not determined
else:
col_from_inport = {}
for i in range(len(required_cols)+1):
col_from_inport[i] = None
meta_inports[self.INPUT_PORT_NAME] = required_cols
meta_outports[self.OUTPUT_PORT_NAME][
MetaDataSchema.META_DATA] = col_from_inport
elif (self.INPUT_PORT_NAME not in input_meta and
self.INPUT_PORT_MODEL_NAME in input_meta):
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[
self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
col_from_inport = copy.copy(required_cols)
if not pred_contribs:
col_from_inport[predict] = None # the type is not determined
else:
col_from_inport = {}
for i in range(len(required_cols)+1):
col_from_inport[i] = None
meta_inports[self.INPUT_PORT_NAME] = required_cols
meta_outports[self.OUTPUT_PORT_NAME][
MetaDataSchema.META_DATA] = col_from_inport
elif (self.INPUT_PORT_NAME in input_meta and
self.INPUT_PORT_MODEL_NAME not in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
if not pred_contribs:
col_from_inport[predict] = None # the type is not determined
meta_outports[self.OUTPUT_PORT_NAME][
MetaDataSchema.META_DATA] = col_from_inport
self.template_meta_setup(
in_ports=meta_inports,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
input_meta = self.get_input_meta()
output_cols = {}
if (self.INPUT_PORT_LEFT_NAME in input_meta
and self.INPUT_PORT_RIGHT_NAME in input_meta):
col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
col_from_left_inport.update(col_from_right_inport)
output_cols = col_from_left_inport
elif self.INPUT_PORT_LEFT_NAME in input_meta:
col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
output_cols = col_from_left_inport
elif self.INPUT_PORT_RIGHT_NAME in input_meta:
col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
output_cols = col_from_right_inport
meta_data = self.template_meta_setup()
meta_outports = meta_data.outports
meta_inports = meta_data.inports
left_required = meta_inports[self.INPUT_PORT_LEFT_NAME]
right_required = meta_inports[self.INPUT_PORT_RIGHT_NAME]
if 'column' in self.conf:
col_name = self.conf['column']
input_meta = self.get_input_meta()
if self.INPUT_PORT_LEFT_NAME not in input_meta:
left_required[col_name] = None
else:
col_from_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
if col_name in col_from_inport:
left_required[col_name] = col_from_inport[col_name]
else:
left_required[col_name] = None
if self.INPUT_PORT_RIGHT_NAME not in input_meta:
right_required[col_name] = None
else:
col_from_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
if col_name in col_from_inport:
right_required[col_name] = col_from_inport[col_name]
else:
right_required[col_name] = None
meta_inports[self.INPUT_PORT_LEFT_NAME] = left_required
meta_inports[self.INPUT_PORT_RIGHT_NAME] = right_required
meta_outports[self.OUTPUT_PORT_NAME][MetaDataSchema.META_DATA] = \
output_cols
self.template_meta_setup(in_ports=meta_inports,
out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
retention = {}
if 'new' in self.conf and 'old' in self.conf:
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME not in input_meta:
retention = {}
else:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
oldType = col_from_inport[self.conf['old']]
del col_from_inport[self.conf['old']]
col_from_inport[self.conf['new']] = oldType
retention = col_from_inport
meta_outports = self.template_meta_setup().outports
meta_outports[self.OUTPUT_PORT_NAME][MetaDataSchema.META_DATA] = \
retention
self.template_meta_setup(in_ports=None, out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
meta_inports = self.template_meta_setup().inports
required = meta_inports[self.INPUT_PORT_NAME]
if 'column' in self.conf:
col_name = self.conf['column']
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME not in input_meta:
required[col_name] = None
else:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
if col_name in col_from_inport:
required[col_name] = col_from_inport[col_name]
else:
required[col_name] = None
meta_inports[self.INPUT_PORT_NAME] = required
self.template_meta_setup(in_ports=meta_inports, out_ports=None)
def update(self):
TemplateNodeMixin.update(self)
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME in input_meta:
meta_inports = self.template_meta_setup().inports
meta_outports = self.template_meta_setup().outports
col_from_inport = input_meta[self.INPUT_PORT_NAME]
enums = [col for col in col_from_inport.keys()]
cols_output = {}
cols_output['train'] = OrderedDict()
cols_output['label'] = OrderedDict()
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
else:
included_colums = [col for col in enums
if col not in self.conf['columns']]
cols_required = {}
for col in included_colums:
if col in col_from_inport:
cols_required[col] = col_from_inport[col]
cols_output['train'][col] = col_from_inport[col]
else:
cols_required[col] = None
cols_output['train'][col] = None
if ('target' in self.conf and
self.conf['target'] in col_from_inport):
cols_required[self.conf['target']
] = col_from_inport[self.conf['target']]
cols_output['label'][
self.conf['target']] = col_from_inport[
self.conf['target']]
else:
cols_required[self.conf['target']] = None
cols_output['label'][
self.conf['target']] = None
meta_inports[self.INPUT_PORT_NAME] = cols_required
meta_outports[self.OUTPUT_PORT_NAME][
MetaDataSchema.META_DATA] = cols_output
self.template_meta_setup(
in_ports=meta_inports,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
sub_dict = {
'year': 'int16',
'month': 'int16',
'sample_id': 'int64',
}
required = {
"date": "datetime64[ns]",
}
required.update(sub_dict)
meta_inports[self.INPUT_PORT_NAME] = required
json_cov = {}
json_dis = {}
json_mean = {}
json_corr = {}
json_std = {}
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME in input_meta:
assets = len(input_meta[self.INPUT_PORT_NAME]) - 4
for i in range(assets * assets):
json_cov[i] = 'float64'
for i in range(assets):
json_mean[i] = 'float64'
json_std[i] = 'float64'
for i in range(assets * (assets - 1) // 2):
json_dis[i] = 'float64'
json_corr[i] = 'float64'
json_cov.update(sub_dict)
json_dis.update(sub_dict)
json_mean.update(sub_dict)
json_std.update(sub_dict)
json_corr.update(sub_dict)
meta_outports[self.MEAN_DF] = json_mean
meta_outports[self.STD_DF] = json_std
meta_outports[self.COV_DF] = json_cov
meta_outports[self.CORR_DF] = json_corr
meta_outports[self.DISTANCE_DF] = json_dis
self.template_meta_setup(in_ports=meta_inports,
out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
col_required = {"date": "date"}
input_meta = self.get_input_meta()
json = OrderedDict()
if self.INPUT_PORT_NAME in input_meta:
assets = len(input_meta[self.INPUT_PORT_NAME]) - 1
for i in range(assets):
json[i] = 'float64'
json['date'] = "datetime64[ns]"
json['sample_id'] = 'int64'
json['year'] = 'int16'
json['month'] = 'int16'
meta_inports[self.INPUT_PORT_NAME] = col_required
meta_outports[self.OUTPUT_DASK_PORT] = json
meta_outports[self.OUTPUT_PORT_NAME] = json
self.template_meta_setup(in_ports=meta_inports,
out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
required = {
'month': 'int16',
'year': 'int16',
'sample_id': 'int64',
}
meta_inports[self.INPUT_PORT_NAME] = required
json = {}
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME in input_meta:
num = len(input_meta[self.INPUT_PORT_NAME]) - 3
assets = (1 + int(math.sqrt(1 + 8 * num))) // 2
for i in range(assets):
json[i] = 'int64'
json.update(required)
meta_outports[self.OUTPUT_PORT_NAME] = json
self.template_meta_setup(in_ports=meta_inports,
out_ports=meta_outports)
def update(self):
TemplateNodeMixin.update(self)
meta_outports = self.template_meta_setup().outports
meta_inports = self.template_meta_setup().inports
return_required = {
"date": "datetime64[ns]",
'sample_id': 'int64',
'year': 'int16',
'month': 'int16',
}
weight_required = {
'sample_id': 'int64',
'year': 'int16',
'month': 'int16',
}
tran_required = {
'sample_id': 'int64',
'year': 'int16',
'month': 'int16',
}
addition = {
'portfolio': 'float64'
}
input_meta = self.get_input_meta()
if self.RETURN_IN not in input_meta:
col_from_inport = return_required.copy()
else:
col_from_inport = input_meta[self.RETURN_IN].copy()
meta_inports[self.RETURN_IN] = return_required
meta_inports[self.WEIGHT_IN] = weight_required
meta_inports[self.TRANS_IN] = tran_required
col_from_inport.update(addition)
# additional ports
meta_outports[self.OUTPUT_PORT_NAME] = col_from_inport
self.template_meta_setup(
in_ports=meta_inports,
out_ports=meta_outports
)
def update(self):
TemplateNodeMixin.update(self)
self.conf_update() # update the conf
task_graph = ""
replacementObj = {}
task_graph_obj = None
if 'taskgraph' in self.conf:
try:
task_graph = get_file_path(self.conf['taskgraph'])
except FileNotFoundError:
task_graph = None
if task_graph is not None and os.path.exists(task_graph):
# with open(task_graph) as f:
# task_graph = hashlib.md5(f.read().encode()).hexdigest()
task_graph_obj = TaskGraph.load_taskgraph(
get_file_path(self.conf['taskgraph']))
self.all_inputs = []
self.all_outputs = []
self.task_graph = task_graph_obj
self.update_replace(replacementObj, task_graph_obj)
self.replacementObj = replacementObj
extra_updated = set()
extra_roots = []
if self.task_graph is not None:
self.task_graph._build(replace=self.replacementObj)
if 'input' in self.conf:
# group input ports by node id
self.inp_groups = group_ports(self.conf['input'])
for inp in self.inp_groups.keys():
if inp in self.task_graph:
inputNode = self.task_graph[inp]
update_inputs = []
replaced_ports = set(self.inp_groups[inp])
for oldInput in inputNode.inputs:
if oldInput['to_port'] in replaced_ports:
# we want to disconnect this old one and
# connect to external node
if hasattr(self, 'inputs'):
for externalInput in self.inputs:
if (_get_node(externalInput['to_port'])
== inputNode.uid and _get_port(
externalInput['to_port'])
== oldInput['to_port']):
newInput = {}
newInput['to_port'] = _get_port(
externalInput['to_port'])
newInput[
'from_port'] = externalInput[
'from_port']
newInput[
'from_node'] = externalInput[
'from_node']
update_inputs.append(newInput)
else:
update_inputs.append(oldInput)
inputNode.inputs = update_inputs
# add all the `updated` parents to the set
for i in inputNode.inputs:
if hasattr(i['from_node'], 'ports_setup_cache'):
extra_updated.add(i['from_node'])
# if all the parents are updated, this is
# a new root node
if all([
i['from_node'] in extra_updated
for i in inputNode.inputs
]):
extra_roots.append(inputNode)
self.all_inputs.append((inputNode, inp))
if 'output' in self.conf:
self.oup_groups = group_ports(self.conf['output'])
for oup in self.oup_groups.keys():
if oup in self.task_graph:
outNode = self.task_graph[oup]
# we do not disconnect anything here, as we take extra
# outputs for composite node.
# Node, we rely on the fact that taskgraph.run method
# will remove the output collector from taskgraph if
# the outputlist is set
self.all_outputs.append((outNode, oup))
# outNode_fun(outNode, oup_groups[oup])
# update all the nodes and cache it
self.task_graph.breadth_first_update(extra_roots=extra_roots,
extra_updated=extra_updated)
def update(self):
TemplateNodeMixin.update(self)