def update(self): TemplateNodeMixin.update(self) input_meta = self.get_input_meta() meta_inports = self.template_meta_setup().inports if self.INPUT_PORT_MODEL_NAME in input_meta: if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]: required_cols = input_meta[self.INPUT_PORT_MODEL_NAME]['train'] else: required_cols = {} meta_inports[self.INPUT_PORT_NAME] = required_cols else: if self.INPUT_PORT_NAME in input_meta: col_from_inport = input_meta[self.INPUT_PORT_NAME] else: col_from_inport = {} enums = [col for col in col_from_inport.keys()] if 'columns' in self.conf: if self.conf.get('include', True): included_colums = self.conf['columns'] else: included_colums = [ col for col in enums if col not in self.conf['columns'] ] for col in included_colums: if col in col_from_inport: meta_inports[ self.INPUT_PORT_NAME][col] = col_from_inport[col] else: meta_inports[self.INPUT_PORT_NAME][col] = None self.template_meta_setup(in_ports=meta_inports, out_ports=None)
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports required = { 'month': 'int16', 'year': 'int16', 'sample_id': 'int64', } meta_inports[self.COV_IN] = required meta_inports[self.ORDER_IN] = required json = {} input_meta = self.get_input_meta() if self.COV_IN in input_meta: assets = int(math.sqrt(len(input_meta[self.COV_IN]) - 3)) for i in range(assets): json[i] = 'float64' elif self.ORDER_IN in input_meta: assets = len(input_meta[self.ORDER_IN]) - 3 for i in range(assets): json[i] = 'float64' json.update(required) meta_outports[self.OUTPUT_PORT_NAME] = json self.template_meta_setup(in_ports=meta_inports, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports required = { 'year': 'int16', 'month': 'int16', 'sample_id': 'int64', } name = self.conf.get("name", "feature") input_meta = self.get_input_meta() if self.FEATURE_DF not in input_meta: col_from_inport = required.copy() else: col_from_inport = input_meta[self.FEATURE_DF].copy() meta_inports[self.SIGNAL_DF] = required meta_inports[self.FEATURE_DF] = required # additional ports cols = { name+"_mean": "float64", name+"_std": "float64" } col_from_inport.update(cols) meta_outports[self.OUTPUT_PORT_NAME] = col_from_inport self.template_meta_setup( in_ports=meta_inports, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports sub_dict = { 'year': 'int16', 'month': 'int16', 'sample_id': 'int64', } required = { } required.update(sub_dict) meta_inports[self.INPUT_PORT_NAME] = required json_drawdown = {} input_meta = self.get_input_meta() if self.INPUT_PORT_NAME in input_meta: assets = len(input_meta[self.INPUT_PORT_NAME]) - 3 for i in range(assets): json_drawdown[i] = 'float64' json_drawdown.update(sub_dict) meta_outports[self.OUTPUT_PORT_NAME] = json_drawdown self.template_meta_setup( in_ports=meta_inports, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports cols_required = {} input_meta = self.get_input_meta() if (self.INPUT_PORT_LEFT_NAME in input_meta and self.INPUT_PORT_RIGHT_NAME in input_meta): col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME] col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME] col_from_left_inport.update(col_from_right_inport) meta_outports[self.OUTPUT_PORT_NAME] = col_from_left_inport elif self.INPUT_PORT_LEFT_NAME in input_meta: col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME] meta_outports[self.OUTPUT_PORT_NAME] = col_from_left_inport elif self.INPUT_PORT_RIGHT_NAME in input_meta: col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME] meta_outports[self.OUTPUT_PORT_NAME] = col_from_right_inport else: meta_outports[self.OUTPUT_PORT_NAME] = {} meta_inports[self.INPUT_PORT_RIGHT_NAME] = cols_required meta_inports[self.INPUT_PORT_LEFT_NAME] = cols_required self.template_meta_setup( in_ports=meta_inports, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) input_meta = self.get_input_meta() meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports required = meta_inports[self.INPUT_PORT_NAME] if self.INPUT_PORT_NAME not in input_meta: col_from_inport = {'sample_id': 'int64'} col_ref = {} else: col_from_inport = {'sample_id': 'int64'} col_ref = input_meta[self.INPUT_PORT_NAME].copy() for key in col_ref.keys(): if key in required: continue new_key = key + "_mean" col_from_inport[new_key] = col_ref[key] for key in col_ref.keys(): if key in required: continue new_key = key + "_std" col_from_inport[new_key] = col_ref[key] meta_outports[self.OUTPUT_PORT_NAME] = col_from_inport self.template_meta_setup(in_ports=None, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) name = self._find_asset_name() asset_retension = {"asset_name": name} meta_outports = self.template_meta_setup().outports meta_outports[self.OUTPUT_ASSET_NAME][ MetaDataSchema.META_DATA] = asset_retension self.template_meta_setup( in_ports=None, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) input_meta = self.get_input_meta() predict = self.conf.get('prediction', 'predict') pred_contribs: bool = self.conf.get('pred_contribs', False) meta_inports = self.template_meta_setup().inports meta_outports = self.template_meta_setup().outports if (self.INPUT_PORT_NAME in input_meta and self.INPUT_PORT_MODEL_NAME in input_meta): col_from_inport = input_meta[self.INPUT_PORT_NAME] if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]: required_cols = input_meta[ self.INPUT_PORT_MODEL_NAME]['train'] else: required_cols = {} if not pred_contribs: col_from_inport[predict] = None # the type is not determined else: col_from_inport = {} for i in range(len(required_cols)+1): col_from_inport[i] = None meta_inports[self.INPUT_PORT_NAME] = required_cols meta_outports[self.OUTPUT_PORT_NAME][ MetaDataSchema.META_DATA] = col_from_inport elif (self.INPUT_PORT_NAME not in input_meta and self.INPUT_PORT_MODEL_NAME in input_meta): if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]: required_cols = input_meta[ self.INPUT_PORT_MODEL_NAME]['train'] else: required_cols = {} col_from_inport = copy.copy(required_cols) if not pred_contribs: col_from_inport[predict] = None # the type is not determined else: col_from_inport = {} for i in range(len(required_cols)+1): col_from_inport[i] = None meta_inports[self.INPUT_PORT_NAME] = required_cols meta_outports[self.OUTPUT_PORT_NAME][ MetaDataSchema.META_DATA] = col_from_inport elif (self.INPUT_PORT_NAME in input_meta and self.INPUT_PORT_MODEL_NAME not in input_meta): col_from_inport = input_meta[self.INPUT_PORT_NAME] if not pred_contribs: col_from_inport[predict] = None # the type is not determined meta_outports[self.OUTPUT_PORT_NAME][ MetaDataSchema.META_DATA] = col_from_inport self.template_meta_setup( in_ports=meta_inports, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) input_meta = self.get_input_meta() output_cols = {} if (self.INPUT_PORT_LEFT_NAME in input_meta and self.INPUT_PORT_RIGHT_NAME in input_meta): col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME] col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME] col_from_left_inport.update(col_from_right_inport) output_cols = col_from_left_inport elif self.INPUT_PORT_LEFT_NAME in input_meta: col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME] output_cols = col_from_left_inport elif self.INPUT_PORT_RIGHT_NAME in input_meta: col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME] output_cols = col_from_right_inport meta_data = self.template_meta_setup() meta_outports = meta_data.outports meta_inports = meta_data.inports left_required = meta_inports[self.INPUT_PORT_LEFT_NAME] right_required = meta_inports[self.INPUT_PORT_RIGHT_NAME] if 'column' in self.conf: col_name = self.conf['column'] input_meta = self.get_input_meta() if self.INPUT_PORT_LEFT_NAME not in input_meta: left_required[col_name] = None else: col_from_inport = input_meta[self.INPUT_PORT_LEFT_NAME] if col_name in col_from_inport: left_required[col_name] = col_from_inport[col_name] else: left_required[col_name] = None if self.INPUT_PORT_RIGHT_NAME not in input_meta: right_required[col_name] = None else: col_from_inport = input_meta[self.INPUT_PORT_RIGHT_NAME] if col_name in col_from_inport: right_required[col_name] = col_from_inport[col_name] else: right_required[col_name] = None meta_inports[self.INPUT_PORT_LEFT_NAME] = left_required meta_inports[self.INPUT_PORT_RIGHT_NAME] = right_required meta_outports[self.OUTPUT_PORT_NAME][MetaDataSchema.META_DATA] = \ output_cols self.template_meta_setup(in_ports=meta_inports, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) retention = {} if 'new' in self.conf and 'old' in self.conf: input_meta = self.get_input_meta() if self.INPUT_PORT_NAME not in input_meta: retention = {} else: col_from_inport = input_meta[self.INPUT_PORT_NAME] oldType = col_from_inport[self.conf['old']] del col_from_inport[self.conf['old']] col_from_inport[self.conf['new']] = oldType retention = col_from_inport meta_outports = self.template_meta_setup().outports meta_outports[self.OUTPUT_PORT_NAME][MetaDataSchema.META_DATA] = \ retention self.template_meta_setup(in_ports=None, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) meta_inports = self.template_meta_setup().inports required = meta_inports[self.INPUT_PORT_NAME] if 'column' in self.conf: col_name = self.conf['column'] input_meta = self.get_input_meta() if self.INPUT_PORT_NAME not in input_meta: required[col_name] = None else: col_from_inport = input_meta[self.INPUT_PORT_NAME] if col_name in col_from_inport: required[col_name] = col_from_inport[col_name] else: required[col_name] = None meta_inports[self.INPUT_PORT_NAME] = required self.template_meta_setup(in_ports=meta_inports, out_ports=None)
def update(self): TemplateNodeMixin.update(self) input_meta = self.get_input_meta() if self.INPUT_PORT_NAME in input_meta: meta_inports = self.template_meta_setup().inports meta_outports = self.template_meta_setup().outports col_from_inport = input_meta[self.INPUT_PORT_NAME] enums = [col for col in col_from_inport.keys()] cols_output = {} cols_output['train'] = OrderedDict() cols_output['label'] = OrderedDict() if 'columns' in self.conf: if self.conf.get('include', True): included_colums = self.conf['columns'] else: included_colums = [col for col in enums if col not in self.conf['columns']] cols_required = {} for col in included_colums: if col in col_from_inport: cols_required[col] = col_from_inport[col] cols_output['train'][col] = col_from_inport[col] else: cols_required[col] = None cols_output['train'][col] = None if ('target' in self.conf and self.conf['target'] in col_from_inport): cols_required[self.conf['target'] ] = col_from_inport[self.conf['target']] cols_output['label'][ self.conf['target']] = col_from_inport[ self.conf['target']] else: cols_required[self.conf['target']] = None cols_output['label'][ self.conf['target']] = None meta_inports[self.INPUT_PORT_NAME] = cols_required meta_outports[self.OUTPUT_PORT_NAME][ MetaDataSchema.META_DATA] = cols_output self.template_meta_setup( in_ports=meta_inports, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports sub_dict = { 'year': 'int16', 'month': 'int16', 'sample_id': 'int64', } required = { "date": "datetime64[ns]", } required.update(sub_dict) meta_inports[self.INPUT_PORT_NAME] = required json_cov = {} json_dis = {} json_mean = {} json_corr = {} json_std = {} input_meta = self.get_input_meta() if self.INPUT_PORT_NAME in input_meta: assets = len(input_meta[self.INPUT_PORT_NAME]) - 4 for i in range(assets * assets): json_cov[i] = 'float64' for i in range(assets): json_mean[i] = 'float64' json_std[i] = 'float64' for i in range(assets * (assets - 1) // 2): json_dis[i] = 'float64' json_corr[i] = 'float64' json_cov.update(sub_dict) json_dis.update(sub_dict) json_mean.update(sub_dict) json_std.update(sub_dict) json_corr.update(sub_dict) meta_outports[self.MEAN_DF] = json_mean meta_outports[self.STD_DF] = json_std meta_outports[self.COV_DF] = json_cov meta_outports[self.CORR_DF] = json_corr meta_outports[self.DISTANCE_DF] = json_dis self.template_meta_setup(in_ports=meta_inports, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports col_required = {"date": "date"} input_meta = self.get_input_meta() json = OrderedDict() if self.INPUT_PORT_NAME in input_meta: assets = len(input_meta[self.INPUT_PORT_NAME]) - 1 for i in range(assets): json[i] = 'float64' json['date'] = "datetime64[ns]" json['sample_id'] = 'int64' json['year'] = 'int16' json['month'] = 'int16' meta_inports[self.INPUT_PORT_NAME] = col_required meta_outports[self.OUTPUT_DASK_PORT] = json meta_outports[self.OUTPUT_PORT_NAME] = json self.template_meta_setup(in_ports=meta_inports, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports required = { 'month': 'int16', 'year': 'int16', 'sample_id': 'int64', } meta_inports[self.INPUT_PORT_NAME] = required json = {} input_meta = self.get_input_meta() if self.INPUT_PORT_NAME in input_meta: num = len(input_meta[self.INPUT_PORT_NAME]) - 3 assets = (1 + int(math.sqrt(1 + 8 * num))) // 2 for i in range(assets): json[i] = 'int64' json.update(required) meta_outports[self.OUTPUT_PORT_NAME] = json self.template_meta_setup(in_ports=meta_inports, out_ports=meta_outports)
def update(self): TemplateNodeMixin.update(self) meta_outports = self.template_meta_setup().outports meta_inports = self.template_meta_setup().inports return_required = { "date": "datetime64[ns]", 'sample_id': 'int64', 'year': 'int16', 'month': 'int16', } weight_required = { 'sample_id': 'int64', 'year': 'int16', 'month': 'int16', } tran_required = { 'sample_id': 'int64', 'year': 'int16', 'month': 'int16', } addition = { 'portfolio': 'float64' } input_meta = self.get_input_meta() if self.RETURN_IN not in input_meta: col_from_inport = return_required.copy() else: col_from_inport = input_meta[self.RETURN_IN].copy() meta_inports[self.RETURN_IN] = return_required meta_inports[self.WEIGHT_IN] = weight_required meta_inports[self.TRANS_IN] = tran_required col_from_inport.update(addition) # additional ports meta_outports[self.OUTPUT_PORT_NAME] = col_from_inport self.template_meta_setup( in_ports=meta_inports, out_ports=meta_outports )
def update(self): TemplateNodeMixin.update(self) self.conf_update() # update the conf task_graph = "" replacementObj = {} task_graph_obj = None if 'taskgraph' in self.conf: try: task_graph = get_file_path(self.conf['taskgraph']) except FileNotFoundError: task_graph = None if task_graph is not None and os.path.exists(task_graph): # with open(task_graph) as f: # task_graph = hashlib.md5(f.read().encode()).hexdigest() task_graph_obj = TaskGraph.load_taskgraph( get_file_path(self.conf['taskgraph'])) self.all_inputs = [] self.all_outputs = [] self.task_graph = task_graph_obj self.update_replace(replacementObj, task_graph_obj) self.replacementObj = replacementObj extra_updated = set() extra_roots = [] if self.task_graph is not None: self.task_graph._build(replace=self.replacementObj) if 'input' in self.conf: # group input ports by node id self.inp_groups = group_ports(self.conf['input']) for inp in self.inp_groups.keys(): if inp in self.task_graph: inputNode = self.task_graph[inp] update_inputs = [] replaced_ports = set(self.inp_groups[inp]) for oldInput in inputNode.inputs: if oldInput['to_port'] in replaced_ports: # we want to disconnect this old one and # connect to external node if hasattr(self, 'inputs'): for externalInput in self.inputs: if (_get_node(externalInput['to_port']) == inputNode.uid and _get_port( externalInput['to_port']) == oldInput['to_port']): newInput = {} newInput['to_port'] = _get_port( externalInput['to_port']) newInput[ 'from_port'] = externalInput[ 'from_port'] newInput[ 'from_node'] = externalInput[ 'from_node'] update_inputs.append(newInput) else: update_inputs.append(oldInput) inputNode.inputs = update_inputs # add all the `updated` parents to the set for i in inputNode.inputs: if hasattr(i['from_node'], 'ports_setup_cache'): extra_updated.add(i['from_node']) # if all the parents are updated, this is # a new root node if all([ i['from_node'] in extra_updated for i in inputNode.inputs ]): extra_roots.append(inputNode) self.all_inputs.append((inputNode, inp)) if 'output' in self.conf: self.oup_groups = group_ports(self.conf['output']) for oup in self.oup_groups.keys(): if oup in self.task_graph: outNode = self.task_graph[oup] # we do not disconnect anything here, as we take extra # outputs for composite node. # Node, we rely on the fact that taskgraph.run method # will remove the output collector from taskgraph if # the outputlist is set self.all_outputs.append((outNode, oup)) # outNode_fun(outNode, oup_groups[oup]) # update all the nodes and cache it self.task_graph.breadth_first_update(extra_roots=extra_roots, extra_updated=extra_updated)
def update(self): TemplateNodeMixin.update(self)