def meta_setup(self):
cols_required = {}
required = {
self.INPUT_PORT_LEFT_NAME: cols_required,
self.INPUT_PORT_RIGHT_NAME: cols_required
}
input_meta = self.get_input_meta()
if (self.INPUT_PORT_LEFT_NAME in input_meta
and self.INPUT_PORT_RIGHT_NAME in input_meta):
col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
col_from_left_inport.update(col_from_right_inport)
output_cols = {self.OUTPUT_PORT_NAME: col_from_left_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif self.INPUT_PORT_LEFT_NAME in input_meta:
col_from_left_inport = input_meta[self.INPUT_PORT_LEFT_NAME]
output_cols = {self.OUTPUT_PORT_NAME: col_from_left_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif self.INPUT_PORT_RIGHT_NAME in input_meta:
col_from_right_inport = input_meta[self.INPUT_PORT_RIGHT_NAME]
output_cols = {self.OUTPUT_PORT_NAME: col_from_right_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
else:
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {}})
return metadata
def meta_setup(self):
cols_required = {}
required = {self.INPUT_PORT_NAME: cols_required}
if 'columns' in self.conf and self.conf.get('include', True):
cols_required = {}
for col in self.conf['columns']:
cols_required[col] = None
required = {self.INPUT_PORT_NAME: cols_required}
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME in input_meta:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
enums = [col for col in col_from_inport.keys()]
cols_output = {}
cols_output['train'] = OrderedDict()
cols_output['label'] = OrderedDict()
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
else:
included_colums = [
col for col in enums if col not in self.conf['columns']
]
cols_required = {}
for col in included_colums:
if col in col_from_inport:
cols_required[col] = col_from_inport[col]
cols_output['train'][col] = col_from_inport[col]
else:
cols_required[col] = None
cols_output['train'][col] = None
if ('target' in self.conf
and self.conf['target'] in col_from_inport):
cols_required[self.conf['target']] = col_from_inport[
self.conf['target']]
cols_output['label'][self.conf[
'target']] = col_from_inport[self.conf['target']]
else:
cols_required[self.conf['target']] = None
cols_output['label'][self.conf['target']] = None
required = {
self.INPUT_PORT_NAME: cols_required,
}
output_cols = {
self.OUTPUT_PORT_NAME: cols_output,
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
else:
col_from_inport = {}
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
def meta_setup(self):
metadata = MetaData()
input_meta = self.get_input_meta()
required = {self.INPUT_PORT_NAME: {}, self.INPUT_PORT_MODEL_NAME: {}}
predict = self.conf.get('prediction', 'predict')
output_cols = {self.OUTPUT_PORT_NAME: {predict: None}}
if (self.INPUT_PORT_NAME in input_meta
and self.INPUT_PORT_MODEL_NAME in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
predict = self.conf.get('prediction', 'predict')
col_from_inport[predict] = None # the type is not determined
required = {
self.INPUT_PORT_NAME: required_cols,
self.INPUT_PORT_MODEL_NAME: {}
}
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PORT_NAME not in input_meta
and self.INPUT_PORT_MODEL_NAME in input_meta):
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
predict = self.conf.get('prediction', 'predict')
col_from_inport = copy.copy(required_cols)
col_from_inport[predict] = None # the type is not determined
required = {
self.INPUT_PORT_NAME: required_cols,
self.INPUT_PORT_MODEL_NAME: {}
}
output_cols = {self.OUTPUT_PORT_NAME: col_from_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PORT_NAME in input_meta
and self.INPUT_PORT_MODEL_NAME not in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
predict = self.conf.get('prediction', 'predict')
col_from_inport[predict] = None # the type is not determined
output_cols = {self.OUTPUT_PORT_NAME: col_from_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
metadata = MetaData(inports=required, outports=output_cols)
return metadata
def meta_setup(self):
cache_key, task_graph, replacementObj = self._compute_hash_key()
if cache_key in CACHE_META:
# print('cache hit')
return CACHE_META[cache_key]
required = {}
out_meta = {}
if 'taskgraph' in self.conf:
task_graph.build(replace=replacementObj)
def inputNode_fun(inputNode, in_ports):
req = {}
# do meta_setup so required columns are ready
input_meta = inputNode.meta_setup().inports
for key in input_meta.keys():
if key in in_ports:
req[key] = input_meta[key]
required.update(fix_port_name(req, inputNode.uid))
def outNode_fun(outNode, out_ports):
oucols = {}
before_fix = outNode.meta_setup().outports
for key in before_fix.keys():
if key in out_ports:
oucols[key] = before_fix[key]
out_meta.update(fix_port_name(oucols,
outNode.uid))
self._make_sub_graph_connection(task_graph,
inputNode_fun, outNode_fun)
metadata = MetaData(inports=required, outports=out_meta)
CACHE_META[cache_key] = metadata
return metadata
def meta_setup(self):
input_meta = self.get_input_meta()
if issubclass(self.instanceClass, TrainableNM):
input_meta = self.get_input_meta()
if self.INPUT_NM in input_meta:
if (share_weight in self.conf
and self.conf[share_weight] == 'Reuse'):
self.conf = input_meta[self.INPUT_NM]
if self.instance is not None:
inports = self.instance.input_ports
outports = self.instance.output_ports
else:
try:
p_inports = self.instanceClass.input_ports
p_outports = self.instanceClass.output_ports
feeder = FeedProperty(self.conf)
inports = p_inports.fget(feeder)
outports = p_outports.fget(feeder)
except Exception:
inports = None
outports = None
required = {}
out_meta = {}
if inports is not None:
for k in inports.keys():
required[k] = serialize_type(inports[k])
if outports is not None:
for k in outports.keys():
out_meta[k] = serialize_type(outports[k])
if self.instance is not None:
out_meta[self.OUTPUT_NM] = self.conf
metadata = MetaData(inports=required, outports=out_meta)
return metadata
def retention_meta_setup(self, retention, required={}):
input_cols = {
self.INPUT_PORT_NAME: required
}
meta_data = MetaData(inports=input_cols,
outports={self.OUTPUT_PORT_NAME: retention})
return meta_data
def meta_setup(self):
output = {}
for inp in inputNode.inputs:
output[inp['to_port']] = inp[
'from_node'].meta_setup().outports[
inp['from_port']]
# it will be something like { input_port: columns }
return MetaData(inports={}, outports=output)
def meta_setup(self):
cols_required = {}
required = {
self.INPUT_PORT_NAME: cols_required
}
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {}})
return metadata
def meta_setup(self):
cols_required = {"datetime": "date",
"strategy_returns": "float64"}
required = {
self.INPUT_PORT_NAME: cols_required
}
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {}})
return metadata
def meta_setup(self):
required = {self.INPUT_PORT_NAME: {}}
input_meta = self.get_input_meta()
output_cols = {self.OUTPUT_PORT_NAME: {}}
if (self.INPUT_PORT_NAME in input_meta):
output_cols = {
self.OUTPUT_PORT_NAME: input_meta[self.INPUT_PORT_NAME]
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
def meta_setup(self, required={}):
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME in input_meta:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport
}
input_cols = {
self.INPUT_PORT_NAME: required
}
meta_data = MetaData(inports=input_cols, outports=output_cols)
return meta_data
else:
input_cols = {
self.INPUT_PORT_NAME: required
}
output_cols = {
self.OUTPUT_PORT_NAME: required
}
meta_data = MetaData(inports=input_cols, outports=output_cols)
return meta_data
def meta_setup(self):
cols_required = {}
if 'col_x' in self.conf:
cols_required[self.conf['col_x']] = None
if 'col_y' in self.conf:
cols_required[self.conf['col_y']] = None
if 'col_color' in self.conf:
cols_required[self.conf['col_color']] = None
required = {self.INPUT_PORT_NAME: cols_required}
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {}})
return metadata
def meta_setup(self):
cols_required = {
"datetime": "date",
"open": "float64",
"close": "float64",
"high": "float64",
"low": "float64",
"volume": "float64"
}
required = {self.INPUT_PORT_NAME: cols_required}
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {}})
return metadata
def meta_setup(self):
required = {}
output = {}
output['axes'] = []
output['element'] = {}
output['element']['types'] = ['VoidType']
output['element']['fields'] = 'None'
output['element']['parameters'] = '{}'
required = self.get_input_meta()
required['input_tensor'] = copy.deepcopy(output)
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {}})
return metadata
def meta_setup(self):
column_types = {}
total_features = self.conf.get("n_features", 20)
dtype = self.conf.get("dtype", "float32")
for i in range(total_features):
column_types["x" + str(i)] = dtype
column_types['y'] = 'int64'
inputs = {}
out_cols = {
CUDF_PORT_NAME: column_types,
DASK_CUDF_PORT_NAME: column_types,
}
metadata = MetaData(inports=inputs, outports=out_cols)
return metadata
def meta_setup(self):
cols_required = {}
icols = self.get_input_meta()
if 'label' in self.conf:
label = self.conf['label']
labeltype = icols.get(self.INPUT_PORT_NAME, {}).get(label)
cols_required[label] = labeltype
if 'prediction' in self.conf:
cols_required[self.conf['prediction']] = None
required = {
self.INPUT_PORT_NAME: cols_required
}
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: {},
self.OUTPUT_VALUE_NAME: {}})
return metadata
def meta_setup(self):
column_types = {"datetime": "date",
"asset": "int64",
"volume": "float64",
"close": "float64",
"open": "float64",
"high": "float64",
"low": "float64"}
out_cols = {
CUDF_PORT_NAME: column_types,
DASK_CUDF_PORT_NAME: column_types,
PANDAS_PORT_NAME: column_types
}
required = {}
metadata = MetaData(inports=required, outports=out_cols)
return metadata
def addition_meta_setup(self, addition, required={}):
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME not in input_meta:
col_from_inport = required
else:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
input_cols = {
self.INPUT_PORT_NAME: required
}
# additional ports
output_cols = {
self.OUTPUT_PORT_NAME: addition
}
output_cols[self.OUTPUT_PORT_NAME].update(col_from_inport)
meta_data = MetaData(inports=input_cols, outports=output_cols)
return meta_data
def deletion_meta_setup(self, deletion, required={}):
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME not in input_meta:
col_from_inport = required
else:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
# delete the columns from the inputs
input_cols = {
self.INPUT_PORT_NAME: required
}
for key in deletion:
if key in col_from_inport:
del col_from_inport[key]
meta_data = MetaData(inports=input_cols,
outports={self.OUTPUT_PORT_NAME: col_from_inport})
return meta_data
def meta_setup(self):
required = {}
column_types = {"asset": "int64",
"asset_name": "object"}
out_cols = {
STOCK_NAME_PORT_NAME: column_types,
}
if self.outport_connected(STOCK_MAP_PORT_NAME):
if 'file' in self.conf:
hash_key = self._compute_hash_key()
if hash_key in CACHE_NAME:
out_cols.update({
STOCK_MAP_PORT_NAME: CACHE_NAME[hash_key]})
else:
path = get_file_path(self.conf['file'])
name_df = cudf.read_csv(path)[['SM_ID', 'SYMBOL']]
name_df.columns = ["asset", 'asset_name']
pdf = name_df.to_pandas()
column_data = pdf.to_dict('list')
CACHE_NAME[hash_key] = column_data
out_cols.update({STOCK_MAP_PORT_NAME: column_data})
metadata = MetaData(inports=required, outports=out_cols)
return metadata
def meta_setup(self):
# if 'no_feature' in self.conf:
# retention = self.conf['no_feature']
# else:
cols_required = {'datetime': 'date', "asset": "int64"}
# self.delayed_process = True
required = {self.INPUT_PORT_NAME: cols_required}
retention = {}
retention['signal'] = 'float64'
# _PortTypesMixin.retention_meta_setup(self, retention)
input_meta = self.get_input_meta()
if self.INPUT_PORT_NAME not in input_meta:
col_from_inport = required[self.INPUT_PORT_NAME]
else:
col_from_inport = input_meta[self.INPUT_PORT_NAME]
# delete the columns from the inputs
if 'no_feature' in self.conf:
for key in self.conf['no_feature']:
if key in col_from_inport:
retention[key] = col_from_inport[key]
metadata = MetaData(inports=required,
outports={self.OUTPUT_PORT_NAME: retention})
return metadata
def meta_setup(self):
required = {self.INPUT_PORT_NAME: {}, self.INPUT_PROJ_NAME: {}}
if 'columns' in self.conf and self.conf.get('include', True):
cols_required = {}
for col in self.conf['columns']:
cols_required[col] = None
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_PROJ_NAME: cols_required
}
output_cols = {
self.OUTPUT_PORT_NAME: required[self.INPUT_PORT_NAME],
self.OUTPUT_PROJ_NAME: required[self.INPUT_PROJ_NAME]
}
input_meta = self.get_input_meta()
if (self.INPUT_PROJ_NAME in input_meta
and self.INPUT_PORT_NAME in input_meta):
cols_required = copy.copy(input_meta[self.INPUT_PROJ_NAME])
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_PROJ_NAME: cols_required
}
col_from_inport = input_meta[self.INPUT_PORT_NAME]
if SPECIAL_OUTPUT_DIM_COL in cols_required:
out_dim = cols_required[SPECIAL_OUTPUT_DIM_COL]
del cols_required[SPECIAL_OUTPUT_DIM_COL]
cols = ['em' + str(i) for i in range(out_dim)]
for col in cols:
col_from_inport[col] = None
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
self.OUTPUT_PROJ_NAME: cols_required
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PROJ_NAME in input_meta
and self.INPUT_PORT_NAME not in input_meta):
cols_required = copy.copy(input_meta[self.INPUT_PROJ_NAME])
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_PROJ_NAME: cols_required
}
output = copy.copy(cols_required)
if SPECIAL_OUTPUT_DIM_COL in cols_required:
out_dim = cols_required[SPECIAL_OUTPUT_DIM_COL]
del cols_required[SPECIAL_OUTPUT_DIM_COL]
cols = ['em' + str(i) for i in range(out_dim)]
for col in cols:
output[col] = None
output_cols = {
self.OUTPUT_PORT_NAME: output,
self.OUTPUT_PROJ_NAME: cols_required
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PROJ_NAME not in input_meta
and self.INPUT_PORT_NAME in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
enums = [col for col in col_from_inport.keys()]
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
else:
included_colums = [
col for col in enums if col not in self.conf['columns']
]
cols_required = OrderedDict()
for col in included_colums:
if col in col_from_inport:
cols_required[col] = col_from_inport[col]
else:
cols_required[col] = None
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_PROJ_NAME: cols_required
}
col_dict = [
'em' + str(i) for i in range(self.conf['out_dimension'])
]
for col in col_dict:
col_from_inport[col] = None
proj_out = copy.copy(cols_required)
proj_out[SPECIAL_OUTPUT_DIM_COL] = self.conf['out_dimension']
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
self.OUTPUT_PROJ_NAME: proj_out
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
metadata = MetaData(inports=required, outports=output_cols)
return metadata
def meta_setup(self):
input_meta = self.get_input_meta()
required = {self.INPUT_PORT_NAME: {}, self.INPUT_PORT_MODEL_NAME: {}}
predict = self.conf.get('prediction', 'predict')
output_cols = {self.OUTPUT_PORT_NAME: {predict: None}}
if (self.INPUT_PORT_NAME in input_meta
and self.INPUT_PORT_MODEL_NAME in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
predict = self.conf.get('prediction', 'predict')
col_from_inport[predict] = None # the type is not determined
required = {
self.INPUT_PORT_NAME: required_cols,
self.INPUT_PORT_MODEL_NAME: {}
}
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PORT_NAME not in input_meta
and self.INPUT_PORT_MODEL_NAME in input_meta):
if 'train' in input_meta[self.INPUT_PORT_MODEL_NAME]:
required_cols = input_meta[self.INPUT_PORT_MODEL_NAME]['train']
else:
required_cols = {}
predict = self.conf.get('prediction', 'predict')
col_from_inport = copy.copy(required_cols)
col_from_inport[predict] = None
# the type is not determined
required = {
self.INPUT_PORT_NAME: required_cols,
self.INPUT_PORT_MODEL_NAME: {}
}
output_cols = {self.OUTPUT_PORT_NAME: col_from_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PORT_NAME in input_meta
and self.INPUT_PORT_MODEL_NAME not in input_meta):
cols_required = {}
col_from_inport = input_meta[self.INPUT_PORT_NAME]
enums = [col for col in col_from_inport.keys()]
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
else:
included_colums = [
col for col in enums if col not in self.conf['columns']
]
for col in included_colums:
if col in col_from_inport:
cols_required[col] = col_from_inport[col]
else:
cols_required[col] = None
predict = self.conf.get('prediction', 'predict')
col_from_inport[predict] = None # the type is not determined
required = {
self.INPUT_PORT_NAME: cols_required,
}
output_cols = {self.OUTPUT_PORT_NAME: col_from_inport}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
elif (self.INPUT_PORT_NAME not in input_meta
and self.INPUT_PORT_MODEL_NAME not in input_meta):
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
cols_required = {}
for col in included_colums:
cols_required[col] = None
required = {
self.INPUT_PORT_NAME: cols_required,
}
metadata = MetaData(inports=required, outports=output_cols)
return metadata
def meta_setup(self):
cols_required = {}
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_NORM_MODEL_NAME: cols_required
}
if 'columns' in self.conf and self.conf.get('include', True):
cols_required = {}
for col in self.conf['columns']:
cols_required[col] = None
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_NORM_MODEL_NAME: cols_required
}
output_cols = {
self.OUTPUT_PORT_NAME: required[self.INPUT_PORT_NAME],
self.OUTPUT_NORM_MODEL_NAME: required[self.INPUT_NORM_MODEL_NAME]
}
input_meta = self.get_input_meta()
if (self.INPUT_NORM_MODEL_NAME in input_meta
and self.INPUT_PORT_NAME in input_meta):
cols_required = input_meta[self.INPUT_NORM_MODEL_NAME]
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_NORM_MODEL_NAME: cols_required
}
col_from_inport = input_meta[self.INPUT_PORT_NAME]
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
self.OUTPUT_NORM_MODEL_NAME: cols_required
}
elif (self.INPUT_NORM_MODEL_NAME in input_meta
and self.INPUT_PORT_NAME not in input_meta):
cols_required = input_meta[self.INPUT_NORM_MODEL_NAME]
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_NORM_MODEL_NAME: cols_required
}
output_cols = {
self.OUTPUT_PORT_NAME: cols_required,
self.OUTPUT_NORM_MODEL_NAME: cols_required
}
elif (self.INPUT_NORM_MODEL_NAME not in input_meta
and self.INPUT_PORT_NAME in input_meta):
col_from_inport = input_meta[self.INPUT_PORT_NAME]
enums = [col for col in col_from_inport.keys()]
if 'columns' in self.conf:
if self.conf.get('include', True):
included_colums = self.conf['columns']
else:
included_colums = [
col for col in enums if col not in self.conf['columns']
]
cols_required = OrderedDict()
for col in included_colums:
if col in col_from_inport:
cols_required[col] = col_from_inport[col]
else:
cols_required[col] = None
required = {
self.INPUT_PORT_NAME: cols_required,
self.INPUT_NORM_MODEL_NAME: cols_required
}
output_cols = {
self.OUTPUT_PORT_NAME: col_from_inport,
self.OUTPUT_NORM_MODEL_NAME: cols_required
}
metadata = MetaData(inports=required, outports=output_cols)
# The port INPUT_NORM_MODEL_NAME connection is optional. If not
# connected do not set in required
isconnected = \
self.INPUT_NORM_MODEL_NAME in self.get_connected_inports()
if not isconnected:
metadata.inports.pop(self.INPUT_NORM_MODEL_NAME, None)
return metadata