def _init_node_parm(self, key):
"""
Init parameter from workflow_data_frame
:return:
"""
try:
wf_data_frame = WorkFlowDataFrame(key)
self.type = wf_data_frame.object_type
self.data_sql_stmt = wf_data_frame.sql_stmt
self.data_src_path = utils.get_source_path(self.net_id,
self.net_ver,
self.node_id)
#xgboost train eval all together
self.data_src_eval_path = utils.get_source_path(
self.net_id, self.net_ver, 'evaldata')
self.data_src_type = wf_data_frame.src_type
self.data_server_type = wf_data_frame.src_server
self.data_preprocess_type = wf_data_frame.step_preprocess
# xgboost train eval all together
self.data_store_path = utils.get_store_path(
self.net_id, self.net_ver, self.node_id)
self.data_store_eval_path = utils.get_store_path(
self.net_id, self.net_ver, 'evaldata')
self.sent_max_len = wf_data_frame.max_sentence_len
self.multi_node_flag = wf_data_frame.multi_node_flag
self.drop_duplicate = wf_data_frame.drop_duplicate
self.combine_label_list = list()
except Exception as e:
raise Exception("WorkFlowDataFrame parms are not set " + str(e))
def _init_train_parm(self, conf_data):
# get initial value
self.conf_data = conf_data
self.cls_pool = conf_data["cls_pool"]
self.nn_id = conf_data["nn_id"]
self.wf_ver = conf_data["wf_ver"]
self.node_id = conf_data["node_id"]
graph = NNCommonManager().get_nn_node_name(conf_data["nn_id"])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
self.netconf_node = net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'netconf_feed':
self.train_feed_name = self.nn_id + "_" + self.wf_ver + "_" + net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'eval_feed':
self.eval_feed_name = self.nn_id + "_" + self.wf_ver + "_" + net['fields']['graph_node_name']
self.feed_node = self.get_prev_node()
#key = conf_data.
wf_net_conf = WorkFlowNetConfXgboost(self.node_id )
self.conf = wf_net_conf.conf
self.data_store_path = utils.get_store_path(self.net_id, self.net_ver, "data_node")
self.data_store_eval_path = utils.get_store_path(self.net_id, self.net_ver, 'evaldata')
self.model_path =self.conf.get('model_path')
self.model_type = self.conf.get('model_type')
print("")
def _init_value(self):
'''
Residual Network Init Value
:return:
'''
self.file_end = '.bin'
self.train_return_data = {}
self.data_store_path = utils.get_store_path(self.net_id, self.net_ver, "data_node")
self.data_store_eval_path = utils.get_store_path(self.net_id, self.net_ver, 'evaldata')
self.model_path =self.conf.get('model_path')
self.model_type = self.conf.get('model_type')
def put_step_store(self, src, form, nnid, wfver, node, input_data):
"""
putter for store
:param obj: config data from view
:return:boolean
"""
try:
store_path = utils.get_store_path(nnid, wfver, node)
obj = models.NN_WF_NODE_INFO.objects.get(nn_wf_node_id=str(nnid) + "_" + str(wfver) + "_" + str(node))
config_data = getattr(obj, 'node_config_data')
config_data['store_path'] = utils.get_store_path(nnid, wfver, node)
setattr(obj, 'node_config_data', config_data)
obj.save()
if (os.path.exists(store_path) == False):
os.makedirs(store_path, exist_ok=True)
return config_data['store_path']
except Exception as e:
raise Exception(e)
def _init_node_parm(self, key):
"""
Init parameter from workflow_data_frame
:return:
"""
try :
wf_data_frame = WorkFlowDataFrame(key)
self.type = wf_data_frame.object_type
self.data_sql_stmt = wf_data_frame.sql_stmt
self.data_src_path = utils.get_source_path(self.net_id, self.net_ver, self.node_id)
#xgboost train eval all together
self.data_src_eval_path = utils.get_source_path(self.net_id, self.net_ver, 'evaldata')
self.data_src_type = wf_data_frame.src_type
self.data_server_type = wf_data_frame.src_server
self.data_preprocess_type = wf_data_frame.step_preprocess
# xgboost train eval all together
self.data_store_path = utils.get_store_path(self.net_id, self.net_ver, self.node_id)
self.data_store_eval_path = utils.get_store_path(self.net_id, self.net_ver, 'evaldata')
self.sent_max_len = wf_data_frame.max_sentence_len
self.multi_node_flag = wf_data_frame.multi_node_flag
self.drop_duplicate = wf_data_frame.drop_duplicate
self.combine_label_list = list()
except Exception as e :
raise Exception ("WorkFlowDataFrame parms are not set " + str(e))