def _init_node_parm(self, key):
"""
Init parameter from workflow_data_frame
:return:
"""
try:
wf_data_frame = WorkFlowDataFrame(key)
self.type = wf_data_frame.object_type
self.data_sql_stmt = wf_data_frame.sql_stmt
self.data_src_path = utils.get_source_path(self.net_id,
self.net_ver,
self.node_id)
#xgboost train eval all together
self.data_src_eval_path = utils.get_source_path(
self.net_id, self.net_ver, 'evaldata')
self.data_src_type = wf_data_frame.src_type
self.data_server_type = wf_data_frame.src_server
self.data_preprocess_type = wf_data_frame.step_preprocess
# xgboost train eval all together
self.data_store_path = utils.get_store_path(
self.net_id, self.net_ver, self.node_id)
self.data_store_eval_path = utils.get_store_path(
self.net_id, self.net_ver, 'evaldata')
self.sent_max_len = wf_data_frame.max_sentence_len
self.multi_node_flag = wf_data_frame.multi_node_flag
self.drop_duplicate = wf_data_frame.drop_duplicate
self.combine_label_list = list()
except Exception as e:
raise Exception("WorkFlowDataFrame parms are not set " + str(e))
def post(self, request, nnid):
"""
Common Network Info Post Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Define new neural network process
"""
try:
input_parm = request.data
max_nnid = NNCommonManager().get_nn_id_max() + 1
if nnid == "new":
nnid = "nn" + str(max_nnid).zfill(8)
else:
return_data = NNCommonManager().get_nn_id_info(nnid)
if return_data != []:
return Response(
json.dumps(nnid + " Network ID already exists"))
input_parm['nn_id'] = nnid
if input_parm.get('automl_parms') == None:
input_parm['automl_parms'] = {}
if input_parm.get('automl_runtime') == None:
input_parm['automl_runtime'] = {}
if input_parm.get('automl_stat') == None:
input_parm['automl_stat'] = {}
input_parm_s = {}
input_parm_s['id'] = max_nnid
input_parm_s['nn_id'] = nnid
return_data = NNCommonManager().insert_nn_info(
input_parm, input_parm_s)
# Data node name
graph = NNCommonManager.get_nn_node_name(None, return_data)
return_param = {}
return_param['nn_id'] = return_data
return_param['graph'] = graph
return Response(json.dumps(return_param))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
finally:
graph = NNCommonManager().get_nn_node_name(nnid)
for net in graph:
if net['fields']['graph_node'] in [
'netconf_data', 'eval_data'
]:
utils.get_source_path(nnid, None,
net['fields']['graph_node_name'])
def post(self, request, nnid):
"""
Common Network Info Post Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Define new neural network process
"""
try:
input_parm = request.data
max_nnid = NNCommonManager().get_nn_id_max() + 1
if nnid == "new":
nnid = "nn" + str(max_nnid).zfill(8)
else:
return_data = NNCommonManager().get_nn_id_info(nnid)
if return_data != []:
return Response(json.dumps(nnid+" Network ID already exists"))
input_parm['nn_id'] = nnid
if input_parm.get('automl_parms') == None:
input_parm['automl_parms'] = {}
if input_parm.get('automl_runtime') == None:
input_parm['automl_runtime'] = {}
if input_parm.get('automl_stat') == None:
input_parm['automl_stat'] = {}
input_parm_s = {}
input_parm_s['id'] = max_nnid
input_parm_s['nn_id'] = nnid
return_data = NNCommonManager().insert_nn_info(input_parm, input_parm_s)
# Data node name
graph = NNCommonManager.get_nn_node_name(None, return_data)
return_param = {}
return_param['nn_id'] = return_data
return_param['graph'] = graph
return Response(json.dumps(return_param))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
finally:
graph = NNCommonManager().get_nn_node_name(nnid)
for net in graph:
if net['fields']['graph_node'] in ['netconf_data','eval_data']:
utils.get_source_path(nnid, None, net['fields']['graph_node_name'])
def put_step_source(self, src, form, prg, nnid, wfver, node, input_data):
"""
putter for source step
:param obj: config data from view
:return:boolean
"""
try:
#obj = models.NN_WF_NODE_INFO.objects.get(wf_state_id=str(nnid) + "_" + str(wfver), nn_wf_node_name='data_node')
obj = models.NN_WF_NODE_INFO.objects.get(
nn_wf_node_id=str(nnid) + "_" + str(wfver) + "_" + str(node))
config_data = getattr(obj, 'node_config_data')
config_data['type'] = input_data['type']
config_data['source_type'] = src
config_data['source_parse_type'] = form
config_data['source_server'] = input_data['source_server']
config_data['source_sql'] = input_data['source_sql']
config_data['source_path'] = utils.get_source_path(
nnid, wfver, node)
config_data['predict_path'] = utils.get_source_predict_path(
nnid, wfver, 'predict')
config_data['max_sentence_len'] = input_data.get(
'max_sentence_len', 0)
config_data['multi_node_flag'] = input_data.get('multi_node_flag')
config_data['drop_duplicate'] = input_data.get(
'drop_duplicate'
) if 'drop_duplicate' in input_data else self.config_data_nvl_bool(
config_data, 'drop_duplicate')
setattr(obj, 'node_config_data', config_data)
obj.save()
return config_data
except Exception as e:
raise Exception(e)
def put_step_source(self, src, form, prg, nnid, wfver, node,input_data):
"""
putter for source step
:param obj: config data from view
:return:boolean
"""
try:
#obj = models.NN_WF_NODE_INFO.objects.get(wf_state_id=str(nnid) + "_" + str(wfver), nn_wf_node_name='data_node')
obj = models.NN_WF_NODE_INFO.objects.get(nn_wf_node_id=str(nnid) + "_" + str(wfver) + "_" + str(node))
config_data = getattr(obj, 'node_config_data')
config_data['type'] = input_data['type']
config_data['source_type'] = src
config_data['source_parse_type'] = form
config_data['source_server'] = input_data['source_server']
config_data['source_sql'] = input_data['source_sql']
config_data['source_path'] = utils.get_source_path(nnid, wfver, node)
config_data['predict_path'] = utils.get_source_predict_path(nnid, wfver, 'predict')
config_data['max_sentence_len'] = input_data.get('max_sentence_len' , 0)
config_data['multi_node_flag'] = input_data.get('multi_node_flag')
config_data['drop_duplicate'] = input_data.get('drop_duplicate') if 'drop_duplicate' in input_data else self.config_data_nvl_bool(config_data, 'drop_duplicate')
setattr(obj, 'node_config_data', config_data)
obj.save()
return config_data
except Exception as e:
raise Exception(e)
def put_step_source(self, src, form, nnid, wfver, node, input_data):
"""
putter for source step
:param obj: config data from view
:return:boolean
"""
try:
if(src == 'local') :
source_path = utils.get_source_path(nnid, wfver, node)
return self.update_wf_node_info(src, form, nnid, wfver, node, input_data, source_path)
except Exception as e:
raise Exception(e)
def _init_node_parm(self, key):
"""
Init parameter from workflow_data_frame
:return:
"""
try :
wf_data_frame = WorkFlowDataFrame(key)
self.type = wf_data_frame.object_type
self.data_sql_stmt = wf_data_frame.sql_stmt
self.data_src_path = utils.get_source_path(self.net_id, self.net_ver, self.node_id)
#xgboost train eval all together
self.data_src_eval_path = utils.get_source_path(self.net_id, self.net_ver, 'evaldata')
self.data_src_type = wf_data_frame.src_type
self.data_server_type = wf_data_frame.src_server
self.data_preprocess_type = wf_data_frame.step_preprocess
# xgboost train eval all together
self.data_store_path = utils.get_store_path(self.net_id, self.net_ver, self.node_id)
self.data_store_eval_path = utils.get_store_path(self.net_id, self.net_ver, 'evaldata')
self.sent_max_len = wf_data_frame.max_sentence_len
self.multi_node_flag = wf_data_frame.multi_node_flag
self.drop_duplicate = wf_data_frame.drop_duplicate
self.combine_label_list = list()
except Exception as e :
raise Exception ("WorkFlowDataFrame parms are not set " + str(e))