def _init_node_parm(self, pself, conf_data):
"""
call on init parms from db
:param node_id:
:return:
"""
# get initial value
pself.conf_data = conf_data
pself.cls_pool = conf_data["cls_pool"]
pself.nn_id = conf_data["nn_id"]
pself.wf_ver = conf_data["wf_ver"]
pself.node_id = conf_data["node_id"]
# net type
nninfo = NNCommonManager().get_nn_id_info(conf_data["nn_id"])
pself.net_type = nninfo[0]['fields']['dir']
graph = NNCommonManager().get_nn_node_name(conf_data["nn_id"])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
pself.netconf_node = net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'netconf_feed':
pself.train_feed_name = pself.nn_id + "_" + pself.wf_ver + "_" + net[
'fields']['graph_node_name']
if net['fields']['graph_node'] == 'eval_feed':
pself.eval_feed_name = pself.nn_id + "_" + pself.wf_ver + "_" + net[
'fields']['graph_node_name']
pself.feed_node = pself.get_prev_node()
# set batch
pself.load_batch = self.get_eval_batch(pself.node_id)
_, pself.train_batch = self.make_batch(pself.node_id)
return pself
def get(self, request, nnid):
"""
Common Network Info Get Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Search deinfed list of neural networks
"""
try:
condition = {}
condition['nn_id'] = nnid
if str(nnid).lower() == 'all':
condition['nn_id'] = '%'
elif str(nnid).lower() == 'seq':
condition['nn_id'] = 'seq'
return_data = NNCommonManager().get_nn_info(condition)
logging.info(return_data)
# Data node name
graph = NNCommonManager.get_nn_node_name(None, nnid)
return_param = {}
return_param['fields'] = return_data
return_param['graph'] = graph
return Response(json.dumps(return_param, cls=DjangoJSONEncoder))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_param, cls=DjangoJSONEncoder))
def ngram_mro(self):
graph = NNCommonManager().get_nn_node_name(self.nn_id)
for net in graph:
if net['fields']['graph_node'] == 'netconf_data':
data_node = net['fields']['graph_node_name']
elif net['fields']['graph_node'] == 'netconf_node':
net_node = net['fields']['graph_node_name']
netconf = NNCommonManager().get_nn_node_info(self.nn_id, str(
self.ver), net_node)[0]['fields']['node_config_data']
self.param['list'] = []
self.param['standard'] = float(netconf['standard'])
self.param['datatype'] = netconf['datatype']
self.param['conninfo'] = netconf['conninfo']
if self.param['datatype'] == 'file':
self.get_file_data(data_node)
elif self.param['datatype'] == 'db':
self.get_db_data()
item = []
for val in self.param['list']:
try:
item_tuple = (val['item_code'].strip(),
val['item_leaf'].strip(),
val['item_desc'].strip())
item.append(item_tuple)
except:
logging.info('Error Data' + val['item_code'])
dataset = ngram.NGram(item, key=lambda x: x[2])
dataset = sorted(dataset, key=lambda x: x[0])
findset = ngram.NGram(item, key=lambda x: x[2])
logging.info(
'================================================================================================'
)
return_data = {}
for data in dataset:
findset.remove(data)
result = findset.search(data[2], self.param['standard'])
for r in range(len(result)):
if return_data.get(data[0]) == None:
return_data[data[0]] = {}
return_data[data[0]]['desc'] = data[2]
# logging.info(str(data[0]) + ':' + str(data[2]))
return_data[data[0]][result[r][0][0]] = {
'item_desc': result[r][0][2],
'item_perc': result[r][1]
}
# logging.info(' - '+str(result[r][0][0])+'('+str(result[r][1])+')' + ':' + str(result[r][0][2]))
logging.info(
str(data[0]) + '-' + str(result[r][0][0]) + '(' +
str(result[r][1]) + ')')
return return_data
def get(self, request, nnid, ver):
"""
Your docs
---
# Class Name (must be separated by `---`)
# Description:
- name: name
description: Foobar long description goes here
"""
try:
return_data = NNCommonManager().get_nn_batch_info(nnid, ver)
conv = []
netconf_node = ""
graph = NNCommonManager().get_nn_node_name(nnid)
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
netconf_node = net['fields']['graph_node_name']
model_path = get_model_path(nnid, ver, netconf_node)
for row in return_data:
row["model"] = "N"
filename = row["nn_batch_ver_id"]
for fn in os.listdir(model_path):
fnsplit = fn.split(".")
fnsplitName = fnsplit[0]
if (fnsplitName == filename):
row["model"] = fnsplitName
row["model_exists"] = "Y"
if row["result_info"] != None:
predicts = row["result_info"]["predicts"]
tfTot = 0
fTot = 0
tTot = 0
for i in range(len(predicts)):
for j in range(len(predicts[i])):
if i == j:
tTot += predicts[i][j]
else:
fTot += predicts[i][j]
row["true_cnt"] = tTot
row["false_cnt"] = fTot
tfTot = tTot + fTot
row["true_false_cnt"] = tfTot
if tfTot == 0:
tfTot = 1
row["true_false_percent"] = round(tTot / tfTot * 100)
conv.append(row)
return Response(json.dumps(conv))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, nnid):
"""
Common Network Info Post Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Define new neural network process
"""
try:
input_parm = request.data
max_nnid = NNCommonManager().get_nn_id_max() + 1
if nnid == "new":
nnid = "nn" + str(max_nnid).zfill(8)
else:
return_data = NNCommonManager().get_nn_id_info(nnid)
if return_data != []:
return Response(
json.dumps(nnid + " Network ID already exists"))
input_parm['nn_id'] = nnid
if input_parm.get('automl_parms') == None:
input_parm['automl_parms'] = {}
if input_parm.get('automl_runtime') == None:
input_parm['automl_runtime'] = {}
if input_parm.get('automl_stat') == None:
input_parm['automl_stat'] = {}
input_parm_s = {}
input_parm_s['id'] = max_nnid
input_parm_s['nn_id'] = nnid
return_data = NNCommonManager().insert_nn_info(
input_parm, input_parm_s)
# Data node name
graph = NNCommonManager.get_nn_node_name(None, return_data)
return_param = {}
return_param['nn_id'] = return_data
return_param['graph'] = graph
return Response(json.dumps(return_param))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
finally:
graph = NNCommonManager().get_nn_node_name(nnid)
for net in graph:
if net['fields']['graph_node'] in [
'netconf_data', 'eval_data'
]:
utils.get_source_path(nnid, None,
net['fields']['graph_node_name'])
def post(self, request, nnid):
"""
- desc : insert cnn configuration data
"""
try:
input_data = request.data
input_data['nn_id'] = nnid
nnManager = NNCommonManager()
nn_wf_ver_id = nnManager.get_nn_max_ver(nnid) + 1
input_data['nn_wf_ver_id'] = nn_wf_ver_id
return_data = nnManager.insert_nn_wf_info(input_data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def run(self, conf_data):
"""
executed on cluster run
:param conf_data:
:return:
"""
try:
# get related nodes
net_node = self.get_prev_node(grp='netconf')
data_node = self.get_prev_node(grp='preprocess')
self._init_node_parm(conf_data['node_id'])
# set result info cls
result = TrainSummaryInfo(type=self.eval_result_type)
result.set_nn_wf_ver_id(conf_data['wf_ver'])
result.set_nn_id(conf_data['nn_id'])
# run eval for each network
result = net_node[0].eval(conf_data['node_id'],
conf_data,
data=data_node[0],
result=result)
if result is None or result == '':
return {}
# set parms for db store
input_data = TrainSummaryInfo.save_result_info(self, result)
input_data['accuracy'] = result.get_accuracy()
condition_data = {}
condition_data['nn_wf_ver_id'] = conf_data['wf_ver']
condition_data[
'condition'] = "3" # 1 Pending, 2 Progress, 3 Finish, 4 Error
# Net Version create
NNCommonManager().update_nn_wf_info(conf_data['nn_id'],
condition_data)
return input_data
except Exception as e:
condition_data = {}
condition_data['nn_wf_ver_id'] = conf_data['wf_ver']
condition_data[
'condition'] = "4" # 1 Pending, 2 Progress, 3 Finish, 4 Error
# Net Version create
NNCommonManager().update_nn_wf_info(conf_data['nn_id'],
condition_data)
logging.error(e)
raise Exception(e)
def get(self, request, nnid, ver):
"""
Your docs
---
# Class Name (must be separated by `---`)
# Description:
- name: name
description: Foobar long description goes here
"""
try:
return_data = NNCommonManager().get_nn_batch_info(nnid, ver)
conv = []
node = WorkFlowSimpleManager().get_train_node()
model_path = get_model_path(nnid, ver, node)
for row in return_data:
row["model"] = "N"
filename = row["nn_batch_ver_id"]
for fn in os.listdir(model_path):
fnsplit = fn.split(".")
fnsplitName = fnsplit[0]
if (fnsplitName == filename):
row["model"] = fn
row["model_exists"] = "Y"
conv.append(row)
return Response(json.dumps(conv))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def _init_train_parm(self, conf_data):
# get initial value
self.conf_data = conf_data
self.cls_pool = conf_data["cls_pool"]
self.nn_id = conf_data["nn_id"]
self.wf_ver = conf_data["wf_ver"]
self.node_id = conf_data["node_id"]
graph = NNCommonManager().get_nn_node_name(conf_data["nn_id"])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
self.netconf_node = net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'netconf_feed':
self.train_feed_name = self.nn_id + "_" + self.wf_ver + "_" + net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'eval_feed':
self.eval_feed_name = self.nn_id + "_" + self.wf_ver + "_" + net['fields']['graph_node_name']
self.feed_node = self.get_prev_node()
#key = conf_data.
wf_net_conf = WorkFlowNetConfXgboost(self.node_id )
self.conf = wf_net_conf.conf
self.data_store_path = utils.get_store_path(self.net_id, self.net_ver, "data_node")
self.data_store_eval_path = utils.get_store_path(self.net_id, self.net_ver, 'evaldata')
self.model_path =self.conf.get('model_path')
self.model_type = self.conf.get('model_type')
print("")
def get(self, request, nnid):
"""
Common Network Info Get Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Search deinfed list of neural networks
"""
try:
condition = {}
condition['nn_id'] = nnid
if str(nnid).lower() == 'all':
condition['nn_id'] = '%'
elif str(nnid).lower() == 'seq':
condition['nn_id'] = 'seq'
return_data = NNCommonManager().get_nn_info(condition)
logging.info(return_data)
# Data node name
graph = NNCommonManager.get_nn_node_name(None, nnid)
return_param = {}
return_param['fields'] = return_data
return_param['graph'] = graph
return Response(json.dumps(return_param, cls=DjangoJSONEncoder))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_param, cls=DjangoJSONEncoder))
def set_train_finish(self, ver_data_sets):
"""
update state flag for version level info on database
:param input_data : raw info we had on database
:return:version info
"""
try :
for input_data in ver_data_sets :
# Create Version
nnCommonManager = NNCommonManager()
del input_data['nn_id']
input_data['condition'] = "3" # 1 Pending, 2 Progress, 3 Finish, 4 Error
# Net Version create
nnCommonManager.update_nn_wf_info(self.nn_id, input_data)
return True
except Exception as e :
return False
def update_summary(self, networks, survive):
"""
update summary info of processing genetic algorithm
arranged by generation (sequece of list means number of generation
each node contains extra info like survive or not
:param networks: networks for one generations
:param survive: number of gene to survive
:return: dict type result info with extra flag
"""
# Regression sorting order add
if self.net_type.rfind("_reg") != -1:
networksNew = copy.deepcopy(
sorted(networks, key=lambda x: x.get('acc'), reverse=False))
else:
networksNew = copy.deepcopy(
sorted(networks, key=lambda x: x.get('acc'), reverse=True))
# result = list(map(lambda x : self.set_value(x, 'survive', True) , networks[0:survive]))
# self.summary['best'] = result
# result = result + list(map(lambda x : self.set_value(x, 'survive', False) , networks[survive:]))
# self.summary['bygen'].append(result)
surcnt = 1
result = []
best = []
for net in networksNew:
if survive < surcnt:
net['survive'] = False
else:
net['survive'] = True
best.append(net)
if survive == surcnt:
self.summary['best'] = best
result.append(net)
surcnt += 1
self.summary['bygen'].append(result)
AutoMlCommon().update_stat_obj(self.nn_id, self.summary)
# Set Best Active
nnCommonManager = NNCommonManager()
input_data = {}
input_data['active_flag'] = "Y"
input_data['nn_wf_ver_id'] = networksNew[0].get('nn_wf_ver_id')
nnCommonManager.update_nn_wf_info(networksNew[0].get('nn_id'),
input_data)
def set_train_finish(self, ver_data_sets):
"""
update state flag for version level info on database
:param input_data : raw info we had on database
:return:version info
"""
try:
for input_data in ver_data_sets:
# Create Version
nnCommonManager = NNCommonManager()
del input_data['nn_id']
input_data[
'condition'] = "3" # 1 Pending, 2 Progress, 3 Finish, 4 Error
# Net Version create
nnCommonManager.update_nn_wf_info(self.nn_id, input_data)
return True
except Exception as e:
return False
def get(self, request, nnid, wfver, desc):
"""
- desc : get data
"""
try:
return_data = NNCommonManager().get_nn_node_info(nnid, wfver, desc)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def get(self, request, nnid):
"""
- desc : get cnn configuration data
"""
try:
return_data = NNCommonManager().get_nn_wf_info(nnid)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def update_summary(self, networks, survive):
"""
update summary info of processing genetic algorithm
arranged by generation (sequece of list means number of generation
each node contains extra info like survive or not
:param networks: networks for one generations
:param survive: number of gene to survive
:return: dict type result info with extra flag
"""
# Regression sorting order add
if self.net_type.rfind("_reg") != -1:
networksNew = copy.deepcopy(sorted(networks, key=lambda x: x.get('acc'), reverse=False))
else:
networksNew = copy.deepcopy(sorted(networks, key=lambda x: x.get('acc'), reverse=True))
# result = list(map(lambda x : self.set_value(x, 'survive', True) , networks[0:survive]))
# self.summary['best'] = result
# result = result + list(map(lambda x : self.set_value(x, 'survive', False) , networks[survive:]))
# self.summary['bygen'].append(result)
surcnt = 1
result = []
best = []
for net in networksNew:
if survive < surcnt:
net['survive'] = False
else:
net['survive'] = True
best.append(net)
if survive == surcnt:
self.summary['best'] = best
result.append(net)
surcnt += 1
self.summary['bygen'].append(result)
AutoMlCommon().update_stat_obj(self.nn_id, self.summary)
# Set Best Active
nnCommonManager = NNCommonManager()
input_data = {}
input_data['active_flag'] = "Y"
input_data['nn_wf_ver_id'] = networksNew[0].get('nn_wf_ver_id')
nnCommonManager.update_nn_wf_info(networksNew[0].get('nn_id'), input_data)
def delete(self, request, nnid):
"""
- desc : delete cnn configuration data
"""
try:
input_parm = request.data
input_parm['nn_id'] = nnid
return_data = NNCommonManager().delete_nn_info(input_parm)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, nnid):
"""
Your docs
---
# Class Name (must be separated by `---`)
# Description:
- name: name
description: Foobar long description goes here
"""
try:
input_data = request.data
input_data['nn_id'] = nnid
nnManager = NNCommonManager()
nn_wf_ver_id = nnManager.get_nn_max_ver(nnid) + 1
input_data['nn_wf_ver_id'] = nn_wf_ver_id
return_data = nnManager.insert_nn_wf_info(input_data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, nnid):
"""
Common Network Version Info Post Method
---
# Class Name : CommonNNInfoVersion
# Description:
Structure : nninfo - <version> - batch version
need to define version info under network definition
"""
try:
input_data = request.data
input_data['nn_id'] = nnid
nnManager = NNCommonManager()
nn_wf_ver_id = nnManager.get_nn_max_ver(nnid) + 1
input_data['nn_wf_ver_id'] = nn_wf_ver_id
return_data = nnManager.insert_nn_wf_info(input_data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def create_version(self, generation):
"""
create version on database
:param generation:
:return:
"""
# Create Version
nnCommonManager = NNCommonManager()
nn_wf_ver_id = nnCommonManager.get_nn_max_ver(self.nn_id) + 1
input_data = {}
input_data['nn_wf_ver_id'] = nn_wf_ver_id
input_data['nn_def_list_info_nn_id'] = generation
input_data['automl_gen'] = generation
input_data['condition'] = "2" # 1 Pending, 2 Progress, 3 Finish, 4 Error
input_data['active_flag'] = "N"
# input_data['nn_wf_ver_desc'] = " ".join(["id:",self.nn_id, "gen:", generation, "ver:", str(nn_wf_ver_id)])
input_data['nn_id'] = self.nn_id
# Net Version create
nnCommonManager.insert_nn_wf_info(input_data)
return nn_wf_ver_id, input_data
def create_version(self, generation):
"""
create version on database
:param generation:
:return:
"""
# Create Version
nnCommonManager = NNCommonManager()
nn_wf_ver_id = nnCommonManager.get_nn_max_ver(self.nn_id) + 1
input_data = {}
input_data['nn_wf_ver_id'] = nn_wf_ver_id
input_data['nn_def_list_info_nn_id'] = generation
input_data['automl_gen'] = generation
input_data[
'condition'] = "2" # 1 Pending, 2 Progress, 3 Finish, 4 Error
input_data['active_flag'] = "N"
# input_data['nn_wf_ver_desc'] = " ".join(["id:",self.nn_id, "gen:", generation, "ver:", str(nn_wf_ver_id)])
input_data['nn_id'] = self.nn_id
# Net Version create
nnCommonManager.insert_nn_wf_info(input_data)
return nn_wf_ver_id, input_data
def get(self, request, nnid):
"""
Common Network Version Info Get Method
---
# Class Name : CommonNNInfoVersion
# Description:
Structure : nninfo - <version> - batch version
Get version information of selected nnid
"""
try:
return_data = NNCommonManager().get_nn_wf_info(nnid)
conv = []
netconf_node = ""
graph = NNCommonManager().get_nn_node_name(nnid)
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
netconf_node = net['fields']['graph_node_name']
for row in return_data:
row["model"] = "N"
train_filename = row["train_batch_ver_id"]
pred_filename = row["pred_batch_ver_id"]
ver = str(row["nn_wf_ver_id"])
model_path = get_model_path(nnid, ver, netconf_node)
for fn in os.listdir(model_path):
fnsplit = fn.split(".")
fnsplitName = fnsplit[0]
if (fnsplitName == train_filename):
row["train_model"] = fn
row["train_model_exists"] = "Y"
if (fnsplitName == pred_filename):
row["pred_model"] = fn
row["pred_model_exists"] = "Y"
conv.append(row)
return Response(json.dumps(conv))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, type, nnid, ver):
"""
Request Deep Neural Network to predict result with given data \n
input formats can be varies on type of networks \n
but usually you can use it with parm input_data \n
---
# Class Name : ServiceManagerPredict
# Description:
request predict service via rest service
It caches the model and vectors on first request
It may can take some time at first for caching, after than we can response the request
within 1.0 sec
"""
try:
if ver == 'active':
condition = {'nn_id': nnid}
ver = NNCommonManager().get_nn_info(
condition)[0]['nn_wf_ver_id']
if (type == "resnet" or type == "inceptionv4"):
return_data = PredictNetImage().run(nnid, ver, request)
elif (type == 'w2v'):
return_data = PredictNetW2V().run(nnid, request.data)
elif (type == "d2v"):
return_data = PredictNetD2V().run(nnid, request.data)
elif (type == "cnn"):
return_data = PredictNetCnn().run(nnid, ver, request.FILES)
elif (type == "wdnn"):
return_data = PredictNetWdnn().run(nnid, ver, request)
elif (type == "seq2seq"):
return_data = PredictNetSeq2Seq().run(nnid, request.data)
elif (type == "autoencoder"):
return_data = PredictNetAutoEncoder().run(nnid, request.data)
elif (type == "anomaly"):
return_data = PredictNetAnomaly().run(nnid, request.data)
elif (type == "wcnn"):
return_data = PredictNetWcnn().run(nnid, request.data)
elif (type == "bilstmcrf"):
return_data = PredictNetBiLstmCrf().run(nnid, request.data)
elif (type == "ngram_mro"):
return_data = PredictNetNgram().run(type, nnid, ver,
request.data)
elif (type == "xgboost_reg"):
return_data = PredictNetXgboost().run(nnid, ver, request.FILES)
return Response(return_data)
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, nnid):
"""
Common Network Version Info Post Method
---
# Class Name : CommonNNInfoVersion
# Description:
Structure : nninfo - <version> - batch version
need to define version info under network definition
"""
try:
input_data = request.data
input_data['nn_id'] = nnid
nnManager = NNCommonManager()
nn_wf_ver_id = nnManager.get_nn_max_ver(nnid) + 1
input_data['nn_wf_ver_id'] = nn_wf_ver_id
if(nn_wf_ver_id == 1 and input_data['nn_wf_ver_info'] == 'single'):
input_data['active_flag'] = 'Y'
return_data = nnManager.insert_nn_wf_info(input_data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, nnid):
"""
Common Network Info Post Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Define new neural network process
"""
try:
input_parm = request.data
max_nnid = NNCommonManager().get_nn_id_max() + 1
if nnid == "":
nnid = "nn" + str(max_nnid).zfill(8)
else:
return_data = NNCommonManager().get_nn_id_info(nnid)
if return_data != []:
return Response(
json.dumps(nnid + " Network ID already exists"))
input_parm['nn_id'] = nnid
if input_parm.get('automl_parms') == None:
input_parm['automl_parms'] = {}
if input_parm.get('automl_runtime') == None:
input_parm['automl_runtime'] = {}
if input_parm.get('automl_stat') == None:
input_parm['automl_stat'] = {}
input_parm_s = {}
input_parm_s['id'] = max_nnid
input_parm_s['nn_id'] = nnid
return_data = NNCommonManager().insert_nn_info(
input_parm, input_parm_s)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, nnid):
"""
Your docs
---
# Class Name (must be separated by `---`)
# Description:
- name: name
description: Foobar long description goes here
"""
try:
input_parm = request.data
max_nnid = NNCommonManager().get_nn_id_max() + 1
if nnid == "":
nnid = "nn" + str(max_nnid).zfill(8)
else:
return_data = NNCommonManager().get_nn_id_info(nnid)
if return_data != []:
return Response(
json.dumps(nnid + " Network ID already exists"))
input_parm['nn_id'] = nnid
if input_parm.get('automl_parms') == None:
input_parm['automl_parms'] = {}
if input_parm.get('automl_runtime') == None:
input_parm['automl_runtime'] = {}
if input_parm.get('automl_stat') == None:
input_parm['automl_stat'] = {}
input_parm_s = {}
input_parm_s['id'] = max_nnid
input_parm_s['nn_id'] = nnid
return_data = NNCommonManager().insert_nn_info(
input_parm, input_parm_s)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def run(self, type, nn_id, ver, parm):
"""
run predict service
1. get node id
2. check json conf format
3. run predict & return
:param parm:
:return:
"""
if(self._valid_check(parm)) :
if ver == 'active':
condition = {'nn_id': nn_id}
ver = NNCommonManager().get_nn_info(condition)[0]['nn_wf_ver_id']
return ThirdPartyNgram().predict(type, nn_id, ver)
def get(self, request, nnid):
"""
- desc : get cnn configuration data
"""
try:
condition = {}
condition['nn_id'] = nnid
if str(nnid).lower() == 'all':
condition['nn_id'] = '%'
return_data = NNCommonManager().get_nn_info(condition)
logging.info(return_data)
return Response(json.dumps(return_data, cls=DjangoJSONEncoder))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data, cls=DjangoJSONEncoder))
def post(self, request, nnid):
"""
- desc : insert cnn configuration data
"""
try:
input_parm = request.data
input_parm['nn_id'] = nnid
input_parm['automl_parms'] = {}
input_parm['automl_runtime'] = {}
input_parm['automl_stat'] = {}
return_data = NNCommonManager().insert_nn_info(input_parm)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def run(self):
"""
run automl
:return:
"""
try:
# get genetic algorithm hyper parameters
generation = self.parm_info.get('generation')
population = self.parm_info.get('population')
survive = self.parm_info.get('survive')
# define gene list
networks = []
# iterate generation
for idx in range(generation):
# create new generations
if (idx == 0):
gen_nets, ver_data_sets = self.create_networks(
idx, population)
networks = networks + gen_nets
AutoMlCommon().reset_stat_obj(self.nn_id)
else:
gen_nets, ver_data_sets = self.create_networks(
idx, population - survive)
networks = networks + gen_nets
# train & evaluate networks
initial = True if idx == 0 else False
networks = self.train_networks(networks, initial=initial)
# set each train set flag to fin
self.set_train_finish(ver_data_sets)
# network acc set
accArr = 0
for net in networks:
return_data = NNCommonManager().get_nn_batch_info(
net['nn_id'], net['nn_wf_ver_id'])
re_acc = return_data[0]['acc_info']['acc']
networks[accArr]['acc'] = float(re_acc[len(re_acc) - 1])
accArr += 1
# print(return_data)
# update traing progress
self.update_summary(networks, survive)
# sort & discard
networks = self.discard_inferior(networks, survive)
return networks
except Exception as e:
logging.error("Error on running AutoML alogorithm : {0}".format(e))
def post(self, request, nnid):
"""
Common Network Info Post Method
---
# Class Name : CommonNNInfoList
# Description:
Structure : <nninfo> - version - batch version
Define new neural network process
"""
try:
input_parm = request.data
max_nnid = NNCommonManager().get_nn_id_max() + 1
if nnid == "new":
nnid = "nn" + str(max_nnid).zfill(8)
else:
return_data = NNCommonManager().get_nn_id_info(nnid)
if return_data != []:
return Response(json.dumps(nnid+" Network ID already exists"))
input_parm['nn_id'] = nnid
if input_parm.get('automl_parms') == None:
input_parm['automl_parms'] = {}
if input_parm.get('automl_runtime') == None:
input_parm['automl_runtime'] = {}
if input_parm.get('automl_stat') == None:
input_parm['automl_stat'] = {}
input_parm_s = {}
input_parm_s['id'] = max_nnid
input_parm_s['nn_id'] = nnid
return_data = NNCommonManager().insert_nn_info(input_parm, input_parm_s)
# Data node name
graph = NNCommonManager.get_nn_node_name(None, return_data)
return_param = {}
return_param['nn_id'] = return_data
return_param['graph'] = graph
return Response(json.dumps(return_param))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
finally:
graph = NNCommonManager().get_nn_node_name(nnid)
for net in graph:
if net['fields']['graph_node'] in ['netconf_data','eval_data']:
utils.get_source_path(nnid, None, net['fields']['graph_node_name'])
def put(self, request, nnid):
"""
Common Network Version Info Delete Method
---
# Class Name : CommonNNInfoVersion
# Description:
Structure : nninfo - <version> - batch version
Modify seleted nnid's information
"""
try:
return_data = NNCommonManager().update_nn_wf_info(nnid, request.data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def _init_train_parm(self, conf_data):
# get initial value
self.conf_data = conf_data
self.cls_pool = conf_data["cls_pool"]
self.nn_id = conf_data["nn_id"]
self.wf_ver = conf_data["wf_ver"]
self.node_id = conf_data["node_id"]
graph = NNCommonManager().get_nn_node_name(conf_data["nn_id"])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
self.netconf_node = net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'netconf_feed':
self.train_feed_name = self.nn_id + "_" + self.wf_ver + "_" + net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'eval_feed':
self.eval_feed_name = self.nn_id + "_" + self.wf_ver + "_" + net['fields']['graph_node_name']
self.feed_node = self.get_prev_node()
def put(self, request, nnid, ver):
"""
Your docs
---
# Class Name (must be separated by `---`)
# Description:
- name: name
description: Foobar long description goes here
"""
try:
return_data = NNCommonManager().update_nn_batch_info(
nnid, ver, request.data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def delete(self, request, nnid):
"""
Common Network Version Info Delete Method
---
# Class Name : CommonNNInfoVersion
# Description:
Structure : nninfo - <version> - batch version
delete selected network info and related data
"""
try:
return_data = NNCommonManager().delete_nn_wf_info(request.data)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def delete(self, request, nnid):
"""
Your docs
---
# Class Name (must be separated by `---`)
# Description:
- name: name
description: Foobar long description goes here
"""
try:
input_parm = request.data
input_parm['nn_id'] = nnid
return_data = NNCommonManager().delete_nn_info(input_parm)
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))