def get(self, request, nnid, type):
"""
- desc : return network data format information
"""
#get_network_config
try:
result_temp = netconf.get_network_config(nnid)
datadesc = netconf.load_ori_format(nnid)
result_datadesc_source = json.loads(datadesc)
result = dict()
if type == "cell_feature":
result = result_datadesc_source["cell_feature"]
elif type == "label":
result = result_datadesc_source["label"]
elif type == "all":
result = result_datadesc_source["cell_feature"]
result.update(result_datadesc_source["label"])
elif type == "labels":
result = data.ImageManager().get_label_list(nnid)
return_data = {"status": "200", "result": result}
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "400", "result": str(e)}
return Response(json.dumps(return_data))
def get_json_by_nnid(self,nnid):
"""get network configuration info json
:param nnid
:return: json string """
datadesc = netconf.load_ori_format(nnid)
result = json.loads(datadesc)
utils.tfmsa_logger("((Wdnn_common_model_get_json_by_nnid)) ## END##")
return result
def get(self, request, baseid, table, nnid):
"""
- desc : return network data format information
"""
try:
result = netconf.load_ori_format(nnid)
return_data = {"status": "200", "result": json.loads(result)}
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "400", "result": str(e)}
return Response(json.dumps(return_data))
def get_init_info_wdnn(self, nnid):
""" Get infomation of Wdnn initial
:param nnid
:param model_dir : directory of chkpoint of wdnn model
"""
json_string = netconf.load_ori_format(nnid)
json_object = json.loads(json_string)
utils.tfmsa_logger("[4]load net conf form db")
conf = netconf.load_conf(nnid)
hidden_layers_value = conf.layer
result_temp = netconf.get_network_config(nnid)
label_cnt = json.loads(json.dumps(result_temp))
label_object = label_cnt["datasets"]
utils.tfmsa_logger("((1.Make WDN Network Build)) set up Hidden Layers (" + str(hidden_layers_value) + ")")
return conf, hidden_layers_value, json_object, label_object
def wdd_eval(self, nnid, filename=None):
"""
Wide & Deep Network predict
:param nnid : network id in tfmsacore_nninfo
:return: acturacy
"""
try:
utils.tfmsa_logger("((3.1 Wide & Deep Network Eval Start)) ## " +
nnid + "##")
json_string = WdnnCommonManager.get_all_info_json_by_nnid(
self, nnid=nnid)
database = json_string['dir']
table_name = json_string['table']
json_string_desc = netconf.load_ori_format(nnid)
json_ob = json.loads(json_string_desc) #get datadesc format
model_dir = str(json_string['query'])
label_column = list(json_ob["label"].keys())[0]
_label_list = json_string['datasets']
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## DB : " + database +
" TABLE : " + table_name)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Model_Dir : " +
model_dir)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Label Column " +
label_column + " Label : " + _label_list)
label_list = eval(_label_list)
wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir,
False)
if filename == None:
df, last_key = data.DataMaster().query_data(
"test_schema_" + database,
table_name,
"a",
use_df=True,
limit_cnt=-1,
with_label=label_column)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Get Test Schema Data Count"
+ str(len(df)) + "LastKey : " + str(last_key))
else:
#Don't use this pat
print("((2.Get Dataframe from CSV)) ##Start## (" + nnid +
" , " + filename + ")")
file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename
print("((2.Get Dataframe from CSV)) ##filePath## (" +
file_path + ")")
print(file_path)
df = pd.read_csv(
tf.gfile.Open(file_path),
# names=COLUMNS,
skipinitialspace=True,
engine="python")
df['label'] = (
df[label_column].apply(lambda x: "Y" in x)).astype(int)
results = {}
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Start Predict in Eval Method ##"
)
predict_results = wdnn_model.predict(
input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid))
df['predict_label'] = list(predict_results)
#print(df['predict_label'] )
utils.tfmsa_logger(
"make label encorder function ######################")
#
#Decode Label(int) to Label(str)
le = LabelEncoder()
le.fit(label_list)
lable_decoder_func = lambda x: le.inverse_transform([x])
df['predict_label'] = df['predict_label'].map(
lable_decoder_func).astype("str")
label_value = list(df['label'].unique())
label_encode = le.inverse_transform(label_value)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Predict)) ## Label Convert ##" +
str(label_value) + "DECODE ---> " + str(label_encode))
#results_acc table manager class create
acc_result_obj = AccStaticResult()
log_count = 0
for value in df.iterrows():
log_count += 1
#print("Inside start row by row")
#print(type(value))
_row = value[1]
ori_label = str(_row[label_column])
predict_label = str(eval(_row["predict_label"])
[0]) #str(list(_row["predict_label"])[0])
#predict_label = le.inverse_transform(_predict_label)
if (log_count % 20 == 0):
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Columns Insert)) " +
str(ori_label) + "------->" + str(predict_label))
acc_result_obj = AccEvalCommon(nnid).set_result(
acc_result_obj, ori_label, predict_label)
return results
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def put_data(self, data_frame, table_name, label, file_set, nnid):
"""
delete label folder under table
:param db_name:
:param table_name:
:return:
"""
try:
tfmsa_logger("[1]Start upload images...")
self.make_inital_path(nnid)
# get network base info
tfmsa_logger("[2]get network base info")
net_info = netconf.get_network_config(nnid)
# get data format info
tfmsa_logger("[3]get network format info")
format_info = json.loads(netconf.load_ori_format(nnid))
# get hbase trasaction table
tfmsa_logger("[4]get hbase trasaction table")
conn, train_table, test_table = self.get_divided_target_table(
data_frame, table_name)
train_buffer = train_table.batch(transaction=True)
test_buffer = test_table.batch(transaction=True)
#get Label list
tfmsa_logger("[5]Updata Label List ")
self.label_info_update(net_info, label)
# get Label list
tfmsa_logger("[6]upload image on Hbase - start ")
file_list = []
train_key_set, test_key_set = self.divide_train_sample(
file_set.keys())
for key in file_set.keys():
file = file_set[key]
row_value = dict()
row_key = table_name + ":" + self.make_hbasekey()
byte_buffer, width, height = self.image_preprocess(
file, net_info, format_info, label)
row_value[':'.join(
('data', 'filebyte'))] = str(list(byte_buffer))
row_value[':'.join(('data', 'label'))] = str(label)
row_value[':'.join(
('data', 'decoder'))] = str(key).split(".")[1]
row_value[':'.join(('data', 'width'))] = str(width)
row_value[':'.join(('data', 'height'))] = str(height)
file_list.append(file._name)
if (key in train_key_set):
train_buffer.put(row_key, row_value)
if (key in test_key_set):
test_buffer.put(row_key, row_value)
train_buffer.send()
test_buffer.send()
tfmsa_logger("[7]upload image on Hbase - finish")
return file_list
except Exception as e:
tfmsa_logger("Error : {0}".format(e))
raise Exception(e)
finally:
conn.close()
tfmsa_logger("Finish upload image...")
def post(self, request, baseid, tb, nnid):
"""
- desc : create a format data
- desc : update data format information \n
<textfield>
<font size = 1>
{ "cross_cell":
{
"col12": {"column2_0": "native_country", "column2_1": "occupation"},
"col1": {"column_1": "occupation", "column_0": "education"}
},
"cell_feature":
{
"hours_per_week": "CONTINUOUS_COLUMNS",
"native_country": "CATEGORICAL",
"relationship": "CATEGORICAL",
"gender": "CATEGORICAL",
"age": "CONTINUOUS_COLUMNS",
"marital_status": "CATEGORICAL",
"race": "CATEGORICAL",
"capital_gain": "CONTINUOUS_COLUMNS",
"workclass": "CATEGORICAL",
"capital_loss": "CONTINUOUS_COLUMNS",
"education": "CATEGORICAL",
"education_num": "CONTINUOUS_COLUMNS",
"occupation": "CATEGORICAL"
},
"label":
{
"income_bracket" : "LABEL"
}
}
</font>
</textfield>
---
parameters:
- name: body
paramType: body
pytype: json
"""
try:
jd = jc.load_obj_json("{}")
jd.dir = baseid
jd.table = tb
jd.nn_id = nnid
jd.preprocess = '1'
jd.datadesc = 'Y'
coll_format_json = dict()
cell_format = str(request.body, 'utf-8')
#if format info null
print(len(request.body))
print(request.body)
if (len(cell_format) == 2):
print("request is null ###################")
json_string = netconf.load_ori_format(nnid)
coll_format_json = json.loads(json_string)
cell_format = json_string
else:
print("request is not null ###################")
coll_format_json = json.loads(cell_format)
print("print cell format")
print(cell_format)
netconf.save_format(nnid, cell_format)
print("dataformat called1###################")
t_label = coll_format_json['label']
label_column = list(t_label.keys())[0]
print("dataformat called2###################" + str(label_column))
# lable column_count check
lable_list = data.DataMaster().get_distinct_label(
baseid, tb, label_column)
#hbase query
lable_sorted_list = sorted(list(lable_list))
jd.datasets = lable_sorted_list
#netconf.save_format(nnid, str(request.body,'utf-8'))
result = netconf.update_network(jd)
netconf.set_on_data(nnid)
return_data = {"status": "200", "result": result}
return Response(json.dumps(return_data))
except Exception as e:
netconf.set_off_data(nnid)
netconf.set_off_data_conf(nnid)
return_data = {"status": "400", "result": str(e)}
return Response(json.dumps(return_data))
def wdd_predict(self, nnid, filename=None):
"""
Wide & Deep Network predict
:param nnid : network id in tfmsacore_nninfo
:return: json list with predict value
"""
try:
utils.tfmsa_logger(
"((0.0 Wide & Deep Network Predict Start)) ## " + nnid + "##")
json_string = WdnnCommonManager.get_all_info_json_by_nnid(
self, nnid)
database = str(json_string['dir'])
table_name = str(json_string['table'])
json_string_desc = netconf.load_ori_format(
nnid) #Get format json string
column_format_json_ob = json.loads(
json_string_desc
) # Cast json object from string of column formats
temp_label = column_format_json_ob[
'label'] #Get label column from data formats
label_column = list(temp_label.keys())[0]
#should be change after deciding model directory
model_dir = str(json_string['query'])
_label_list = json_string['datasets']
label_list = eval(_label_list) #Cast a list from string
utils.tfmsa_logger(
"((0.0 Wide & Deep Network label List Get)) ## " +
str(label_list) + "##")
wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir,
False)
if filename == None:
#Doesn;t need this part now
limit_no = 100
print("((2.Get Dataframe from Hbase)) ##Start## (" + database +
" , " + table_name + " , " + label_column + ")")
df = data.DataMaster().query_data(database,
table_name,
"a",
limit_no,
with_label=label_column)
print("((2.Get Dataframe from Hbase)) ##End## (" + database +
" , " + table_name + " , " + label_column + " , " +
str(limit_no) + ")")
else:
utils.tfmsa_logger("((2.Get Dataframe from CSV)) ##Start##" +
nnid + " , " + filename + ")")
file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename
utils.tfmsa_logger(
"((2.Get Dataframe from CSV)) ##filePath###" + file_path +
")")
df = pd.read_csv(
tf.gfile.Open(file_path),
# names=COLUMNS,
skipinitialspace=True,
engine="python")
df['label'] = df[label_column]
results = dict() #Make Return Dictionary
predict_results = wdnn_model.predict(
input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid))
df['predict_label'] = list(predict_results)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Predict Complete)) ## " + nnid +
"##")
le = LabelEncoder()
le.fit(label_list)
lable_decoder_func = lambda x: le.inverse_transform(x)
df['predict_label'] = df['predict_label'].map(
lable_decoder_func).astype(
"str") #convert label code to label string.
label_value = list(df['label'].unique())
label_encode = le.inverse_transform(label_value)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Predict)) ## Label Convert ##" +
str(label_value) + "DECODE ---> " + str(label_encode))
resultList = df.values.tolist()
resultList.insert(0, df.columns.values.tolist())
results = json.loads(json.dumps(resultList))
return results
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def run_wdd_train(self, nnid, start_pnt=1, batch_size=1000):
"""
Wide & Deep Network Training
:param nnid : network id in tfmsacore_nninfo
:return: acturacy
"""
try:
tfmsa_logger("[2] start run wdd_Train " + nnid)
#make wide & deep model
wdnn_model = WdnnCommonManager.wdnn_build(self, nnid=nnid)
#get json from postgres by nnid
json_string = WdnnCommonManager.get_all_info_json_by_nnid(
self, nnid=nnid)
database = json_string["dir"]
table_name = json_string["table"]
#Make NetworkConfiguration Json Objct
json_string = netconf.load_ori_format(nnid)
json_ob = json.loads(json_string)
#get label column from hbase nn config json
t_label = json_ob["label"]
label_column = list(t_label.keys())[0]
#get train hyper param
job_parm = JobStateLoader().get_selected_job_info(nnid)
batch_size = int(job_parm.batchsize)
model_lint_cnt = int(job_parm.epoch)
tfmsa_logger(
"[3] Get Dataframe from Hbase ##Start## {0},{1},{2},{3} ".
format(start_pnt, database, table_name, label_column))
df, pnt = data.DataMaster().query_data(database,
table_name,
"a",
use_df=True,
limit_cnt=batch_size,
with_label=label_column,
start_pnt=start_pnt)
df_eval = df.copy()
tfmsa_logger("[4] Get Dataframe from Hbase ##End## (" +
str(batch_size) + ")")
##MAKE MONITOR
tfmsa_logger("[5] Make Monitor Class")
customsMonitor = Monitors.MonitorCommon(p_nn_id=nnid,
p_max_steps=model_lint_cnt,
p_every_n_steps=1000)
tfmsa_logger("[6] start fitting")
wdnn_model.fit(
input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid),
steps=model_lint_cnt,
monitors=[customsMonitor])
if (len(df_eval) < 10):
tfmsa_logger("[7] Train Result")
results = wdnn_model.evaluate(
input_fn=lambda: WdnnCommonManager.input_fn(
self, df_eval, nnid),
steps=1)
for key in sorted(results):
tfmsa_logger("%s: %s" % (key, results[key]))
return nnid
else:
JobStateLoader().inc_job_data_pointer(nnid)
self.run_wdd_train(nnid=nnid, start_pnt=pnt)
return nnid
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
