def get_all_info_json_by_nnid(self,nnid):
"""get network configuration info json
:param nnid
:return: json string """
result = netconf.get_network_config(nnid)
utils.tfmsa_logger("((Wdnn_common_model_get_json_by_nnid)) ## END##")
return result
def get_predict_data(self, nn_id, predict_data):
"""
(1) get net column descritions
(2) modify predict data for 'categorical data'
(3) caculate size of arrays need for neural networks
:param nn_id:neural network id want to train
:return: Train Data Sets
"""
try:
tfmsa_logger("modifying predict start!")
# (1) get data configuration info
net_conf = netconf.get_network_config(nn_id)
datadesc = JsonDataConverter().load_obj_json(net_conf['datadesc'])
datasets = JsonDataConverter().load_obj_json(net_conf['datasets'])
# (2) modify train data for 'categorical data'
self.m_train[:] = []
self.m_tag[:] = []
self.m_train, self.m_tag = self.reform_train_data(
predict_data, datasets, datadesc)
# (3) caculate size of arrays need for neural networks
self.train_len = len(next(iter(self.m_train), None))
tfmsa_logger("modified predict data : {0} ".format(self.m_train))
return self
except Exception as e:
print("Exception {0} , {1}".format(self.__class__, e))
raise Exception(e)
def test_insert_train_acc(self):
tfmsa_logger("================TEST START================")
init_data = {
"nn_id": "test00001",
"category": "MES",
"subcate": "csv",
"name": "CENSUS_INCOME",
"desc": "INCOME PREDICT"
}
netconf.create_new_network(init_data)
tfmsa_logger("[1] insert net base info : Done")
now = datetime.datetime.now()
nowDate = now.strftime('%Y-%m-%d %H:%M:%S')
logInfoData = dict()
logInfoData["nn_id"] = "test00001"
logInfoData["label"] = str("A")
logInfoData["guess"] = str("B")
logInfoData["ratio"] = str("0.5")
for i in range(0, 10):
netconf.post_train_acc(logInfoData)
tfmsa_logger("[2] insert train loss info : Done")
result = netconf.get_train_acc("test00001")
self.assertGreater(len(result), 0)
tfmsa_logger("[3] check train loss inserted right : Done")
tfmsa_logger("================TEST FINISH================")
def get_json_by_nnid(self,nnid):
"""get network configuration info json
:param nnid
:return: json string """
datadesc = netconf.load_ori_format(nnid)
result = json.loads(datadesc)
utils.tfmsa_logger("((Wdnn_common_model_get_json_by_nnid)) ## END##")
return result
def __init__(self,
p_nn_id,
p_max_steps,
p_every_n_steps=const.LOG_OUT_STEPS,
p_first_row_check=True):
utils.tfmsa_logger(" ## MonitorCommon Class Init ##")
monitors.EveryN.__init__(self, every_n_steps=p_every_n_steps)
self.max_steps = p_max_steps
self.nn_id = p_nn_id
self.first_row_check = p_first_row_check
def every_n_step_end(self, step, outputs):
utils.tfmsa_logger("MonitorCommon.every_n_step_end ---> Loggin " +
str(step))
now = datetime.datetime.now()
nowDate = now.strftime('%Y-%m-%d %H:%M:%S')
logInfoData = dict()
logInfoData["nn_id"] = self.nn_id
logInfoData["loss"] = str(self.get_loss(outputs))
logInfoData["step"] = str(step)
logInfoData["max_step"] = self.max_steps
logInfoData["trainDate"] = nowDate
logInfoData["testsets"] = "1"
#print(logInfoData)
self.monitors_update(logInfoData)
def get_init_info_wdnn(self, nnid):
""" Get infomation of Wdnn initial
:param nnid
:param model_dir : directory of chkpoint of wdnn model
"""
json_string = netconf.load_ori_format(nnid)
json_object = json.loads(json_string)
utils.tfmsa_logger("[4]load net conf form db")
conf = netconf.load_conf(nnid)
hidden_layers_value = conf.layer
result_temp = netconf.get_network_config(nnid)
label_cnt = json.loads(json.dumps(result_temp))
label_object = label_cnt["datasets"]
utils.tfmsa_logger("((1.Make WDN Network Build)) set up Hidden Layers (" + str(hidden_layers_value) + ")")
return conf, hidden_layers_value, json_object, label_object
def prepare_test_image_data(self, nn_id, net_info):
"""
prepare image type data for test
convert image to analizerble array
:param nn_id:
:return:
"""
# check current data pointer
utils.tfmsa_logger("[1]check current data pointer")
"""
TO-DO : case when data size is big
"""
# get train data from HDFS
utils.tfmsa_logger("[2]load data from hdfs")
row_data_arr = ImageManager().load_data("test_schema_" + net_info['dir'], net_info['table'], "0", "10")
# conver image to array
utils.tfmsa_logger("[3]convert image to array")
out_index = json.loads(net_info['datasets'])
train_data_set = []
train_label_set = []
for row_data in row_data_arr:
train_data_set.append(json.loads(str(row_data['bt'],'utf-8')))
train_label_set.append(netcommon.return_index_position(out_index, str(row_data['label'], 'utf-8')))
return train_data_set, train_label_set
def get_train_data(self, nn_id):
"""
(1) get net column descritions
(2) get user selected data , exclude user check None
(3) modify train data for 'categorical data'
(4) caculate size of arrays need for neural networks
(5) change neural network configurioatns automtically
:param nn_id:neural network id want to train
:return: Train Data Sets
"""
try:
tfmsa_logger("modifying Train start!")
# (1) get data configuration info
net_conf = netconf.get_network_config(nn_id)
datadesc = JsonDataConverter().load_obj_json(net_conf['datadesc'])
datasets = JsonDataConverter().load_obj_json(net_conf['datasets'])
# (2) get user seleceted data from spark
sql_stmt = self.get_sql_state(datadesc, net_conf['table'])
origin_data = HbaseManager().query_data(net_conf['dir'],
net_conf['table'],
sql_stmt)
# (3) modify train data for 'categorical data'
self.m_train[:] = []
self.m_tag[:] = []
self.m_train, self.m_tag = self.reform_train_data(
origin_data, datasets, datadesc)
# (4) caculate size of arrays need for neural networks
self.train_len = len(next(iter(self.m_train), None))
self.tag_len = len(next(iter(self.m_tag), None))
tfmsa_logger("modifying Train End!")
return self
except IOError as e:
return e
def test_insert_train_loss(self):
tfmsa_logger("================TEST START================")
init_data = {
"nn_id": "test00001",
"category": "MES",
"subcate": "csv",
"name": "CENSUS_INCOME",
"desc": "INCOME PREDICT"
}
netconf.create_new_network(init_data)
# just consider worsk fine if no exception occurs
netconf.set_on_net_conf("test00001")
netconf.set_on_net_vaild("test00001")
netconf.set_on_train("test00001")
netconf.set_on_data_conf("test00001")
netconf.set_on_data("test00001")
result = netconf.get_network_config("test00001")
idx_set = ["datavaild", "config", "train", "confvaild"]
for idx in idx_set:
self.assertEqual(result[idx], 'Y')
netconf.set_off_net_conf("test00001")
netconf.set_off_net_vaild("test00001")
netconf.set_off_train("test00001")
netconf.set_off_data_conf("test00001")
netconf.set_off_data("test00001")
result = netconf.get_network_config("test00001")
idx_set = ["datavaild", "config", "train", "confvaild"]
for idx in idx_set:
self.assertEqual(result[idx], 'N')
tfmsa_logger("================TEST END================")
def monitors_update(self, logInfoData):
""" Monitor update
:param nnid
:param model_dir : directory of chkpoint of wdnn model
"""
try:
utils.tfmsa_logger(
"MonitorCommon.monitors_update_end ---> postgres")
#bool firstRowFlag = True
print(self.first_row_check)
if self.first_row_check == True:
#netconf.delete_train_loss(self.nn_id)
utils.tfmsa_logger(
"MonitorCommon.monitors_delete_firstCheck ---> postgres")
self.first_row_check = False
print(self.first_row_check)
body = json.loads(json.dumps(logInfoData))
return_data = netconf.post_train_loss(body)
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
finally:
return return_data
def predict_conv_network(nn_id, predict_data):
try:
# check network is ready to train
utils.tfmsa_logger("[1]check pre steps ready")
utils.check_requested_nn(nn_id)
# get network base info
utils.tfmsa_logger("[2]get network base info")
net_info = netconf.get_network_config(nn_id)
# get network format info
utils.tfmsa_logger("[3]get network format info")
conf_info = netconf.load_conf(nn_id)
learnrate = conf_info.data.learnrate
label_set = json.loads(net_info['datasets'])
conf_info.n_class = len(label_set)
# define classifier
utils.tfmsa_logger("[4]define classifier")
classifier = learn.Estimator(
model_fn=ConvCommonManager(conf_info).struct_cnn_layer,
model_dir=netconf.nn_model_manager.get_model_save_path(nn_id),
config=learn.RunConfig(save_checkpoints_secs=1))
# start train
#TODO : need to find way to predict without fit
utils.tfmsa_logger("[5]fit dummy")
train_x = np.array([
ConvCommonManager(conf_info).create_dummy_matrix(
len(predict_data[0]))
], np.float32)
train_y = np.array(
netcommon.convert_to_index(json.loads(net_info['datasets'])),
np.int32)
classifier.fit(train_x, train_y, steps=int(1))
# predict result
utils.tfmsa_logger("[6]predict result")
y_predicted = [
label_set[int(p['class'])]
for p in classifier.predict(x=np.array(predict_data, np.float32),
batch_size=1,
as_iterable=True)
]
return y_predicted
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def struct_cnn_layer(self, train_data_set, train_label_set):
"""
dynamically struct cnn
:param num_layers:
:param conf_info:
:return:
"""
conf_info = self.conf
num_layers = len(conf_info.layer)
matrix = conf_info.data.matrix.copy()
train_data_set = tf.reshape(train_data_set, [-1, matrix[0], matrix[1], 1])
curren_matrix = matrix
curren_matrix_num = 0
train_label_set = tf.one_hot(train_label_set, int(conf_info.n_class), 1, 0)
for i in range(0, int(num_layers)):
data = conf_info.layer[i]
utils.tfmsa_logger("[{0}]define layers : {1}".format(i, data.type))
if (data.type == "cnn" and i ==0):
train_data_set = tf.reshape(train_data_set, [-1, matrix[0], matrix[1], 1])
network = tf.contrib.layers.conv2d(train_data_set,
num_outputs=data.node_in_out[1],
kernel_size=data.cnnfilter,
activation_fn=self.get_activation(str(data.active)))
network = tf.nn.max_pool(network,
ksize=[1, data.maxpoolmatrix[0], data.maxpoolmatrix[1], 1],
strides=[1, data.maxpoolstride[0], data.maxpoolstride[1], 1],
padding=data.padding)
curren_matrix = self.mat_size_cal(curren_matrix, data.padding, data.maxpoolmatrix, data.maxpoolstride)
curren_matrix_num = data.node_in_out[1]
elif (data.type == "cnn"):
network = tf.contrib.layers.conv2d(network,
num_outputs=data.node_in_out[1],
kernel_size=data.cnnfilter,
activation_fn=self.get_activation(str(data.active)))
network = tf.nn.max_pool(network, ksize=[1, data.maxpoolmatrix[0], data.maxpoolmatrix[1], 1],
strides=[1, data.maxpoolstride[0], data.maxpoolstride[1], 1],
padding=data.padding)
curren_matrix = self.mat_size_cal(curren_matrix, data.padding, data.maxpoolmatrix, data.maxpoolstride)
curren_matrix_num = data.node_in_out[1]
elif (data.type == "reshape"):
network = tf.reshape(network, [-1, curren_matrix[0] * curren_matrix[1] * curren_matrix_num])
elif (data.type == "drop"):
data_num = curren_matrix[0] * curren_matrix[1] * curren_matrix_num
network = tf.contrib.layers.dropout(
tf.contrib.layers.legacy_fully_connected(
network, data_num, weight_init=None,
activation_fn=self.get_activation(str(data.active))),
keep_prob=float(data.droprate)
)
elif (data.type == "out"):
network = learn.models.logistic_regression(x=network, y=train_label_set)
else:
raise SyntaxError("there is no such kind of layer type : " + str(data.type))
with tf.name_scope('loss'):
prediction, loss = (network)
with tf.name_scope('train_op'):
train_op = tf.contrib.layers.optimize_loss(
loss, tf.contrib.framework.get_global_step(), optimizer='Adagrad',
learning_rate=0.1)
return {'class': tf.argmax(prediction, 1), 'prob': prediction}, loss, train_op
def input_fn(self, df, nnid):
"""Wide & Deep Network input tensor maker
V1.0 16.11.04 Initial
:param df : dataframe from hbase
:param df, nnid
:return: tensor sparse, constraint """
try:
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## START ##")
##Make List for Continuous, Categorical Columns
CONTINUOUS_COLUMNS = []
CATEGORICAL_COLUMNS = []
##Get datadesc Continuous and Categorical infomation from Postgres nninfo
json_string = self.get_json_by_nnid(nnid)
json_object = json_string
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## JSON CELL FEATURE LOADS ##")
j_feature = json_object['cell_feature']
for cn, c_value in j_feature.items():
if c_value["column_type"] == "CATEGORICAL":
CATEGORICAL_COLUMNS.append(cn)
elif c_value["column_type"] == "CONTINUOUS":
CONTINUOUS_COLUMNS.append(cn)
elif c_value["column_type"] =="CATEGORICAL_KEY":
CATEGORICAL_COLUMNS.append(cn)
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## SPARSE TENSOR ##" + "".join(CATEGORICAL_COLUMNS))
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## REAL VALUE TENSOR ##" + "".join(CONTINUOUS_COLUMNS))
# Check Continuous Column is exsist?
if len(CONTINUOUS_COLUMNS)>0 :
#print(CONTINUOUS_COLUMNS)
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CONTINUES : " + str(CONTINUOUS_COLUMNS))
continuous_cols = {k: tf.constant(df[k].values) for k in CONTINUOUS_COLUMNS}
# Check Categorical Column is exsist?
if len(CATEGORICAL_COLUMNS) > 0 :
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CATEGORICAL(SPARSE TENSOR) ")
for k in CATEGORICAL_COLUMNS:
df[k] = df[k].astype('str')
categorical_cols = {k: tf.SparseTensor(
indices=[[i, 0] for i in range(df[k].size)],
values=df[k].values,
shape=[df[k].size, 1])
for k in CATEGORICAL_COLUMNS}
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ####### CATEGORICAL(SPARSE TENSOR) END")
# Merges the two dictionaries into one.
feature_cols = {}
if(len(CONTINUOUS_COLUMNS)>0):
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CONTINUE ADD LIST : " + str(CONTINUOUS_COLUMNS))
feature_cols.update(continuous_cols)
if len(CATEGORICAL_COLUMNS) > 0:
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CATEGORICAL ADD LIST" + str(CATEGORICAL_COLUMNS))
feature_cols.update(categorical_cols)
#Get label distinct list from postgres 16.12.04
json_string = WdnnCommonManager.get_all_info_json_by_nnid(self, nnid=nnid)
_label_list = json_string['datasets']
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## Make Label List ##")
label_list = eval(_label_list)
le = LabelEncoder()
le.fit(label_list)
lable_encoder_func = lambda x: le.transform([x])
df['label'] = df['label'].map(lable_encoder_func).astype(int)
label_encode = le.transform(label_list)
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## Label Convert ##" + str(label_list) + "ENCODE ---> " + str(label_encode))
label = tf.constant(df["label"].values)
utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## END ##")
return feature_cols, label
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def end(self):
utils.tfmsa_logger("MonitorCommon.end --> Completed run.")
def __init__(self):
utils.tfmsa_logger("Create Wdnn eval ")
WdnnCommonManager.__init__(self)
def wdnn_build(self,nnid, model_dir = "No", train=True):
""" wide & deep netowork builder
:param nnid
:param model_dir : directory of chkpoint of wdnn model
:param train : train or predict
:return: tensorflow network model """
try:
utils.tfmsa_logger("((0.0 Wide & Deep Network wdnn build))")
conf,hidden_layers_value,json_object, label_object = self.get_init_info_wdnn(nnid)
#Check Train or Predict
if(train):
#model_dir = settings.HDFS_MODEL_ROOT + "/"+nnid + "/"+tempfile.mkdtemp().split("/")[2]
model_dir = settings.HDFS_MODEL_ROOT + "/" + nnid
else:
if(model_dir != "No"):
model_dir = model_dir
utils.tfmsa_logger("((1.Make WDN Network Build)) set up WDNN directory("+nnid +") ---> " + model_dir)
label_cnt = len(list(label_object))
utils.tfmsa_logger("((1.Make WDN Network Build)) set up WDNN label count("+nnid +") ---> " + str(label_cnt))
# continuous, categorical and embeddingforCategorical(deep) list
featureColumnCategorical = {}
featureColumnContinuous = {}
featureDeepEmbedding={}
j_feature = json_object["cell_feature"]
utils.tfmsa_logger("((1.Make WDN Network Build)) Get Column DataType (" + nnid + ") ---> " + str(j_feature))
utils.tfmsa_logger("((1.Make WDN Network Build)) set up Hidden Layers (" + str(hidden_layers_value) + ")")
for cn, c_value in j_feature.items(): #change 3.5python
utils.tfmsa_logger("((1.Make WDN Network Build)) first get feature columns [" + cn + "]: " + str(c_value["column_type"]))
if c_value["column_type"] == "CATEGORICAL":
featureColumnCategorical[cn] = tf.contrib.layers.sparse_column_with_hash_bucket(
cn, hash_bucket_size=1000)
elif c_value["column_type"] == "CATEGORICAL_KEY":
featureColumnCategorical[cn] = tf.contrib.layers.sparse_column_with_keys(column_name=cn,keys=c_value["keys"])
elif c_value["column_type"] == "CONTINUOUS": #CONTINUOUS
featureColumnContinuous[cn] = tf.contrib.layers.real_valued_column(cn)
# embedding column add
for key, value in featureColumnCategorical.items(): #3.5python
utils.tfmsa_logger("((1.Make WDN Network Build)) Categorical Embedding add ["+ str(key)+"]")
featureDeepEmbedding[key] = tf.contrib.layers.embedding_column(value, dimension=8)
wide_columns = []
for sparseTensor in featureColumnCategorical:
wide_columns.append(featureColumnCategorical[sparseTensor])
# cross_cell checks null
cross_col1 = []
if 'cross_cell' in json_object: #json_object.has_key('cross_cell'):
j_cross = json_object["cross_cell"]
for jc, values in j_cross.items():
utils.tfmsa_logger("((1.Make WDN Network Build)) Cross rows " + str(values) )
for c_key, c_value in values.items(): #3.5python
cross_col1.append(featureColumnCategorical[c_value])
wide_columns.append(tf.contrib.layers.crossed_column(cross_col1,hash_bucket_size=int(1e4)))
##Transformations column for wide
transfomation_col= {}
if 'Transformations' in json_object: #json_object.has_key('Transformations'):
j_boundaries = json_object["Transformations"]
for jc, values in j_boundaries.items(): #3.5python
utils.tfmsa_logger("((1-1.Make WDN Network Build)) TransForm Columns " + str(values))
trans_col_name = values["column_name"]
trans_boundaries = values["boundaries"]
utils.tfmsa_logger("((1-1 get age columns )) ")
utils.tfmsa_logger(type(featureColumnContinuous[trans_col_name]))
rvc = featureColumnContinuous[trans_col_name]
transfomation_col[jc] = tf.contrib.layers.bucketized_column(featureColumnContinuous[trans_col_name],trans_boundaries)
wide_columns.append(tf.contrib.layers.bucketized_column(featureColumnContinuous[trans_col_name],trans_boundaries))
utils.tfmsa_logger("((1-1 transform tensor insert))")
deep_columns = []
for realTensor in featureColumnContinuous:
deep_columns.append(featureColumnContinuous[realTensor])
for embeddingTensor in featureDeepEmbedding:
deep_columns.append(featureDeepEmbedding[embeddingTensor])
if FLAGS.model_type == "wide_n_deep":
utils.tfmsa_logger("((1-1 wide_n_deep Selected)############)")
m = tf.contrib.learn.DNNLinearCombinedClassifier(
model_dir=model_dir,
linear_feature_columns=wide_columns,
dnn_feature_columns=deep_columns,
n_classes=label_cnt, # 0.11 bug
dnn_hidden_units=hidden_layers_value)
elif FLAGS.model_type == "wide":
utils.tfmsa_logger("((1-1 wide Selected)############)")
m = tf.contrib.learn.LinearClassifier(model_dir=model_dir,
feature_columns=wide_columns
,enable_centered_bias = True)
elif FLAGS.model_type =="deep":
utils.tfmsa_logger("((1-1 deep Selected)############)")
m = tf.contrib.learn.DNNClassifier(model_dir=model_dir,
feature_columns=deep_columns,
n_classes = label_cnt, #0.11 bug
hidden_units=hidden_layers_value)
rv = self.network_update(nnid,model_dir)
utils.tfmsa_logger("((1.Make WDN Network Build)) wdnn directory info update sucess")
return m
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def wdd_predict(self, nnid, filename=None):
"""
Wide & Deep Network predict
:param nnid : network id in tfmsacore_nninfo
:return: json list with predict value
"""
try:
utils.tfmsa_logger(
"((0.0 Wide & Deep Network Predict Start)) ## " + nnid + "##")
json_string = WdnnCommonManager.get_all_info_json_by_nnid(
self, nnid)
database = str(json_string['dir'])
table_name = str(json_string['table'])
json_string_desc = netconf.load_ori_format(
nnid) #Get format json string
column_format_json_ob = json.loads(
json_string_desc
) # Cast json object from string of column formats
temp_label = column_format_json_ob[
'label'] #Get label column from data formats
label_column = list(temp_label.keys())[0]
#should be change after deciding model directory
model_dir = str(json_string['query'])
_label_list = json_string['datasets']
label_list = eval(_label_list) #Cast a list from string
utils.tfmsa_logger(
"((0.0 Wide & Deep Network label List Get)) ## " +
str(label_list) + "##")
wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir,
False)
if filename == None:
#Doesn;t need this part now
limit_no = 100
print("((2.Get Dataframe from Hbase)) ##Start## (" + database +
" , " + table_name + " , " + label_column + ")")
df = data.DataMaster().query_data(database,
table_name,
"a",
limit_no,
with_label=label_column)
print("((2.Get Dataframe from Hbase)) ##End## (" + database +
" , " + table_name + " , " + label_column + " , " +
str(limit_no) + ")")
else:
utils.tfmsa_logger("((2.Get Dataframe from CSV)) ##Start##" +
nnid + " , " + filename + ")")
file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename
utils.tfmsa_logger(
"((2.Get Dataframe from CSV)) ##filePath###" + file_path +
")")
df = pd.read_csv(
tf.gfile.Open(file_path),
# names=COLUMNS,
skipinitialspace=True,
engine="python")
df['label'] = df[label_column]
results = dict() #Make Return Dictionary
predict_results = wdnn_model.predict(
input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid))
df['predict_label'] = list(predict_results)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Predict Complete)) ## " + nnid +
"##")
le = LabelEncoder()
le.fit(label_list)
lable_decoder_func = lambda x: le.inverse_transform(x)
df['predict_label'] = df['predict_label'].map(
lable_decoder_func).astype(
"str") #convert label code to label string.
label_value = list(df['label'].unique())
label_encode = le.inverse_transform(label_value)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Predict)) ## Label Convert ##" +
str(label_value) + "DECODE ---> " + str(label_encode))
resultList = df.values.tolist()
resultList.insert(0, df.columns.values.tolist())
results = json.loads(json.dumps(resultList))
return results
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def eval_conv_network(nn_id, samplenum=0.1, samplemethod=1):
try:
# check network is ready to train
utils.tfmsa_logger("[1]check pre steps ready")
utils.check_requested_nn(nn_id)
# get network base info
utils.tfmsa_logger("[2]get network base info")
net_info = netconf.get_network_config(nn_id)
# get network format info
utils.tfmsa_logger("[3]get network format info")
conf_info = netconf.load_conf(nn_id)
# load train data
utils.tfmsa_logger("[4]load train data")
train_data_set = []
train_label_set = []
# TODO : need to change data loader to get sample data (not all data)
if (const.TYPE_IMAGE == net_info['preprocess']):
train_data_set, train_label_set = ConvCommonManager(
conf_info).prepare_test_image_data(nn_id, net_info)
elif (const.TYPE_DATA_FRAME == net_info['preprocess']):
raise Exception("function not ready")
elif (const.TYPE_TEXT == net_info['preprocess']):
raise Exception("function not ready")
else:
raise Exception("unknown data type")
# data size info change
utils.tfmsa_logger("[5]modify data stuctor info")
#ConvCommonManager(conf_info).save_changed_data_info(nn_id, train_data_set)
learnrate = conf_info.data.learnrate
label_set = json.loads(net_info['datasets'])
conf_info.n_class = len(label_set)
# change to nummpy array
train_x = np.array(train_data_set, np.float32)
train_y = np.array(train_label_set, np.int32)
# define classifier
utils.tfmsa_logger("[6]define classifier")
classifier = learn.Estimator(
model_fn=ConvCommonManager(conf_info).struct_cnn_layer,
model_dir=netconf.nn_model_manager.get_model_save_path(nn_id))
# start train
#TODO : need to find way to predict without fit
utils.tfmsa_logger("[5]fit dummy")
dummy_x = np.array([
ConvCommonManager(conf_info).create_dummy_matrix(len(train_x[0]))
], np.float32)
dummy_y = np.array(
netcommon.convert_to_index(json.loads(net_info['datasets'])),
np.int32)
classifier.fit(dummy_x, dummy_y, steps=int(1))
# start train
utils.tfmsa_logger("[8]evaluate prediction result")
counter = 0
acc_result_obj = AccStaticResult()
for p in classifier.predict(x=np.array(train_x, np.float32),
batch_size=1,
as_iterable=True):
acc_result_obj = AccEvalCommon(nn_id).set_result(
acc_result_obj, label_set[train_y[counter]],
label_set[int(p['class'])])
counter = counter + 1
return len(train_y)
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def train_conv_network(nn_id, epoch=50, testset=100):
try:
# check network is ready to train
utils.tfmsa_logger("[1]check pre steps ready")
utils.check_requested_nn(nn_id)
# get network base info
utils.tfmsa_logger("[2]get network base info")
net_info = netconf.get_network_config(nn_id)
# get network format info
utils.tfmsa_logger("[3]get network format info")
conf_info = netconf.load_conf(nn_id)
# load train data
utils.tfmsa_logger("[4]load train data")
train_data_set = []
train_label_set = []
if (const.TYPE_IMAGE == net_info['preprocess']):
train_data_set, train_label_set = ConvCommonManager(
conf_info).prepare_image_data(nn_id, net_info)
elif (const.TYPE_DATA_FRAME == net_info['preprocess']):
raise Exception("function not ready")
elif (const.TYPE_TEXT == net_info['preprocess']):
raise Exception("function not ready")
else:
raise Exception("unknown data type")
# data size info change
utils.tfmsa_logger("[5]modify data stuctor info")
# ConvCommonManager(conf_info).save_changed_data_info(nn_id, train_data_set)
learnrate = conf_info.data.learnrate
conf_info.n_class = len(json.loads(net_info['datasets']))
# change to nummpy array
train_x = np.array(np.array(train_data_set).astype(float), np.float32)
train_y = np.array(train_label_set, np.int32)
# define classifier
utils.tfmsa_logger("[6]define classifier")
classifier = learn.Estimator(
model_fn=ConvCommonManager(conf_info).struct_cnn_layer,
model_dir=netconf.nn_model_manager.get_model_save_path(nn_id))
# start train
utils.tfmsa_logger("[7]fit CNN")
customsMonitor = Monitors.MonitorCommon(p_nn_id=nn_id,
p_max_steps=10000,
p_every_n_steps=100)
classifier.fit(train_x, train_y, steps=1000, monitors=[customsMonitor])
return len(train_y)
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def wdd_eval(self, nnid, filename=None):
"""
Wide & Deep Network predict
:param nnid : network id in tfmsacore_nninfo
:return: acturacy
"""
try:
utils.tfmsa_logger("((3.1 Wide & Deep Network Eval Start)) ## " +
nnid + "##")
json_string = WdnnCommonManager.get_all_info_json_by_nnid(
self, nnid=nnid)
database = json_string['dir']
table_name = json_string['table']
json_string_desc = netconf.load_ori_format(nnid)
json_ob = json.loads(json_string_desc) #get datadesc format
model_dir = str(json_string['query'])
label_column = list(json_ob["label"].keys())[0]
_label_list = json_string['datasets']
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## DB : " + database +
" TABLE : " + table_name)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Model_Dir : " +
model_dir)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Label Column " +
label_column + " Label : " + _label_list)
label_list = eval(_label_list)
wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir,
False)
if filename == None:
df, last_key = data.DataMaster().query_data(
"test_schema_" + database,
table_name,
"a",
use_df=True,
limit_cnt=-1,
with_label=label_column)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Get Test Schema Data Count"
+ str(len(df)) + "LastKey : " + str(last_key))
else:
#Don't use this pat
print("((2.Get Dataframe from CSV)) ##Start## (" + nnid +
" , " + filename + ")")
file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename
print("((2.Get Dataframe from CSV)) ##filePath## (" +
file_path + ")")
print(file_path)
df = pd.read_csv(
tf.gfile.Open(file_path),
# names=COLUMNS,
skipinitialspace=True,
engine="python")
df['label'] = (
df[label_column].apply(lambda x: "Y" in x)).astype(int)
results = {}
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Start)) ## Start Predict in Eval Method ##"
)
predict_results = wdnn_model.predict(
input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid))
df['predict_label'] = list(predict_results)
#print(df['predict_label'] )
utils.tfmsa_logger(
"make label encorder function ######################")
#
#Decode Label(int) to Label(str)
le = LabelEncoder()
le.fit(label_list)
lable_decoder_func = lambda x: le.inverse_transform([x])
df['predict_label'] = df['predict_label'].map(
lable_decoder_func).astype("str")
label_value = list(df['label'].unique())
label_encode = le.inverse_transform(label_value)
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Predict)) ## Label Convert ##" +
str(label_value) + "DECODE ---> " + str(label_encode))
#results_acc table manager class create
acc_result_obj = AccStaticResult()
log_count = 0
for value in df.iterrows():
log_count += 1
#print("Inside start row by row")
#print(type(value))
_row = value[1]
ori_label = str(_row[label_column])
predict_label = str(eval(_row["predict_label"])
[0]) #str(list(_row["predict_label"])[0])
#predict_label = le.inverse_transform(_predict_label)
if (log_count % 20 == 0):
utils.tfmsa_logger(
"((3.1 Wide & Deep Network Eval Columns Insert)) " +
str(ori_label) + "------->" + str(predict_label))
acc_result_obj = AccEvalCommon(nnid).set_result(
acc_result_obj, ori_label, predict_label)
return results
except Exception as e:
print("Error Message : {0}".format(e))
raise Exception(e)
def __init__(self):
utils.tfmsa_logger("Create Wdnn common ")
