def get_all_info_json_by_nnid(self,nnid): """get network configuration info json :param nnid :return: json string """ result = netconf.get_network_config(nnid) utils.tfmsa_logger("((Wdnn_common_model_get_json_by_nnid)) ## END##") return result
def get_predict_data(self, nn_id, predict_data): """ (1) get net column descritions (2) modify predict data for 'categorical data' (3) caculate size of arrays need for neural networks :param nn_id:neural network id want to train :return: Train Data Sets """ try: tfmsa_logger("modifying predict start!") # (1) get data configuration info net_conf = netconf.get_network_config(nn_id) datadesc = JsonDataConverter().load_obj_json(net_conf['datadesc']) datasets = JsonDataConverter().load_obj_json(net_conf['datasets']) # (2) modify train data for 'categorical data' self.m_train[:] = [] self.m_tag[:] = [] self.m_train, self.m_tag = self.reform_train_data( predict_data, datasets, datadesc) # (3) caculate size of arrays need for neural networks self.train_len = len(next(iter(self.m_train), None)) tfmsa_logger("modified predict data : {0} ".format(self.m_train)) return self except Exception as e: print("Exception {0} , {1}".format(self.__class__, e)) raise Exception(e)
def test_insert_train_acc(self): tfmsa_logger("================TEST START================") init_data = { "nn_id": "test00001", "category": "MES", "subcate": "csv", "name": "CENSUS_INCOME", "desc": "INCOME PREDICT" } netconf.create_new_network(init_data) tfmsa_logger("[1] insert net base info : Done") now = datetime.datetime.now() nowDate = now.strftime('%Y-%m-%d %H:%M:%S') logInfoData = dict() logInfoData["nn_id"] = "test00001" logInfoData["label"] = str("A") logInfoData["guess"] = str("B") logInfoData["ratio"] = str("0.5") for i in range(0, 10): netconf.post_train_acc(logInfoData) tfmsa_logger("[2] insert train loss info : Done") result = netconf.get_train_acc("test00001") self.assertGreater(len(result), 0) tfmsa_logger("[3] check train loss inserted right : Done") tfmsa_logger("================TEST FINISH================")
def get_json_by_nnid(self,nnid): """get network configuration info json :param nnid :return: json string """ datadesc = netconf.load_ori_format(nnid) result = json.loads(datadesc) utils.tfmsa_logger("((Wdnn_common_model_get_json_by_nnid)) ## END##") return result
def __init__(self, p_nn_id, p_max_steps, p_every_n_steps=const.LOG_OUT_STEPS, p_first_row_check=True): utils.tfmsa_logger(" ## MonitorCommon Class Init ##") monitors.EveryN.__init__(self, every_n_steps=p_every_n_steps) self.max_steps = p_max_steps self.nn_id = p_nn_id self.first_row_check = p_first_row_check
def every_n_step_end(self, step, outputs): utils.tfmsa_logger("MonitorCommon.every_n_step_end ---> Loggin " + str(step)) now = datetime.datetime.now() nowDate = now.strftime('%Y-%m-%d %H:%M:%S') logInfoData = dict() logInfoData["nn_id"] = self.nn_id logInfoData["loss"] = str(self.get_loss(outputs)) logInfoData["step"] = str(step) logInfoData["max_step"] = self.max_steps logInfoData["trainDate"] = nowDate logInfoData["testsets"] = "1" #print(logInfoData) self.monitors_update(logInfoData)
def get_init_info_wdnn(self, nnid): """ Get infomation of Wdnn initial :param nnid :param model_dir : directory of chkpoint of wdnn model """ json_string = netconf.load_ori_format(nnid) json_object = json.loads(json_string) utils.tfmsa_logger("[4]load net conf form db") conf = netconf.load_conf(nnid) hidden_layers_value = conf.layer result_temp = netconf.get_network_config(nnid) label_cnt = json.loads(json.dumps(result_temp)) label_object = label_cnt["datasets"] utils.tfmsa_logger("((1.Make WDN Network Build)) set up Hidden Layers (" + str(hidden_layers_value) + ")") return conf, hidden_layers_value, json_object, label_object
def prepare_test_image_data(self, nn_id, net_info): """ prepare image type data for test convert image to analizerble array :param nn_id: :return: """ # check current data pointer utils.tfmsa_logger("[1]check current data pointer") """ TO-DO : case when data size is big """ # get train data from HDFS utils.tfmsa_logger("[2]load data from hdfs") row_data_arr = ImageManager().load_data("test_schema_" + net_info['dir'], net_info['table'], "0", "10") # conver image to array utils.tfmsa_logger("[3]convert image to array") out_index = json.loads(net_info['datasets']) train_data_set = [] train_label_set = [] for row_data in row_data_arr: train_data_set.append(json.loads(str(row_data['bt'],'utf-8'))) train_label_set.append(netcommon.return_index_position(out_index, str(row_data['label'], 'utf-8'))) return train_data_set, train_label_set
def get_train_data(self, nn_id): """ (1) get net column descritions (2) get user selected data , exclude user check None (3) modify train data for 'categorical data' (4) caculate size of arrays need for neural networks (5) change neural network configurioatns automtically :param nn_id:neural network id want to train :return: Train Data Sets """ try: tfmsa_logger("modifying Train start!") # (1) get data configuration info net_conf = netconf.get_network_config(nn_id) datadesc = JsonDataConverter().load_obj_json(net_conf['datadesc']) datasets = JsonDataConverter().load_obj_json(net_conf['datasets']) # (2) get user seleceted data from spark sql_stmt = self.get_sql_state(datadesc, net_conf['table']) origin_data = HbaseManager().query_data(net_conf['dir'], net_conf['table'], sql_stmt) # (3) modify train data for 'categorical data' self.m_train[:] = [] self.m_tag[:] = [] self.m_train, self.m_tag = self.reform_train_data( origin_data, datasets, datadesc) # (4) caculate size of arrays need for neural networks self.train_len = len(next(iter(self.m_train), None)) self.tag_len = len(next(iter(self.m_tag), None)) tfmsa_logger("modifying Train End!") return self except IOError as e: return e
def test_insert_train_loss(self): tfmsa_logger("================TEST START================") init_data = { "nn_id": "test00001", "category": "MES", "subcate": "csv", "name": "CENSUS_INCOME", "desc": "INCOME PREDICT" } netconf.create_new_network(init_data) # just consider worsk fine if no exception occurs netconf.set_on_net_conf("test00001") netconf.set_on_net_vaild("test00001") netconf.set_on_train("test00001") netconf.set_on_data_conf("test00001") netconf.set_on_data("test00001") result = netconf.get_network_config("test00001") idx_set = ["datavaild", "config", "train", "confvaild"] for idx in idx_set: self.assertEqual(result[idx], 'Y') netconf.set_off_net_conf("test00001") netconf.set_off_net_vaild("test00001") netconf.set_off_train("test00001") netconf.set_off_data_conf("test00001") netconf.set_off_data("test00001") result = netconf.get_network_config("test00001") idx_set = ["datavaild", "config", "train", "confvaild"] for idx in idx_set: self.assertEqual(result[idx], 'N') tfmsa_logger("================TEST END================")
def monitors_update(self, logInfoData): """ Monitor update :param nnid :param model_dir : directory of chkpoint of wdnn model """ try: utils.tfmsa_logger( "MonitorCommon.monitors_update_end ---> postgres") #bool firstRowFlag = True print(self.first_row_check) if self.first_row_check == True: #netconf.delete_train_loss(self.nn_id) utils.tfmsa_logger( "MonitorCommon.monitors_delete_firstCheck ---> postgres") self.first_row_check = False print(self.first_row_check) body = json.loads(json.dumps(logInfoData)) return_data = netconf.post_train_loss(body) except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e) finally: return return_data
def predict_conv_network(nn_id, predict_data): try: # check network is ready to train utils.tfmsa_logger("[1]check pre steps ready") utils.check_requested_nn(nn_id) # get network base info utils.tfmsa_logger("[2]get network base info") net_info = netconf.get_network_config(nn_id) # get network format info utils.tfmsa_logger("[3]get network format info") conf_info = netconf.load_conf(nn_id) learnrate = conf_info.data.learnrate label_set = json.loads(net_info['datasets']) conf_info.n_class = len(label_set) # define classifier utils.tfmsa_logger("[4]define classifier") classifier = learn.Estimator( model_fn=ConvCommonManager(conf_info).struct_cnn_layer, model_dir=netconf.nn_model_manager.get_model_save_path(nn_id), config=learn.RunConfig(save_checkpoints_secs=1)) # start train #TODO : need to find way to predict without fit utils.tfmsa_logger("[5]fit dummy") train_x = np.array([ ConvCommonManager(conf_info).create_dummy_matrix( len(predict_data[0])) ], np.float32) train_y = np.array( netcommon.convert_to_index(json.loads(net_info['datasets'])), np.int32) classifier.fit(train_x, train_y, steps=int(1)) # predict result utils.tfmsa_logger("[6]predict result") y_predicted = [ label_set[int(p['class'])] for p in classifier.predict(x=np.array(predict_data, np.float32), batch_size=1, as_iterable=True) ] return y_predicted except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def struct_cnn_layer(self, train_data_set, train_label_set): """ dynamically struct cnn :param num_layers: :param conf_info: :return: """ conf_info = self.conf num_layers = len(conf_info.layer) matrix = conf_info.data.matrix.copy() train_data_set = tf.reshape(train_data_set, [-1, matrix[0], matrix[1], 1]) curren_matrix = matrix curren_matrix_num = 0 train_label_set = tf.one_hot(train_label_set, int(conf_info.n_class), 1, 0) for i in range(0, int(num_layers)): data = conf_info.layer[i] utils.tfmsa_logger("[{0}]define layers : {1}".format(i, data.type)) if (data.type == "cnn" and i ==0): train_data_set = tf.reshape(train_data_set, [-1, matrix[0], matrix[1], 1]) network = tf.contrib.layers.conv2d(train_data_set, num_outputs=data.node_in_out[1], kernel_size=data.cnnfilter, activation_fn=self.get_activation(str(data.active))) network = tf.nn.max_pool(network, ksize=[1, data.maxpoolmatrix[0], data.maxpoolmatrix[1], 1], strides=[1, data.maxpoolstride[0], data.maxpoolstride[1], 1], padding=data.padding) curren_matrix = self.mat_size_cal(curren_matrix, data.padding, data.maxpoolmatrix, data.maxpoolstride) curren_matrix_num = data.node_in_out[1] elif (data.type == "cnn"): network = tf.contrib.layers.conv2d(network, num_outputs=data.node_in_out[1], kernel_size=data.cnnfilter, activation_fn=self.get_activation(str(data.active))) network = tf.nn.max_pool(network, ksize=[1, data.maxpoolmatrix[0], data.maxpoolmatrix[1], 1], strides=[1, data.maxpoolstride[0], data.maxpoolstride[1], 1], padding=data.padding) curren_matrix = self.mat_size_cal(curren_matrix, data.padding, data.maxpoolmatrix, data.maxpoolstride) curren_matrix_num = data.node_in_out[1] elif (data.type == "reshape"): network = tf.reshape(network, [-1, curren_matrix[0] * curren_matrix[1] * curren_matrix_num]) elif (data.type == "drop"): data_num = curren_matrix[0] * curren_matrix[1] * curren_matrix_num network = tf.contrib.layers.dropout( tf.contrib.layers.legacy_fully_connected( network, data_num, weight_init=None, activation_fn=self.get_activation(str(data.active))), keep_prob=float(data.droprate) ) elif (data.type == "out"): network = learn.models.logistic_regression(x=network, y=train_label_set) else: raise SyntaxError("there is no such kind of layer type : " + str(data.type)) with tf.name_scope('loss'): prediction, loss = (network) with tf.name_scope('train_op'): train_op = tf.contrib.layers.optimize_loss( loss, tf.contrib.framework.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return {'class': tf.argmax(prediction, 1), 'prob': prediction}, loss, train_op
def input_fn(self, df, nnid): """Wide & Deep Network input tensor maker V1.0 16.11.04 Initial :param df : dataframe from hbase :param df, nnid :return: tensor sparse, constraint """ try: utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## START ##") ##Make List for Continuous, Categorical Columns CONTINUOUS_COLUMNS = [] CATEGORICAL_COLUMNS = [] ##Get datadesc Continuous and Categorical infomation from Postgres nninfo json_string = self.get_json_by_nnid(nnid) json_object = json_string utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## JSON CELL FEATURE LOADS ##") j_feature = json_object['cell_feature'] for cn, c_value in j_feature.items(): if c_value["column_type"] == "CATEGORICAL": CATEGORICAL_COLUMNS.append(cn) elif c_value["column_type"] == "CONTINUOUS": CONTINUOUS_COLUMNS.append(cn) elif c_value["column_type"] =="CATEGORICAL_KEY": CATEGORICAL_COLUMNS.append(cn) utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## SPARSE TENSOR ##" + "".join(CATEGORICAL_COLUMNS)) utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## REAL VALUE TENSOR ##" + "".join(CONTINUOUS_COLUMNS)) # Check Continuous Column is exsist? if len(CONTINUOUS_COLUMNS)>0 : #print(CONTINUOUS_COLUMNS) utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CONTINUES : " + str(CONTINUOUS_COLUMNS)) continuous_cols = {k: tf.constant(df[k].values) for k in CONTINUOUS_COLUMNS} # Check Categorical Column is exsist? if len(CATEGORICAL_COLUMNS) > 0 : utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CATEGORICAL(SPARSE TENSOR) ") for k in CATEGORICAL_COLUMNS: df[k] = df[k].astype('str') categorical_cols = {k: tf.SparseTensor( indices=[[i, 0] for i in range(df[k].size)], values=df[k].values, shape=[df[k].size, 1]) for k in CATEGORICAL_COLUMNS} utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ####### CATEGORICAL(SPARSE TENSOR) END") # Merges the two dictionaries into one. feature_cols = {} if(len(CONTINUOUS_COLUMNS)>0): utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CONTINUE ADD LIST : " + str(CONTINUOUS_COLUMNS)) feature_cols.update(continuous_cols) if len(CATEGORICAL_COLUMNS) > 0: utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## IF CATEGORICAL ADD LIST" + str(CATEGORICAL_COLUMNS)) feature_cols.update(categorical_cols) #Get label distinct list from postgres 16.12.04 json_string = WdnnCommonManager.get_all_info_json_by_nnid(self, nnid=nnid) _label_list = json_string['datasets'] utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## Make Label List ##") label_list = eval(_label_list) le = LabelEncoder() le.fit(label_list) lable_encoder_func = lambda x: le.transform([x]) df['label'] = df['label'].map(lable_encoder_func).astype(int) label_encode = le.transform(label_list) utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## Label Convert ##" + str(label_list) + "ENCODE ---> " + str(label_encode)) label = tf.constant(df["label"].values) utils.tfmsa_logger("((3.1 Wide & Deep Network Make Tensor)) ## END ##") return feature_cols, label except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def end(self): utils.tfmsa_logger("MonitorCommon.end --> Completed run.")
def __init__(self): utils.tfmsa_logger("Create Wdnn eval ") WdnnCommonManager.__init__(self)
def wdnn_build(self,nnid, model_dir = "No", train=True): """ wide & deep netowork builder :param nnid :param model_dir : directory of chkpoint of wdnn model :param train : train or predict :return: tensorflow network model """ try: utils.tfmsa_logger("((0.0 Wide & Deep Network wdnn build))") conf,hidden_layers_value,json_object, label_object = self.get_init_info_wdnn(nnid) #Check Train or Predict if(train): #model_dir = settings.HDFS_MODEL_ROOT + "/"+nnid + "/"+tempfile.mkdtemp().split("/")[2] model_dir = settings.HDFS_MODEL_ROOT + "/" + nnid else: if(model_dir != "No"): model_dir = model_dir utils.tfmsa_logger("((1.Make WDN Network Build)) set up WDNN directory("+nnid +") ---> " + model_dir) label_cnt = len(list(label_object)) utils.tfmsa_logger("((1.Make WDN Network Build)) set up WDNN label count("+nnid +") ---> " + str(label_cnt)) # continuous, categorical and embeddingforCategorical(deep) list featureColumnCategorical = {} featureColumnContinuous = {} featureDeepEmbedding={} j_feature = json_object["cell_feature"] utils.tfmsa_logger("((1.Make WDN Network Build)) Get Column DataType (" + nnid + ") ---> " + str(j_feature)) utils.tfmsa_logger("((1.Make WDN Network Build)) set up Hidden Layers (" + str(hidden_layers_value) + ")") for cn, c_value in j_feature.items(): #change 3.5python utils.tfmsa_logger("((1.Make WDN Network Build)) first get feature columns [" + cn + "]: " + str(c_value["column_type"])) if c_value["column_type"] == "CATEGORICAL": featureColumnCategorical[cn] = tf.contrib.layers.sparse_column_with_hash_bucket( cn, hash_bucket_size=1000) elif c_value["column_type"] == "CATEGORICAL_KEY": featureColumnCategorical[cn] = tf.contrib.layers.sparse_column_with_keys(column_name=cn,keys=c_value["keys"]) elif c_value["column_type"] == "CONTINUOUS": #CONTINUOUS featureColumnContinuous[cn] = tf.contrib.layers.real_valued_column(cn) # embedding column add for key, value in featureColumnCategorical.items(): #3.5python utils.tfmsa_logger("((1.Make WDN Network Build)) Categorical Embedding add ["+ str(key)+"]") featureDeepEmbedding[key] = tf.contrib.layers.embedding_column(value, dimension=8) wide_columns = [] for sparseTensor in featureColumnCategorical: wide_columns.append(featureColumnCategorical[sparseTensor]) # cross_cell checks null cross_col1 = [] if 'cross_cell' in json_object: #json_object.has_key('cross_cell'): j_cross = json_object["cross_cell"] for jc, values in j_cross.items(): utils.tfmsa_logger("((1.Make WDN Network Build)) Cross rows " + str(values) ) for c_key, c_value in values.items(): #3.5python cross_col1.append(featureColumnCategorical[c_value]) wide_columns.append(tf.contrib.layers.crossed_column(cross_col1,hash_bucket_size=int(1e4))) ##Transformations column for wide transfomation_col= {} if 'Transformations' in json_object: #json_object.has_key('Transformations'): j_boundaries = json_object["Transformations"] for jc, values in j_boundaries.items(): #3.5python utils.tfmsa_logger("((1-1.Make WDN Network Build)) TransForm Columns " + str(values)) trans_col_name = values["column_name"] trans_boundaries = values["boundaries"] utils.tfmsa_logger("((1-1 get age columns )) ") utils.tfmsa_logger(type(featureColumnContinuous[trans_col_name])) rvc = featureColumnContinuous[trans_col_name] transfomation_col[jc] = tf.contrib.layers.bucketized_column(featureColumnContinuous[trans_col_name],trans_boundaries) wide_columns.append(tf.contrib.layers.bucketized_column(featureColumnContinuous[trans_col_name],trans_boundaries)) utils.tfmsa_logger("((1-1 transform tensor insert))") deep_columns = [] for realTensor in featureColumnContinuous: deep_columns.append(featureColumnContinuous[realTensor]) for embeddingTensor in featureDeepEmbedding: deep_columns.append(featureDeepEmbedding[embeddingTensor]) if FLAGS.model_type == "wide_n_deep": utils.tfmsa_logger("((1-1 wide_n_deep Selected)############)") m = tf.contrib.learn.DNNLinearCombinedClassifier( model_dir=model_dir, linear_feature_columns=wide_columns, dnn_feature_columns=deep_columns, n_classes=label_cnt, # 0.11 bug dnn_hidden_units=hidden_layers_value) elif FLAGS.model_type == "wide": utils.tfmsa_logger("((1-1 wide Selected)############)") m = tf.contrib.learn.LinearClassifier(model_dir=model_dir, feature_columns=wide_columns ,enable_centered_bias = True) elif FLAGS.model_type =="deep": utils.tfmsa_logger("((1-1 deep Selected)############)") m = tf.contrib.learn.DNNClassifier(model_dir=model_dir, feature_columns=deep_columns, n_classes = label_cnt, #0.11 bug hidden_units=hidden_layers_value) rv = self.network_update(nnid,model_dir) utils.tfmsa_logger("((1.Make WDN Network Build)) wdnn directory info update sucess") return m except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def wdd_predict(self, nnid, filename=None): """ Wide & Deep Network predict :param nnid : network id in tfmsacore_nninfo :return: json list with predict value """ try: utils.tfmsa_logger( "((0.0 Wide & Deep Network Predict Start)) ## " + nnid + "##") json_string = WdnnCommonManager.get_all_info_json_by_nnid( self, nnid) database = str(json_string['dir']) table_name = str(json_string['table']) json_string_desc = netconf.load_ori_format( nnid) #Get format json string column_format_json_ob = json.loads( json_string_desc ) # Cast json object from string of column formats temp_label = column_format_json_ob[ 'label'] #Get label column from data formats label_column = list(temp_label.keys())[0] #should be change after deciding model directory model_dir = str(json_string['query']) _label_list = json_string['datasets'] label_list = eval(_label_list) #Cast a list from string utils.tfmsa_logger( "((0.0 Wide & Deep Network label List Get)) ## " + str(label_list) + "##") wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir, False) if filename == None: #Doesn;t need this part now limit_no = 100 print("((2.Get Dataframe from Hbase)) ##Start## (" + database + " , " + table_name + " , " + label_column + ")") df = data.DataMaster().query_data(database, table_name, "a", limit_no, with_label=label_column) print("((2.Get Dataframe from Hbase)) ##End## (" + database + " , " + table_name + " , " + label_column + " , " + str(limit_no) + ")") else: utils.tfmsa_logger("((2.Get Dataframe from CSV)) ##Start##" + nnid + " , " + filename + ")") file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename utils.tfmsa_logger( "((2.Get Dataframe from CSV)) ##filePath###" + file_path + ")") df = pd.read_csv( tf.gfile.Open(file_path), # names=COLUMNS, skipinitialspace=True, engine="python") df['label'] = df[label_column] results = dict() #Make Return Dictionary predict_results = wdnn_model.predict( input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid)) df['predict_label'] = list(predict_results) utils.tfmsa_logger( "((3.1 Wide & Deep Network Predict Complete)) ## " + nnid + "##") le = LabelEncoder() le.fit(label_list) lable_decoder_func = lambda x: le.inverse_transform(x) df['predict_label'] = df['predict_label'].map( lable_decoder_func).astype( "str") #convert label code to label string. label_value = list(df['label'].unique()) label_encode = le.inverse_transform(label_value) utils.tfmsa_logger( "((3.1 Wide & Deep Network Predict)) ## Label Convert ##" + str(label_value) + "DECODE ---> " + str(label_encode)) resultList = df.values.tolist() resultList.insert(0, df.columns.values.tolist()) results = json.loads(json.dumps(resultList)) return results except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def eval_conv_network(nn_id, samplenum=0.1, samplemethod=1): try: # check network is ready to train utils.tfmsa_logger("[1]check pre steps ready") utils.check_requested_nn(nn_id) # get network base info utils.tfmsa_logger("[2]get network base info") net_info = netconf.get_network_config(nn_id) # get network format info utils.tfmsa_logger("[3]get network format info") conf_info = netconf.load_conf(nn_id) # load train data utils.tfmsa_logger("[4]load train data") train_data_set = [] train_label_set = [] # TODO : need to change data loader to get sample data (not all data) if (const.TYPE_IMAGE == net_info['preprocess']): train_data_set, train_label_set = ConvCommonManager( conf_info).prepare_test_image_data(nn_id, net_info) elif (const.TYPE_DATA_FRAME == net_info['preprocess']): raise Exception("function not ready") elif (const.TYPE_TEXT == net_info['preprocess']): raise Exception("function not ready") else: raise Exception("unknown data type") # data size info change utils.tfmsa_logger("[5]modify data stuctor info") #ConvCommonManager(conf_info).save_changed_data_info(nn_id, train_data_set) learnrate = conf_info.data.learnrate label_set = json.loads(net_info['datasets']) conf_info.n_class = len(label_set) # change to nummpy array train_x = np.array(train_data_set, np.float32) train_y = np.array(train_label_set, np.int32) # define classifier utils.tfmsa_logger("[6]define classifier") classifier = learn.Estimator( model_fn=ConvCommonManager(conf_info).struct_cnn_layer, model_dir=netconf.nn_model_manager.get_model_save_path(nn_id)) # start train #TODO : need to find way to predict without fit utils.tfmsa_logger("[5]fit dummy") dummy_x = np.array([ ConvCommonManager(conf_info).create_dummy_matrix(len(train_x[0])) ], np.float32) dummy_y = np.array( netcommon.convert_to_index(json.loads(net_info['datasets'])), np.int32) classifier.fit(dummy_x, dummy_y, steps=int(1)) # start train utils.tfmsa_logger("[8]evaluate prediction result") counter = 0 acc_result_obj = AccStaticResult() for p in classifier.predict(x=np.array(train_x, np.float32), batch_size=1, as_iterable=True): acc_result_obj = AccEvalCommon(nn_id).set_result( acc_result_obj, label_set[train_y[counter]], label_set[int(p['class'])]) counter = counter + 1 return len(train_y) except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def train_conv_network(nn_id, epoch=50, testset=100): try: # check network is ready to train utils.tfmsa_logger("[1]check pre steps ready") utils.check_requested_nn(nn_id) # get network base info utils.tfmsa_logger("[2]get network base info") net_info = netconf.get_network_config(nn_id) # get network format info utils.tfmsa_logger("[3]get network format info") conf_info = netconf.load_conf(nn_id) # load train data utils.tfmsa_logger("[4]load train data") train_data_set = [] train_label_set = [] if (const.TYPE_IMAGE == net_info['preprocess']): train_data_set, train_label_set = ConvCommonManager( conf_info).prepare_image_data(nn_id, net_info) elif (const.TYPE_DATA_FRAME == net_info['preprocess']): raise Exception("function not ready") elif (const.TYPE_TEXT == net_info['preprocess']): raise Exception("function not ready") else: raise Exception("unknown data type") # data size info change utils.tfmsa_logger("[5]modify data stuctor info") # ConvCommonManager(conf_info).save_changed_data_info(nn_id, train_data_set) learnrate = conf_info.data.learnrate conf_info.n_class = len(json.loads(net_info['datasets'])) # change to nummpy array train_x = np.array(np.array(train_data_set).astype(float), np.float32) train_y = np.array(train_label_set, np.int32) # define classifier utils.tfmsa_logger("[6]define classifier") classifier = learn.Estimator( model_fn=ConvCommonManager(conf_info).struct_cnn_layer, model_dir=netconf.nn_model_manager.get_model_save_path(nn_id)) # start train utils.tfmsa_logger("[7]fit CNN") customsMonitor = Monitors.MonitorCommon(p_nn_id=nn_id, p_max_steps=10000, p_every_n_steps=100) classifier.fit(train_x, train_y, steps=1000, monitors=[customsMonitor]) return len(train_y) except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def wdd_eval(self, nnid, filename=None): """ Wide & Deep Network predict :param nnid : network id in tfmsacore_nninfo :return: acturacy """ try: utils.tfmsa_logger("((3.1 Wide & Deep Network Eval Start)) ## " + nnid + "##") json_string = WdnnCommonManager.get_all_info_json_by_nnid( self, nnid=nnid) database = json_string['dir'] table_name = json_string['table'] json_string_desc = netconf.load_ori_format(nnid) json_ob = json.loads(json_string_desc) #get datadesc format model_dir = str(json_string['query']) label_column = list(json_ob["label"].keys())[0] _label_list = json_string['datasets'] utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## DB : " + database + " TABLE : " + table_name) utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Model_Dir : " + model_dir) utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Label Column " + label_column + " Label : " + _label_list) label_list = eval(_label_list) wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir, False) if filename == None: df, last_key = data.DataMaster().query_data( "test_schema_" + database, table_name, "a", use_df=True, limit_cnt=-1, with_label=label_column) utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Get Test Schema Data Count" + str(len(df)) + "LastKey : " + str(last_key)) else: #Don't use this pat print("((2.Get Dataframe from CSV)) ##Start## (" + nnid + " , " + filename + ")") file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename print("((2.Get Dataframe from CSV)) ##filePath## (" + file_path + ")") print(file_path) df = pd.read_csv( tf.gfile.Open(file_path), # names=COLUMNS, skipinitialspace=True, engine="python") df['label'] = ( df[label_column].apply(lambda x: "Y" in x)).astype(int) results = {} utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Start Predict in Eval Method ##" ) predict_results = wdnn_model.predict( input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid)) df['predict_label'] = list(predict_results) #print(df['predict_label'] ) utils.tfmsa_logger( "make label encorder function ######################") # #Decode Label(int) to Label(str) le = LabelEncoder() le.fit(label_list) lable_decoder_func = lambda x: le.inverse_transform([x]) df['predict_label'] = df['predict_label'].map( lable_decoder_func).astype("str") label_value = list(df['label'].unique()) label_encode = le.inverse_transform(label_value) utils.tfmsa_logger( "((3.1 Wide & Deep Network Predict)) ## Label Convert ##" + str(label_value) + "DECODE ---> " + str(label_encode)) #results_acc table manager class create acc_result_obj = AccStaticResult() log_count = 0 for value in df.iterrows(): log_count += 1 #print("Inside start row by row") #print(type(value)) _row = value[1] ori_label = str(_row[label_column]) predict_label = str(eval(_row["predict_label"]) [0]) #str(list(_row["predict_label"])[0]) #predict_label = le.inverse_transform(_predict_label) if (log_count % 20 == 0): utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Columns Insert)) " + str(ori_label) + "------->" + str(predict_label)) acc_result_obj = AccEvalCommon(nnid).set_result( acc_result_obj, ori_label, predict_label) return results except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def __init__(self): utils.tfmsa_logger("Create Wdnn common ")