def eval(self, node_id, conf_data, data=None, result=None): """ :param node_id: :param parm: :return: """ logging.info("eval_data") self._init_node_parm(node_id.split('_')[0] + "_" + node_id.split('_')[1]+ "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder config = {"type": self.model_type, "labels": self.label_values, "nn_id":conf_data.get('nn_id'), "nn_wf_ver_id":conf_data.get('wf_ver')} train = TrainSummaryInfo(conf=config) print(config) self.batch = self.get_eval_batch(node_id) #print(train) self.model_eval_path = ''.join([self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag #conf_data['cls_pool'].get('nn00001_1_pre_feed_fr2wdnn_test') print("model_path : " + str(self.model_path)) print("hidden_layers : " + str(self.hidden_layers)) print("activation_function : " + str(self.activation_function)) print("batch_size : " + str(self.batch_size)) print("epoch : " + str(self.epoch)) print("model_type : " + str(self.model_type)) # data_store_path = WorkFlowDataFrame(conf_data['nn_id']+"_"+conf_data['wf_ver']+"_"+ "data_node").step_store data_conf_info = self.data_conf # make wide & deep model wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_eval_path)) # feed # TODO file이 여러개면 어떻하지? # get prev node for load data #data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') #train_data_set = self.cls_pool[data_node_name[0]] # get filename train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): print("h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict #self.batch_size = 2 for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] #if i == 0: #eval_data_Set = data_set # input_fn2(self, mode, data_file, df, nnid, dataconf): predict_value = wdnn_model.predict( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info)) data_set_count = len(data_set.index) predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error("wdnn eval error check : dataframe count({0}) predict count({1})".format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)') data_set['predict_label'] = predict_val_list #list(predict_value) #_predict = list(predict_value) predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting print(len(ori_list)) print(len(pre_list)) #logging.error("wdnn eval ori list : {0}".format(ori_list) ) logging.info("wdnn eval ori list : {0}".format(len(ori_list)) ) #logging.info("wdnn eval ori list : {0}".format('info')) #logging.debug("wdnn eval ori list : {0}".format('debug')) #logging.critical("wdnn eval ori list : {0}".format('critical')) #print("model fitting h5 " + str(data_set)) # #Select Next file train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if self.model_type == "category": # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] train.set_result_info(str(_ori), str(le.inverse_transform(pre_list[_i]))) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") return train
def eval(self, node_id, conf_data, data=None, result=None): """ :param node_id: :param parm: :return: """ logging.info("eval_starting ------> {0}".format(node_id)) self._init_node_parm( node_id.split('_')[0] + "_" + node_id.split('_')[1] + "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder config = { "type": self.model_type, "labels": self.label_values, "nn_id": conf_data.get('nn_id'), "nn_wf_ver_id": conf_data.get('wf_ver') } train = TrainSummaryInfo(conf=config) print(config) self.batch = self.get_eval_batch(node_id) #print(train) self.model_eval_path = ''.join([self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag #conf_data['cls_pool'].get('nn00001_1_pre_feed_fr2wdnn_test') logging.info("model_path : {0}".format(self.model_path)) print("hidden_layers : {0}".format(self.hidden_layers)) print("activation_function : {0}".format(self.activation_function)) print("batch_size : {0}".format(self.batch_size)) print("epoch : {0}".format(self.epoch)) print("model_type : {0}".format(self.model_type)) # data_store_path = WorkFlowDataFrame(conf_data['nn_id']+"_"+conf_data['wf_ver']+"_"+ "data_node").step_store data_conf_info = self.data_conf # make wide & deep modelnot wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_eval_path)) # feed # TODO file이 여러개면 어떻하지? # get prev node for load data #data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') #train_data_set = self.cls_pool[data_node_name[0]] # get filename train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): print("h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict #self.batch_size = 2 for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] #if i == 0: #eval_data_Set = data_set # input_fn2(self, mode, data_file, df, nnid, dataconf): predict_value = wdnn_model.predict( input_fn=lambda: train_data_set.input_fn2( tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info)) data_set_count = len(data_set.index) predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error( "wdnn eval error check : dataframe count({0}) predict count({1})" .format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)' ) data_set[ 'predict_label'] = predict_val_list #list(predict_value) #_predict = list(predict_value) predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting print(len(ori_list)) print(len(pre_list)) #logging.error("wdnn eval ori list : {0}".format(ori_list) ) logging.info("wdnn eval ori list : {0}".format( len(ori_list))) #logging.info("wdnn eval ori list : {0}".format('info')) #logging.debug("wdnn eval ori list : {0}".format('debug')) #logging.critical("wdnn eval ori list : {0}".format('critical')) #print("model fitting h5 " + str(data_set)) # #Select Next file train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if self.model_type == "category": # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] train.set_result_info( str(_ori), str(le.inverse_transform(pre_list[_i]))) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") return train
class NeuralNetNodeReNet(NeuralNetNode): """ """ def _init_train_parm(self, conf_data): # get initial value self.conf_data = conf_data self.cls_pool = conf_data["cls_pool"] self.nn_id = conf_data["nn_id"] self.wf_ver = conf_data["wf_ver"] self.node_id = conf_data["node_id"] self.node = WorkFlowSimpleManager().get_train_node() # get feed name self.train_feed_name = self.nn_id + "_" + self.wf_ver + "_" + WorkFlowSimpleManager( ).get_train_feed_node() self.eval_feed_name = self.nn_id + "_" + self.wf_ver + "_" + WorkFlowSimpleManager( ).get_eval_feed_node() self.feed_node = self.get_prev_node() def _init_value(self): self.g_train_cnt = 0 self.file_end = '.bin' self.train_return_data = {} self.train_return_arr = [ "Trainning .................................................." ] #################################################################################################################### def _set_netconf_parm(self): netconf = WorkFlowNetConfCNN().get_view_obj(self.node_id) try: netconf = WorkFlowNetConfCNN().set_num_classes_predcnt( self.nn_id, self.wf_ver, self.node, self.node_id, netconf) except: None self.netconf = netconf try: self.train_cnt = self.netconf["param"]["traincnt"] self.epoch = self.netconf["param"]["epoch"] self.batch_size = self.netconf["param"]["batch_size"] self.model_path = self.netconf["modelpath"] self.modelname = self.netconf["modelname"] except Exception as e: logging.info("NetConf is not exist.") logging.info(e) def _set_dataconf_parm(self, dataconf): self.dataconf = dataconf #################################################################################################################### def set_saver_model(self): self.save_path = self.model_path + "/" + str(self.batch) + str( self.file_end) keras.models.save_model(self.model, self.save_path) loss = round(self.loss * 100, 2) accR = round(self.acc * 100, 2) val_loss = round(self.val_loss * 100, 2) val_acc = round(self.val_acc * 100, 2) msg = "Global Step: " + str(self.g_train_cnt) msg += ", Training Loss: " + str( loss) + "%" + ", Training Accuracy: " + str(accR) + "%" msg += ", Test Loss: " + str( val_loss) + "%" + ", Test Accuracy: " + str(val_acc) + "%" logging.info(msg) config = { "nn_id": self.nn_id, "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch } result = TrainSummaryAccLossInfo(config) result.loss_info["loss"] = str(val_loss) result.acc_info["acc"] = str(val_acc) self.save_accloss_info(result) result = [msg] # self.model_file_delete(self.model_path, self.modelname) self.train_return_arr.append(result) self.eval(self.node_id, self.conf_data, None, None) def get_model_resnet(self): try: keras.backend.tensorflow_backend.clear_session() self.lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) self.early_stopper = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10) self.csv_logger = CSVLogger('resnet.csv') num_classes = self.netconf["config"]["num_classes"] numoutputs = self.netconf["config"]["layeroutputs"] x_size = self.dataconf["preprocess"]["x_size"] y_size = self.dataconf["preprocess"]["y_size"] channel = self.dataconf["preprocess"]["channel"] optimizer = self.netconf["config"]["optimizer"] filelist = os.listdir(self.model_path) filelist.sort(reverse=True) last_chk_path = self.model_path + "/" + self.load_batch + self.file_end try: self.model = keras.models.load_model(last_chk_path) logging.info("Train Restored checkpoint from:" + last_chk_path) except Exception as e: if numoutputs == 18: self.model = resnet.ResnetBuilder.build_resnet_18( (channel, x_size, y_size), num_classes) elif numoutputs == 34: self.model = resnet.ResnetBuilder.build_resnet_34( (channel, x_size, y_size), num_classes) elif numoutputs == 50: self.model = resnet.ResnetBuilder.build_resnet_50( (channel, x_size, y_size), num_classes) elif numoutputs == 101: self.model = resnet.ResnetBuilder.build_resnet_101( (channel, x_size, y_size), num_classes) elif numoutputs == 152: self.model = resnet.ResnetBuilder.build_resnet_152( (channel, x_size, y_size), num_classes) elif numoutputs == 200: self.model = resnet.ResnetBuilder.build_resnet_200( (channel, x_size, y_size), num_classes) logging.info( "None to restore checkpoint. Initializing variables instead." + last_chk_path) logging.info(e) self.model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) except Exception as e: logging.error( "===Error on Residualnet build model : {0}".format(e)) #################################################################################################################### def train_run_resnet(self, input_data, test_data): data_augmentation = self.netconf["param"]["augmentation"] try: if data_augmentation == "N" or data_augmentation == "n": logging.info('Not using data augmentation.') else: logging.info('Using real-time data augmentation.') while (input_data.has_next()): data_set = input_data[0:input_data.data_size()] x_batch, y_batch, n_batch = self.get_batch_img_data( data_set, "T") test_set = test_data[0:test_data.data_size()] x_tbatch, y_tbatch, n_tbatch = self.get_batch_img_data( test_set, "T") for i in range(self.train_cnt): if data_augmentation == "N" or data_augmentation == "n": history = self.model.fit(x_batch, y_batch, batch_size=self.batch_size, epochs=self.epoch, validation_data=(x_tbatch, y_tbatch), shuffle=True, callbacks=[ self.lr_reducer, self.early_stopper, self.csv_logger ]) else: # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center= False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization= False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 0, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) # randomly flip images # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_batch) # Fit the model on the batches generated by datagen.flow(). history = self.model.fit_generator( datagen.flow(x_batch, y_batch, batch_size=self.batch_size), steps_per_epoch=x_batch.shape[0] // self.batch_size, validation_data=(x_tbatch, y_tbatch), epochs=self.epoch, verbose=1, max_q_size=100, callbacks=[ self.lr_reducer, self.early_stopper, self.csv_logger ]) self.loss = history.history["loss"][0] self.acc = history.history["acc"][0] self.val_loss = history.history["val_loss"][0] self.val_acc = history.history["val_acc"][0] self.g_train_cnt += 1 logging.info("Save Train Count=" + str(self.g_train_cnt)) self.set_saver_model() input_data.next() except Exception as e: logging.info( "Error[400] ..............................................") logging.info(e) def run(self, conf_data): try: logging.info("run NeuralNetNodeResnet Train") # init data setup self._init_train_parm(conf_data) self._init_value() # get data & dataconf test_data, dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.eval_feed_name) input_data, dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.train_feed_name) # set netconf, dataconf self._set_netconf_parm() self._set_dataconf_parm(dataconf) # set batch self.load_batch = self.get_eval_batch(self.node_id) if self.epoch != 0 and self.train_cnt != 0: self.train_batch, self.batch = self.make_batch(self.node_id) else: self.batch = self.load_batch self.get_model_resnet() self.train_run_resnet(input_data, test_data) self.train_return_data["TrainResult"] = self.train_return_arr if self.epoch == 0 or self.train_cnt == 0: self.eval(self.node_id, self.conf_data, None, None) return self.train_return_data except Exception as e: logging.info("===Error on running residualnet : {0}".format(e)) #################################################################################################################### def eval_run(self, input_data): self.batch_size = self.netconf["param"]["batch_size"] labels = self.netconf["labels"] pred_cnt = self.netconf["param"]["predictcnt"] try: predlog = self.netconf["param"]["predictlog"] except: predlog = "N" # logging.info(labels) t_cnt_arr = [] f_cnt_arr = [] for i in range(len(labels)): t_cnt_arr.append(0) f_cnt_arr.append(0) input_data.pointer = 0 # eval config = { "type": self.netconf["config"]["eval_type"], "labels": self.netconf["labels"], "nn_id": self.nn_id, "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch } self.eval_data = TrainSummaryInfo(conf=config) while (input_data.has_next()): data_set = input_data[0:input_data.data_size()] x_batch, y_batch, n_batch = self.get_batch_img_data(data_set, "E") try: logits = self.model.predict(x_batch) for i in range(len(logits)): true_name = y_batch[i] file_name = n_batch[i] logit = [] logit.append(logits[i]) # idx = labels.index(true_name) retrun_data = self.set_predict_return_cnn_img( labels, logit, pred_cnt) pred_name = retrun_data["key"][0] if self.eval_flag == "E": if true_name == pred_name: t_cnt_arr[idx] = t_cnt_arr[idx] + 1 strLog = "[True] : " if (predlog == "TT"): logging.info(strLog + true_name + " FileName=" + file_name) logging.info(retrun_data["key"]) logging.info(retrun_data["val"]) else: f_cnt_arr[idx] = f_cnt_arr[idx] + 1 strLog = "[False] : " if (predlog == "FF"): logging.info(strLog + true_name + " FileName=" + file_name) logging.info(retrun_data["key"]) logging.info(retrun_data["val"]) if (predlog == "AA"): logging.info(strLog + true_name + " FileName=" + file_name) logging.info(retrun_data["key"]) logging.info(retrun_data["val"]) else: try: listTF = retrun_data["key"].index(true_name) t_cnt_arr[idx] = t_cnt_arr[idx] + 1 strLog = "[True] : " if (predlog == "T"): logging.info(strLog + true_name + " FileName=" + file_name) logging.info(retrun_data["key"]) logging.info(retrun_data["val"]) except: f_cnt_arr[idx] = f_cnt_arr[idx] + 1 strLog = "[False] : " if (predlog == "F"): logging.info(strLog + true_name + " FileName=" + file_name) logging.info(retrun_data["key"]) logging.info(retrun_data["val"]) if (predlog == "A"): logging.info(strLog + true_name + " FileName=" + file_name) logging.info(retrun_data["key"]) logging.info(retrun_data["val"]) self.eval_data.set_result_info(true_name, pred_name) except Exception as e: logging.info(e) logging.info( "None to restore checkpoint. Initializing variables instead." ) input_data.next() # set parms for db store input_data = TrainSummaryInfo.save_result_info(self, self.eval_data) self.eval_print(labels, t_cnt_arr, f_cnt_arr) def eval_print(self, labels, t_cnt_arr, f_cnt_arr): logging.info( "####################################################################################################" ) result = [] strResult = "['Eval ......................................................']" result.append(strResult) totCnt = 0 tCnt = 0 fCnt = 0 for i in range(len(labels)): strResult = "Category : " + self.spaceprint(labels[i], 15) + " " strResult += "TotalCnt=" + self.spaceprint( str(t_cnt_arr[i] + f_cnt_arr[i]), 8) + " " strResult += "TrueCnt=" + self.spaceprint(str(t_cnt_arr[i]), 8) + " " strResult += "FalseCnt=" + self.spaceprint(str(f_cnt_arr[i]), 8) + " " if t_cnt_arr[i] + f_cnt_arr[i] != 0: strResult += "True Percent(TrueCnt/TotalCnt*100)=" + str( round(t_cnt_arr[i] / (t_cnt_arr[i] + f_cnt_arr[i]) * 100)) + "%" totCnt += t_cnt_arr[i] + f_cnt_arr[i] tCnt += t_cnt_arr[i] fCnt += f_cnt_arr[i] logging.info(strResult) result.append(strResult) strResult = "---------------------------------------------------------------------------------------------------" logging.info(strResult) strResult = "Total Category=" + self.spaceprint(str(len(labels)), 11) + " " strResult += "TotalCnt=" + self.spaceprint(str(totCnt), 8) + " " strResult += "TrueCnt=" + self.spaceprint(str(tCnt), 8) + " " strResult += "FalseCnt=" + self.spaceprint(str(fCnt), 8) + " " if totCnt != 0: strResult += "True Percent(TrueCnt/TotalCnt*100)=" + str( round(tCnt / totCnt * 100)) + "%" logging.info(strResult) result.append(strResult) logging.info( "###################################################################################################" ) def eval(self, node_id, conf_data, data=None, result=None): logging.info("run NeuralNetNodeCnn eval") if data == None: self.eval_flag = "T" else: self.eval_flag = "E" # get data & dataconf test_data, dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.eval_feed_name) self.eval_run(test_data) return self.eval_data #################################################################################################################### def predict(self, node_id, filelist): """ """ logging.info("run NeuralNetNodeCnn Predict") self.node_id = node_id self._init_value() # net, data config setup data_node_name = self._get_backward_node_with_type(node_id, 'data') dataconf = WorkFlowNetConfCNN().get_view_obj(data_node_name[0]) self._set_netconf_parm() self._set_dataconf_parm(dataconf) # data shape change MultiValuDict -> nd array filename_arr, filedata_arr = self.change_predict_fileList( filelist, dataconf) # get unique key self.load_batch = self.get_active_batch(self.node_id) unique_key = '_'.join([node_id, self.load_batch]) logging.info("getModelPath:" + self.model_path + "/" + self.load_batch + self.file_end) ## create tensorflow graph if (NeuralNetModel.dict.get(unique_key)): self = NeuralNetModel.dict.get(unique_key) graph = NeuralNetModel.graph.get(unique_key) else: self.get_model_resnet() NeuralNetModel.dict[unique_key] = self NeuralNetModel.graph[unique_key] = tf.get_default_graph() graph = tf.get_default_graph() pred_return_data = {} for i in range(len(filename_arr)): file_name = filename_arr[i] file_data = filedata_arr[i] logits = self.model.predict(file_data) labels = self.netconf["labels"] pred_cnt = self.netconf["param"]["predictcnt"] retrun_data = self.set_predict_return_cnn_img( labels, logits, pred_cnt) pred_return_data[file_name] = retrun_data logging.info("Return Data.......................................") logging.info(pred_return_data) return pred_return_data
class NeuralNetNodeCnn(NeuralNetNode): """ """ def one_hot_encoded(self, num_classes): one = np.zeros((num_classes, num_classes)) for i in range(num_classes): for j in range(num_classes): if i == j: one[i][j] = 1 return one ######################################################################## def spaceprint(self, val, cnt): leng = len(str(val)) cnt = cnt - leng restr = "" for i in range(cnt): restr += " " restr = restr + str(val) return restr ######################################################################## def model_file_delete(self, model_path, modelname): existcnt = 10 filelist = os.listdir(model_path) flist = [] i = 0 for filename in filelist: filetime = datetime.datetime.fromtimestamp( os.path.getctime(model_path + '/' + filename)).strftime('%Y%m%d%H%M%S') tmp = [filename, filetime] if filename.find(modelname) > -1: flist.append(tmp) i += 1 flistsort = sorted(flist, key=operator.itemgetter(1), reverse=True) for i in range(len(flistsort)): if i > existcnt * 3: os.remove(model_path + "/" + flistsort[i][0]) ######################################################################## def _init_train_parm(self, conf_data): # get initial value self.conf_data = conf_data self.cls_pool = conf_data["cls_pool"] self.nn_id = conf_data["nn_id"] self.wf_ver = conf_data["wf_ver"] self.node_id = conf_data["node_id"] self.node = WorkFlowSimpleManager().get_train_node() # get feed name self.train_feed_name = self.nn_id + "_" + self.wf_ver + "_" + WorkFlowSimpleManager( ).get_train_feed_node() self.eval_feed_name = self.nn_id + "_" + self.wf_ver + "_" + WorkFlowSimpleManager( ).get_eval_feed_node() self.feed_node = self.get_prev_node() ######################################################################## def _init_predict_parm(self, node_id): self.node_id = node_id ######################################################################## def _init_value(self): self.g_ffile_print = "N" self.g_train_cnt = 0 self.g_epoch_cnt = 0 self.step_gap = 1 self.file_end = '.h5' self.train_return_data = {} self.train_return_arr = [ "Trainning .................................................." ] self.pred_return_data = {} ######################################################################## def _set_netconf_parm(self): netconf = WorkFlowNetConfCNN().get_view_obj(self.node_id) try: netconf = WorkFlowNetConfCNN().set_num_classes_predcnt( self.nn_id, self.wf_ver, self.node, self.node_id, netconf) except: None self.netconf = netconf self.net_type = self.netconf["config"]["net_type"] self.train_cnt = self.netconf["param"]["traincnt"] self.epoch = self.netconf["param"]["epoch"] self.train_cnt = self.netconf["param"]["traincnt"] self.batch_size = self.netconf["param"]["batch_size"] self.model_path = self.netconf["modelpath"] self.modelname = self.netconf["modelname"] ######################################################################## def _set_dataconf_parm(self, dataconf): self.dataconf = dataconf ######################################################################## def get_batch_img_data(self, data_set, type): num_classes = self.netconf["config"]["num_classes"] labels = self.netconf["labels"] x_size = self.dataconf["preprocess"]["x_size"] y_size = self.dataconf["preprocess"]["y_size"] channel = self.dataconf["preprocess"]["channel"] labelsHot = self.one_hot_encoded(num_classes) name_data_batch = data_set[2] label_data_batch = data_set[1] img_data_batch = data_set[0] if type == "T": r = 0 y_batch = np.zeros((len(label_data_batch), num_classes)) for j in label_data_batch: j = j.decode('UTF-8') k = labels.index(j) y_batch[r] = labelsHot[k] r += 1 else: y_batch = [] for j in label_data_batch: j = j.decode('UTF-8') y_batch.append(j) n_batch = [] for j in name_data_batch: j = j.decode('UTF-8') n_batch.append(j) try: x_batch = np.zeros((len(img_data_batch), len(img_data_batch[0]))) except Exception as e: println(e) r = 0 for j in img_data_batch: j = j.tolist() x_batch[r] = j r += 1 x_batch = np.reshape(x_batch, (-1, x_size, y_size, channel)) # println("Image Label ////////////////////////////////////////////////") # println(label_data_batch) # println(y_batch) # println("Image /////////////////////////////////////////////////") # println(x_batch) return x_batch, y_batch, n_batch ######################################################################## ######################################################################## def get_model_cnn(self, sess, type=None): prenumoutputs = 1 num_classes = self.netconf["config"]["num_classes"] learnrate = self.netconf["config"]["learnrate"] numoutputs = self.netconf["config"]["layeroutputs"] optimizer = self.netconf["config"]["optimizer"] node_out = self.netconf["out"]["node_out"] x_size = self.dataconf["preprocess"]["x_size"] y_size = self.dataconf["preprocess"]["y_size"] channel = self.dataconf["preprocess"]["channel"] ################################################################ X = tf.placeholder(tf.float32, shape=[None, x_size, y_size, channel], name='x') Y = tf.placeholder(tf.float32, shape=[None, num_classes], name='y') ################################################################ stopper = 1 model = X while True: try: layer = self.netconf["layer" + str(stopper)] except Exception as e: if stopper == 1: return "Error[100] layer is None ..............................." break stopper += 1 try: layercnt = layer["layercnt"] for i in range(layercnt): # println(layer) if prenumoutputs == 1: prenumoutputs = numoutputs else: numoutputs = prenumoutputs * 2 prenumoutputs = numoutputs active = str(layer["active"]) convkernelsize = [ int((layer["cnnfilter"][0])), int((layer["cnnfilter"][1])) ] maxpkernelsize = [ int((layer["maxpoolmatrix"][0])), int((layer["maxpoolmatrix"][1])) ] stride = [ int((layer["maxpoolstride"][0])), int((layer["maxpoolstride"][1])) ] padding = str((layer["padding"])) if active == 'relu': activitaion = tf.nn.relu else: activitaion = tf.nn.relu if str(layer["droprate"]) is not "": droprate = float((layer["droprate"])) else: droprate = 0.0 model = tf.contrib.layers.conv2d( inputs=model, num_outputs=numoutputs, kernel_size=convkernelsize, activation_fn=activitaion, weights_initializer=tf.contrib.layers. xavier_initializer_conv2d(), padding=padding) model = tf.contrib.layers.max_pool2d( inputs=model, kernel_size=maxpkernelsize, stride=stride, padding=padding) if droprate > 0.0 and type == "T": model = tf.nn.dropout(model, droprate) # println(model) except Exception as e: println("Error[200] Model Create Fail.") println(e) reout = int(model.shape[1]) * int(model.shape[2]) * int(model.shape[3]) model = tf.reshape(model, [-1, reout]) # println(model) W1 = tf.Variable(tf.truncated_normal([reout, node_out], stddev=0.1)) model = tf.nn.relu(tf.matmul(model, W1)) W5 = tf.Variable( tf.truncated_normal([node_out, num_classes], stddev=0.1)) model = tf.matmul(model, W5) # println(model) if type == "P": model = tf.nn.softmax(model) cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=Y)) if optimizer == "AdamOptimizer": optimizer = tf.train.AdamOptimizer( learning_rate=learnrate).minimize(cost) else: optimizer = tf.train.RMSPropOptimizer(learnrate, 0.9).minimize(cost) y_pred_cls = tf.argmax(model, 1) check_prediction = tf.equal(y_pred_cls, tf.argmax(Y, 1)) accuracy = tf.reduce_mean(tf.cast(check_prediction, tf.float32)) self.model = model self.X = X self.Y = Y self.optimizer = optimizer self.y_pred_cls = y_pred_cls self.accuracy = accuracy self.cost = cost self.model_path = self.netconf["modelpath"] self.modelname = self.netconf["modelname"] last_chk_path = tf.train.latest_checkpoint( checkpoint_dir=self.model_path) try: step = last_chk_path.split("-") self.step_gap = int(step[1]) + 1 saver = tf.train.Saver() saver.restore(sess, save_path=last_chk_path) println("Train Restored checkpoint from:" + last_chk_path) except: self.step_gap = 1 println( "None to restore checkpoint. Initializing variables instead.") self.save_path = self.model_path + "/" + self.modelname + "-" + str( self.step_gap) return sess, saver ######################################################################## def get_model_resnet(self, sess): self.lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) self.early_stopper = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10) self.csv_logger = CSVLogger('resnet.csv') num_classes = self.netconf["config"]["num_classes"] numoutputs = self.netconf["config"]["layeroutputs"] x_size = self.dataconf["preprocess"]["x_size"] y_size = self.dataconf["preprocess"]["y_size"] channel = self.dataconf["preprocess"]["channel"] self.data_augmentation = self.dataconf["preprocess"]["augmentation"] filelist = os.listdir(self.model_path) try: for filename in filelist: step1 = filename.split("-") step2 = step1[1].split(".") if self.step_gap < int(step2[0]): self.step_gap = int(step2[0]) last_chk_path = self.model_path + "/" + self.modelname + "-" + str( self.step_gap) + str(self.file_end) println(last_chk_path) self.model = keras.models.load_model(last_chk_path) self.model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) self.step_gap = int(step2[0]) + 1 println("Train Restored checkpoint from:" + last_chk_path) except: println( "None to restore checkpoint. Initializing variables instead.") if numoutputs == 18: self.model = resnet.ResnetBuilder.build_resnet_18( (channel, x_size, y_size), num_classes) elif numoutputs == 34: self.model = resnet.ResnetBuilder.build_resnet_34( (channel, x_size, y_size), num_classes) elif numoutputs == 50: self.model = resnet.ResnetBuilder.build_resnet_50( (channel, x_size, y_size), num_classes) elif numoutputs == 101: self.model = resnet.ResnetBuilder.build_resnet_101( (channel, x_size, y_size), num_classes) elif numoutputs == 152: self.model = resnet.ResnetBuilder.build_resnet_152( (channel, x_size, y_size), num_classes) elif numoutputs == 200: self.model = resnet.ResnetBuilder.build_resnet_200( (channel, x_size, y_size), num_classes) self.model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) self.save_path = self.model_path + "/" + self.modelname + "-" + str( self.step_gap) + str(self.file_end) ######################################################################## def set_saver_model(self, sess): saver = tf.train.Saver() saver.save(sess, save_path=self.save_path) batch_accR = round(self.batch_acc * 100, 2) msg = "Global Step: " + str( self.step_gap) + ", Training Batch Accuracy: " + str( batch_accR) + "%" + ", Cost: " + str(self.i_cost) println(msg) config = { "nn_id": self.nn_id, "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch } result = TrainSummaryAccLossInfo(config) result.loss_info["loss"] = str(self.i_cost) result.acc_info["acc"] = str(batch_accR) self.save_accloss_info(result) result = [msg] self.step_gap = self.step_gap + self.g_epoch_cnt self.save_path = self.model_path + "/" + self.modelname + "-" + str( self.step_gap) self.model_file_delete(self.model_path, self.modelname) self.train_return_arr.append(result) self.eval(self.node_id, self.conf_data, None, None) def set_saver_model_keras(self): keras.models.save_model(self.model, self.save_path) loss = round(self.loss * 100, 2) accR = round(self.acc * 100, 2) val_loss = round(self.val_loss * 100, 2) val_acc = round(self.val_acc * 100, 2) msg = "Global Step: " + str(self.step_gap) msg += ", Training Loss: " + str( loss) + "%" + ", Training Accuracy: " + str(accR) + "%" msg += ", Test Loss: " + str( val_loss) + "%" + ", Test Accuracy: " + str(val_acc) + "%" println(msg) config = { "nn_id": self.nn_id, "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch } result = TrainSummaryAccLossInfo(config) result.loss_info["loss"] = str(val_loss) result.acc_info["acc"] = str(val_acc) self.save_accloss_info(result) result = [msg] self.step_gap = self.step_gap + self.g_epoch_cnt self.save_path = self.model_path + "/" + self.modelname + "-" + str( self.step_gap) + str(self.file_end) self.model_file_delete(self.model_path, self.modelname) self.train_return_arr.append(result) self.eval(self.node_id, self.conf_data, None, None) def run(self, conf_data): println("run NeuralNetNodeCnn Train") # init data setup self._init_train_parm(conf_data) self._init_value() # set batch self.train_batch, self.batch = self.make_batch(self.node_id) # get data & dataconf test_data, dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.eval_feed_name) input_data, dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.train_feed_name) # set netconf, dataconf self._set_netconf_parm() self._set_dataconf_parm(dataconf) # train with tf.Session() as sess: if self.net_type == "resnet": self.get_model_resnet(sess) sess.run(tf.global_variables_initializer()) self.train_run_resnet(input_data, test_data) else: sess, saver = self.get_model_cnn(sess, "T") sess.run(tf.global_variables_initializer()) self.train_run_cnn(sess, input_data, test_data) self.train_return_data["TrainResult"] = self.train_return_arr if self.epoch == 0 or self.train_cnt == 0: self.eval(self.node_id, self.conf_data, None, None) return self.train_return_data def train_run_resnet(self, input_data, test_data): try: if self.data_augmentation == "N" or self.data_augmentation == "n": println('Not using data augmentation.') else: println('Using real-time data augmentation.') while (input_data.has_next()): data_set = input_data[0:input_data.data_size()] x_batch, y_batch, n_batch = self.get_batch_img_data( data_set, "T") test_set = test_data[0:test_data.data_size()] x_tbatch, y_tbatch, n_tbatch = self.get_batch_img_data( test_set, "T") for i in range(self.train_cnt): if self.data_augmentation == "N" or self.data_augmentation == "n": history = self.model.fit(x_batch, y_batch, batch_size=self.batch_size, epochs=self.epoch, validation_data=(x_tbatch, y_tbatch), shuffle=True, callbacks=[ self.lr_reducer, self.early_stopper, self.csv_logger ]) else: # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center= False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization= False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 0, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) # randomly flip images # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_batch) # Fit the model on the batches generated by datagen.flow(). history = self.model.fit_generator( datagen.flow(x_batch, y_batch, batch_size=self.batch_size), steps_per_epoch=x_batch.shape[0] // self.batch_size, validation_data=(x_tbatch, y_tbatch), epochs=self.epoch, verbose=1, max_q_size=100, callbacks=[ self.lr_reducer, self.early_stopper, self.csv_logger ]) self.loss = history.history["loss"][0] self.acc = history.history["acc"][0] self.val_loss = history.history["val_loss"][0] self.val_acc = history.history["val_acc"][0] self.g_train_cnt += 1 self.g_epoch_cnt = self.g_train_cnt println("Save Train Count=" + str(self.g_train_cnt)) self.set_saver_model_keras() input_data.next() except Exception as e: println( "Error[400] ..............................................") println(e) def train_run_cnn(self, sess, input_data, test_data): try: while (input_data.has_next()): for i in range(self.train_cnt): for i in range(0, input_data.size(), self.batch_size): data_set = input_data[i:i + self.batch_size] x_batch, y_batch, n_batch = self.get_batch_img_data( data_set, "T") for i in range(self.epoch): feed_dict_train = { self.X: x_batch, self.Y: y_batch } _, self.i_cost, self.batch_acc = sess.run( [self.optimizer, self.cost, self.accuracy], feed_dict=feed_dict_train) self.g_epoch_cnt += 1 println("Epoch Count=" + str(self.g_epoch_cnt)) self.g_train_cnt += 1 println("Save Train Count=" + str(self.g_train_cnt)) self.set_saver_model(sess) input_data.next() except Exception as e: println( "Error[400] ..............................................") println(e) return self.train_return_data ######################################################################## def eval(self, node_id, conf_data, data=None, result=None): println("run NeuralNetNodeCnn eval") self._init_train_parm(conf_data) if data == None: self.eval_flag = "T" else: self.eval_flag = "E" #eval self.batch = self.get_eval_batch(node_id) config = { "type": self.netconf["config"]["eval_type"], "labels": self.netconf["labels"], "nn_id": self.nn_id, "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch } self.eval_data = TrainSummaryInfo(conf=config) # config = {"type": self.netconf["config"]["eval_type"], "labels": self.netconf["labels"]} # self.eval_data = TrainSummaryInfo(conf=config) # self.eval_data.set_nn_id(self.nn_id) # self.eval_data.set_nn_wf_ver_id(self.wf_ver) # get data & dataconf test_data, dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.eval_feed_name) with tf.Session() as sess: if self.net_type == "resnet": self.get_model_resnet(sess) sess.run(tf.global_variables_initializer()) else: sess, saver = self.get_model_cnn(sess, "T") sess.run(tf.global_variables_initializer()) self.eval_run(sess, test_data) # keras.backend.clear_session() if self.eval_flag == "E": keras.backend.clear_session() return self.eval_data def eval_run(self, sess, input_data): self.batch_size = self.netconf["param"]["batch_size"] labels = self.netconf["labels"] pred_cnt = self.netconf["param"]["predictcnt"] try: predlog = self.netconf["param"]["predlog"] except: predlog = "N" # println(labels) t_cnt_arr = [] f_cnt_arr = [] for i in range(len(labels)): t_cnt_arr.append(0) f_cnt_arr.append(0) input_data.pointer = 0 while (input_data.has_next()): for i in range(0, input_data.size(), self.batch_size): data_set = input_data[i:i + self.batch_size] x_batch, y_batch, n_batch = self.get_batch_img_data( data_set, "E") try: if self.net_type == "cnn": logits = sess.run([self.model], feed_dict={self.X: x_batch}) logits = logits[0] elif self.net_type == "resnet": logits = self.model.predict(x_batch) for i in range(len(logits)): true_name = y_batch[i] file_name = n_batch[i] logit = [] logit.append(logits[i]) # idx = labels.index(true_name) retrun_data = self.set_predict_return_cnn_img( labels, logit, pred_cnt) pred_name = retrun_data["key"][0] if self.eval_flag == "E": if true_name == pred_name: t_cnt_arr[idx] = t_cnt_arr[idx] + 1 strLog = "[True] : " if (predlog == "TT"): println(strLog + true_name + " FileName=" + file_name) println(retrun_data["key"]) println(retrun_data["val"]) else: f_cnt_arr[idx] = f_cnt_arr[idx] + 1 strLog = "[False] : " if (predlog == "FF"): println(strLog + true_name + " FileName=" + file_name) println(retrun_data["key"]) println(retrun_data["val"]) if (predlog == "AA"): println(strLog + true_name + " FileName=" + file_name) println(retrun_data["key"]) println(retrun_data["val"]) else: try: listTF = retrun_data["key"].index(true_name) t_cnt_arr[idx] = t_cnt_arr[idx] + 1 strLog = "[True] : " if (predlog == "T"): println(strLog + true_name + " FileName=" + file_name) println(retrun_data["key"]) println(retrun_data["val"]) except: f_cnt_arr[idx] = f_cnt_arr[idx] + 1 strLog = "[False] : " if (predlog == "F"): println(strLog + true_name + " FileName=" + file_name) println(retrun_data["key"]) println(retrun_data["val"]) if (predlog == "A"): println(strLog + true_name + " FileName=" + file_name) println(retrun_data["key"]) println(retrun_data["val"]) self.eval_data.set_result_info(true_name, pred_name) except Exception as e: println(e) println( "None to restore checkpoint. Initializing variables instead." ) input_data.next() self.eval_print(labels, t_cnt_arr, f_cnt_arr) def eval_print(self, labels, t_cnt_arr, f_cnt_arr): println( "####################################################################################################" ) result = [] strResult = "['Eval ......................................................']" result.append(strResult) totCnt = 0 tCnt = 0 fCnt = 0 for i in range(len(labels)): strResult = "Category : " + self.spaceprint(labels[i], 15) + " " strResult += "TotalCnt=" + self.spaceprint( str(t_cnt_arr[i] + f_cnt_arr[i]), 8) + " " strResult += "TrueCnt=" + self.spaceprint(str(t_cnt_arr[i]), 8) + " " strResult += "FalseCnt=" + self.spaceprint(str(f_cnt_arr[i]), 8) + " " if t_cnt_arr[i] + f_cnt_arr[i] != 0: strResult += "True Percent(TrueCnt/TotalCnt*100)=" + str( round(t_cnt_arr[i] / (t_cnt_arr[i] + f_cnt_arr[i]) * 100)) + "%" totCnt += t_cnt_arr[i] + f_cnt_arr[i] tCnt += t_cnt_arr[i] fCnt += f_cnt_arr[i] println(strResult) result.append(strResult) strResult = "---------------------------------------------------------------------------------------------------" println(strResult) strResult = "Total Category=" + self.spaceprint(str(len(labels)), 11) + " " strResult += "TotalCnt=" + self.spaceprint(str(totCnt), 8) + " " strResult += "TrueCnt=" + self.spaceprint(str(tCnt), 8) + " " strResult += "FalseCnt=" + self.spaceprint(str(fCnt), 8) + " " if totCnt != 0: strResult += "True Percent(TrueCnt/TotalCnt*100)=" + str( round(tCnt / totCnt * 100)) + "%" println(strResult) result.append(strResult) println( "###################################################################################################" ) def predict(self, node_id, filelist): """ """ println("run NeuralNetNodeCnn Predict") # init data setup self._init_predict_parm(node_id) self._init_value() # net, data config setup data_node_name = self._get_backward_node_with_type(node_id, 'data') dataconf = WorkFlowNetConfCNN().get_view_obj(data_node_name[0]) self._set_netconf_parm() self._set_dataconf_parm(dataconf) self.net_type = self.netconf["config"]["net_type"] # data shape change MultiValuDict -> nd array filename_arr, filedata_arr = self.change_predict_fileList( filelist, dataconf) # get unique key unique_key = '_'.join([node_id, self.get_eval_batch(node_id)]) # prepare net conf tf.reset_default_graph() ## create tensorflow graph if (NeuralNetModel.dict.get(unique_key)): self = NeuralNetModel.dict.get(unique_key) graph = NeuralNetModel.graph.get(unique_key) else: if self.net_type == "cnn": self.get_model_cnn("P") elif self.net_type == "resnet": self.get_model_resnet() NeuralNetModel.dict[unique_key] = self NeuralNetModel.graph[unique_key] = tf.get_default_graph() graph = tf.get_default_graph() # predict with tf.Session(graph=graph) as sess: sess.run(tf.global_variables_initializer()) for i in range(len(filename_arr)): file_name = filename_arr[i] file_data = filedata_arr[i] if self.net_type == "cnn": sess, saver = self.get_saver_model(sess) logits = sess.run([self.model], feed_dict={self.X: file_data}) logits = logits[0] elif self.net_type == "resnet": logits = self.model.predict(file_data) labels = self.netconf["labels"] pred_cnt = self.netconf["param"]["predictcnt"] retrun_data = self.set_predict_return_cnn_img( labels, logits, pred_cnt) self.pred_return_data[file_name] = retrun_data println("Return Data.......................................") println(self.pred_return_data) return self.pred_return_data
def eval(self, node_id, conf_data, data=None, result=None): """ Tensorflow Wide and Deep Network Eval Method :param node_id: :param parm: :return: None """ logging.info("eval_starting ------> {0}".format(node_id)) try: #self._init_node_parm(conf_data['node_id']) #conf_data['node_id'] #self._init_node_parm(conf_data['node_id']) self._init_node_parm(conf_data.get('nn_id') + "_" + conf_data.get('wf_ver')+ "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) config = {"type": self.model_type, "labels": self.label_values, "nn_id":conf_data.get('nn_id'), "nn_wf_ver_id":conf_data.get('wf_ver')} train = TrainSummaryInfo(conf=config) print(config) self.batch_eval = self.get_eval_batch(node_id) #print(train) self.model_eval_path = ''.join([self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag #conf_data['cls_pool'].get('nn00001_1_pre_feed_fr2wdnn_test') logging.info("model_path : {0}".format(self.model_path)) logging.info("hidden_layers : {0}".format(self.hidden_layers)) logging.info("activation_function : {0}".format(self.activation_function)) logging.info("batch_size : {0}".format(self.batch_size)) logging.info("epoch : {0}".format(self.epoch)) logging.info("model_type : {0}".format(self.model_type)) logging.info("auto_demension : {0}".format(self.auto_demension)) config_acc = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": conf_data.get('wf_ver'), "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config_acc) data_conf_info = self.data_conf #validation_monitor = _LossCheckerHook(acc_result) # make wide & deep modelnot wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_eval_path), self.train, self.auto_demension) #, self.train, self.auto_demension # feed # TODO file이 여러개면 어떻하지? # get prev node for load data #data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') #train_data_set = self.cls_pool[data_node_name[0]] # get filename train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): logging.info("Wdnn eval process from h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict #self.batch_size = 2 for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) #print("model fitting h5 " + str(data_set)) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) predict_value = wdnn_model.predict( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info)) data_set_count = len(data_set.index) predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error("wdnn eval error check : dataframe count({0}) predict count({1})".format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)') data_set['predict_label'] = predict_val_list #list(predict_value) predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting logging.info("wdnn eval ori list : {0}".format(len(ori_list)) ) logging.info("wdnn eval pre list : {0}".format(len(pre_list)) ) train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch_eval) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if (self.model_type == "category" or self.model_type == "deep"): # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] train.set_result_info(str(_ori), str(le.inverse_transform(pre_list[_i]))) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") except Exception as oe: logging.info(oe) raise Exception(e) return train
def eval(self, node_id, conf_data, data=None, result=None): """ Tensorflow Wide and Deep Network Eval Method :param node_id: :param parm: :return: None """ logging.info("eval_starting ------> {0}".format(node_id)) try: self._init_node_parm(conf_data.get('nn_id') + "_" + conf_data.get('wf_ver')+ "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) config = {"type": self.model_type, "labels": self.label_values, "nn_id":conf_data.get('nn_id'), "nn_wf_ver_id":conf_data.get('wf_ver')} train = TrainSummaryInfo(conf=config) print(config) self.batch_eval = self.get_eval_batch(node_id) self.model_eval_path = ''.join([self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag logging.info("model_path : {0}".format(self.model_path)) logging.info("ml_class : {0}".format(self.ml_class)) logging.info("config : {0}".format(self.config)) config_acc = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": conf_data.get('wf_ver'), "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config_acc) data_conf_info = self.data_conf # make ML modelnot clf = joblib.load(self.model_path+'/model.pkl') # feed # TODO file이 여러개면 어떻하지? # get prev node for load data train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): logging.info("Wdnn eval process from h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] keys = list(data_conf_info['cell_feature'].keys()) keys.remove(data_conf_info['label']) keys = np.asarray(keys) data = data_set[keys].values label = data_set[data_conf_info['label']].values acc = cross_val_score(clf, data, label, scoring='accuracy').mean() loss = cross_val_score(clf, data, label, scoring='neg_log_loss').mean() # acc = eval_result['accuracy'] # loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(acc)) acc_result.acc_info["acc"].append(str(loss)) iris = load_iris() predict_val_list = list() for row in data : row = [row] predict_value = clf.predict(row) predict_val_list.append(predict_value) # predict_value = clf.predict( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info)) data_set_count = len(data_set.index) #predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error("ML eval error check : dataframe count({0}) predict count({1})".format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)') data_set['predict_label'] = predict_val_list predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting logging.info("ML eval ori list : {0}".format(len(ori_list)) ) logging.info("ML eval pre list : {0}".format(len(pre_list)) ) train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch_eval) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if (self.model_type == "category" or self.model_type == "deep"): # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] #train.set_result_info(str(_ori), str(le.inverse_transform(pre_list[_i]))) train.set_result_info(str(_ori), str(pre_list[_i][0])) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") except Exception as oe: logging.info(oe) raise Exception(e) return train
def eval(self, node_id, conf_data, data=None, result=None): """ Tensorflow Wide and Deep Network Eval Method :param node_id: :param parm: :return: None """ logging.info("eval_starting ------> {0}".format(node_id)) try: self._init_node_parm( conf_data.get('nn_id') + "_" + conf_data.get('wf_ver') + "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) config = { "type": self.model_type, "labels": self.label_values, "nn_id": conf_data.get('nn_id'), "nn_wf_ver_id": conf_data.get('wf_ver') } train = TrainSummaryInfo(conf=config) print(config) self.batch_eval = self.get_eval_batch(node_id) self.model_eval_path = ''.join( [self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag logging.info("model_path : {0}".format(self.model_path)) logging.info("ml_class : {0}".format(self.ml_class)) logging.info("config : {0}".format(self.config)) config_acc = { "nn_id": conf_data['node_id'], "nn_wf_ver_id": conf_data.get('wf_ver'), "nn_batch_ver_id": self.batch } acc_result = TrainSummaryAccLossInfo(config_acc) data_conf_info = self.data_conf # make ML modelnot clf = joblib.load(self.model_path + '/model.pkl') # feed # TODO file이 여러개면 어떻하지? # get prev node for load data train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): logging.info("Wdnn eval process from h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] keys = list(data_conf_info['cell_feature'].keys()) keys.remove(data_conf_info['label']) keys = np.asarray(keys) data = data_set[keys].values label = data_set[data_conf_info['label']].values acc = cross_val_score(clf, data, label, scoring='accuracy').mean() loss = cross_val_score(clf, data, label, scoring='neg_log_loss').mean() # acc = eval_result['accuracy'] # loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(acc)) acc_result.acc_info["acc"].append(str(loss)) iris = load_iris() predict_val_list = list() for row in data: row = [row] predict_value = clf.predict(row) predict_val_list.append(predict_value) # predict_value = clf.predict( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info)) data_set_count = len(data_set.index) #predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error( "ML eval error check : dataframe count({0}) predict count({1})" .format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)' ) data_set['predict_label'] = predict_val_list predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting logging.info("ML eval ori list : {0}".format( len(ori_list))) logging.info("ML eval pre list : {0}".format( len(pre_list))) train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch_eval) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if (self.model_type == "category" or self.model_type == "deep"): # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] #train.set_result_info(str(_ori), str(le.inverse_transform(pre_list[_i]))) train.set_result_info(str(_ori), str(pre_list[_i][0])) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") except Exception as oe: logging.info(oe) raise Exception(e) return train
class NeuralNetNodeImage(NeuralNetNode): def lr_schedule(self, epoch): """Learning Rate Schedule Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs. Called automatically every epoch as part of callbacks during training. # Arguments epoch (int): The number of epochs # Returns lr (float32): learning rate """ lr = 1e-3 if epoch > 180: lr *= 0.5e-3 elif epoch > 160: lr *= 1e-3 elif epoch > 120: lr *= 1e-2 elif epoch > 80: lr *= 1e-1 return lr def keras_get_model(self): # keras.backend.tensorflow_backend.clear_session() backendK.clear_session() # if settings.GPU_FLAG == True: # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) # sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) # backendK.set_session(sess) try: self.model = keras.models.load_model(self.last_chk_path) logging.info("Train Restored checkpoint from:" + self.last_chk_path) except Exception as e: logging.info("None to restore checkpoint. Initializing variables instead." + self.last_chk_path) logging.info(e) if self.optimizer == 'sgd': self.optimizer = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True) elif self.optimizer == 'rmsprop': self.optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=1e-6) elif self.optimizer == 'adagrad': self.optimizer = optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=1e-6) elif self.optimizer == 'adadelta': self.optimizer = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=1e-6) elif self.optimizer == 'adam': self.optimizer = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) # self.optimizer = optimizers.Adam(lr=self.lr_schedule(0)) elif self.optimizer == 'adamax': self.optimizer = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) elif self.optimizer == 'nadam': self.optimizer = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) if self.net_type == 'inceptionv4': # self.labels_cnt = 1001 self.model = inception_v4_model(self.labels_cnt, 0.2, self.pretrain_model_path) # elif self.net_type == 'nasnet': # self.model = NASNetLarge(input_shape=(331, 331, 3)) elif self.net_type == 'resnet': numoutputs = self.netconf["config"]["layeroutputs"] if numoutputs == 18: self.model = resnet.ResnetBuilder.build_resnet_18((self.channel, self.x_size, self.y_size), self.labels_cnt) elif numoutputs == 34: self.model = resnet.ResnetBuilder.build_resnet_34((self.channel, self.x_size, self.y_size), self.labels_cnt) elif numoutputs == 50: self.model = resnet.ResnetBuilder.build_resnet_50((self.channel, self.x_size, self.y_size), self.labels_cnt) elif numoutputs == 101: self.model = resnet.ResnetBuilder.build_resnet_101((self.channel, self.x_size, self.y_size), self.labels_cnt) elif numoutputs == 152: self.model = resnet.ResnetBuilder.build_resnet_152((self.channel, self.x_size, self.y_size), self.labels_cnt) # if settings.GPU_FLAG == True: # self.model = multi_gpu_model(self.model, gpus=1) self.model.compile(loss='categorical_crossentropy', optimizer=self.optimizer, metrics=['accuracy']) # self.model.summary() def train_run_image(self, input_data, test_data): ''' Train Run :param input_data: :param test_data: :return: ''' self.epoch = self.netconf["param"]["epoch"] self.data_augmentation = self.netconf["param"]["augmentation"] try: self.fit_size = self.netconf["param"]["fit_size"] except: self.fit_size = 9999999999 self.lr_scheduler = LearningRateScheduler(self.lr_schedule) self.lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) self.early_stopper = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10) try: while_cnt = 0 self.loss = 0 self.acc = 0 self.val_loss = 0 self.val_acc = 0 input_data.reset_pointer() test_data.reset_pointer() test_set = test_data[0:test_data.data_size()] x_tbatch = self.get_convert_img_x(test_set[0], self.x_size, self.y_size, self.channel) # img_data_batch y_tbatch = self.get_convert_img_y(test_set[1], self.labels, self.labels_cnt) # label_data_batch while (input_data.has_next()): run_size = 0 while( run_size < input_data.data_size()): if run_size + self.fit_size > input_data.data_size(): input_set = input_data[run_size:input_data.data_size()] else: input_set = input_data[run_size:run_size + self.fit_size] run_size += self.fit_size + 1 x_batch = self.get_convert_img_x(input_set[0], self.x_size, self.y_size, self.channel) # img_data_batch y_batch = self.get_convert_img_y(input_set[1], self.labels, self.labels_cnt) # label_data_batch if len(x_batch) < self.batch_size: self.batch_size = len(x_batch) # # Normalize data. # x_batch = x_batch.astype('float32') / 255 # x_tbatch = x_tbatch.astype('float32') / 255 # # If subtract pixel mean is enabled # if self.subtract_pixel_mean: # x_train_mean = np.mean(x_batch, axis=0) # x_batch -= x_train_mean # x_tbatch -= x_train_mean if self.data_augmentation == "N" or self.data_augmentation == "n": history = self.model.fit(x_batch, y_batch, batch_size=self.batch_size, epochs=self.epoch, validation_data=(x_tbatch, y_tbatch), shuffle=True, callbacks=[self.lr_reducer, self.early_stopper, self.lr_scheduler]) else: # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) height_shift_range=0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) # randomly flip images # Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(x_batch) # Fit the model on the batches generated by datagen.flow(). history = self.model.fit_generator( datagen.flow(x_batch, y_batch, batch_size=self.batch_size), validation_data=(x_tbatch, y_tbatch), epochs=self.epoch, verbose=1, workers=5, steps_per_epoch=x_batch.shape[0] // self.batch_size, callbacks=[self.lr_reducer, self.early_stopper, self.lr_scheduler]) self.loss += history.history["loss"][len(history.history["loss"])-1] self.acc += history.history["acc"][len(history.history["acc"])-1] self.val_loss += history.history["val_loss"][len(history.history["val_loss"])-1] self.val_acc += history.history["val_acc"][len(history.history["val_acc"])-1] while_cnt += 1 input_data.next() if while_cnt > 0: self.loss =self.loss/while_cnt self.acc = self.acc / while_cnt self.val_loss = self.val_loss / while_cnt self.val_acc = self.val_acc / while_cnt except Exception as e: logging.info("Error[400] ..............................................") logging.info(e) def run(self, conf_data): ''' Train run init :param conf_data: :return: ''' try : logging.info("run NeuralNetNodeImage Train") # Common Start ############################################################################################# # init value self = NeuralNetNode()._init_node_parm(self, conf_data) # netconf self.netconf = WorkFlowNetConf().get_view_obj(self.node_id) # dataconf & get data input_data, self.dataconf = self.get_input_data(self.feed_node, self.cls_pool, self.train_feed_name) test_data, self.dataconf_eval = self.get_input_data(self.feed_node, self.cls_pool, self.eval_feed_name) # Common End ############################################################################################### # Label Setup (1: HDF label row) self.labels, self.labels_cnt = self._get_netconf_labels(self.netconf, input_data, 1) self.channel = self.dataconf["preprocess"]["channel"] self.x_size = self.dataconf["preprocess"]["x_size"] self.y_size = self.dataconf["preprocess"]["y_size"] self.train_cnt = self.netconf["param"]["traincnt"] self.batch_size = self.netconf["param"]["batch_size"] self.predlog = self.netconf["param"]["predictlog"] self.optimizer = self.netconf["config"]["optimizer"] # Subtracting pixel mean improves accuracy self.subtract_pixel_mean = True # get model self.keras_get_model() if self.train_cnt == 0: self.train_batch = self.load_batch # Eval & Result Save self.eval(self.node_id, self.conf_data, test_data, None) # Eval Result Print self.eval_result_print(self.eval_data, self.predlog) else: _, self.train_batch = self.make_batch(self.node_id) self.save_path = self.model_path + "/" + str(self.train_batch) + self.file_end # Acc & Loss Init config = {"nn_id": self.nn_id, "nn_wf_ver_id": self.nn_wf_ver_id, "nn_batch_ver_id": self.train_batch} self.acc_loss_result = TrainSummaryAccLossInfo(config) for i in range(self.train_cnt): # Train self.train_run_image(input_data, test_data) # Model Save : _init_node_parm : self.save_path keras.models.save_model(self.model, self.save_path) # Acc Loss Save : _init_node_parm : self.acc_loss_result self.set_acc_loss_result(self.acc_loss_result, self.loss, self.acc, self.val_loss, self.val_acc) # Eval & Result Save self.eval(self.node_id, self.conf_data, test_data, None) # Eval Result Print self.eval_result_print(self.eval_data, self.predlog) return self.eval_data except Exception as e : logging.info("===Error on Train : {0}".format(e)) #################################################################################################################### def eval(self, node_id, conf_data, data=None, result=None): ''' eval run init :param node_id: :param conf_data: :param data: :param result: :return: ''' try : logging.info("run NeuralNetNodeImage eval") pred_cnt = self.netconf["param"]["predictcnt"] eval_type = self.netconf["config"]["eval_type"] # eval result config = {"type": eval_type, "labels": self.labels, "nn_id": self.nn_id, "nn_wf_ver_id": self.nn_wf_ver_id, "nn_batch_ver_id": self.train_batch} self.eval_data = TrainSummaryInfo(conf=config) if data is None: return self.eval_data data.reset_pointer() while (data.has_next()): data_set = data[0:data.data_size()] x_batch = self.get_convert_img_x(data_set[0], self.x_size, self.y_size, self.channel) # img_data_batch # # Normalize data. # x_batch = x_batch.astype('float32') / 255 # # If subtract pixel mean is enabled # if self.subtract_pixel_mean: # x_train_mean = np.mean(x_batch, axis=0) # x_batch -= x_train_mean logits = self.model.predict(x_batch) y_batch = self.get_convert_img_y_eval(data_set[1]) n_batch = self.get_convert_img_y_eval(data_set[2]) # File Name for i in range(len(logits)): true_name = y_batch[i] logit = [] logit.append(logits[i]) retrun_data = self.set_predict_return_cnn_img(self.labels, logit, pred_cnt) pred_name = retrun_data["key"] pred_value = retrun_data["val"] #예측값이 배열로 넘어온다 한개라도 맞으면참 t_pred_name = pred_name[0] for p_cnt in range(pred_cnt): if pred_name[p_cnt] == true_name: t_pred_name = pred_name[p_cnt] # eval result self.eval_data.set_result_info(true_name, t_pred_name) # Row log를 찍기위해서 호출한다. file_name = n_batch[i] self.eval_data.set_tf_log(file_name, true_name, pred_name, pred_value) data.next() # eval result if self.train_cnt != 0: TrainSummaryInfo.save_result_info(self, self.eval_data) return self.eval_data except Exception as e : logging.info("===Error on Eval : {0}".format(e)) #################################################################################################################### def predict(self, nn_id, ver, filelist): ''' predict :param node_id: :param filelist: :return: ''' logging.info("run NeuralNetNodeImage Predict") self.subtract_pixel_mean = True self = NeuralNetNode()._init_pred_parm(self, nn_id, ver) # net config setup self.netconf = WorkFlowNetConf().get_node_info(nn_id, ver, self.netconf_name) self.dataconf = WorkFlowNetConf().get_node_info(nn_id, ver, self.dataconf_name) # data shape change MultiValuDict -> nd array filename_arr, filedata_arr = self.change_predict_fileList(filelist, self.dataconf) # get unique key unique_key = '_'.join([str(nn_id), str(ver), self.load_batch]) logging.info("getModelPath:"+self.model_path + "/" + self.load_batch+self.file_end) ## create tensorflow graph if (NeuralNetModel.dict.get(unique_key)): self = NeuralNetModel.dict.get(unique_key) # graph = NeuralNetModel.graph.get(unique_key) else: self.keras_get_model() NeuralNetModel.dict[unique_key] = self NeuralNetModel.graph[unique_key] = tf.get_default_graph() # graph = tf.get_default_graph() pred_return_data = {} for i in range(len(filename_arr)): file_name = filename_arr[i] file_data = filedata_arr[i] # # Normalize data. # file_data = file_data.astype('float32') / 255 # # If subtract pixel mean is enabled # if self.subtract_pixel_mean: # x_train_mean = np.mean(file_data, axis=0) # file_data -= x_train_mean try: logits = self.model.predict(file_data) except Exception as e: self.keras_get_model() NeuralNetModel.dict[unique_key] = self NeuralNetModel.graph[unique_key] = tf.get_default_graph() # graph = tf.get_default_graph() logits = self.model.predict(file_data) labels = self.netconf["labels"] pred_cnt = self.netconf["param"]["predictcnt"] retrun_data = self.set_predict_return_cnn_img(labels, logits, pred_cnt) pred_return_data[file_name] = retrun_data logging.info("Return Data.......................................") logging.info(pred_return_data) return pred_return_data
def run(self, conf_data): """ Wide and Deep Network Training :param : conf_data :return: None """ logging.info("NeuralNetNode Xgboost Run called") #nodeid 필요 try: self._init_train_parm(conf_data) #self._init_value() train, test= self.get_input_data() spec = importlib.util.spec_from_file_location("data_preprocess", "/hoya_src_root/data_preprocess.py") foo = importlib.util.module_from_spec(spec) spec.loader.exec_module(foo) _label, _label_info, _label_values = foo.label_info() y_train = train[_label].ravel() x_train = train.drop([_label,"id"], axis=1) y_test = test[_label].ravel() x_test = test.drop([_label,"id"], axis=1) #x_train = train.values # Creates an array of the train data #x_test = test.values # Creats an array of the test data self.load_batch = self.get_eval_batch(self.node_id) # Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기 self.train_batch, self.batch = self.make_batch(self.node_id) logging.info("Xgboost Train get batch -> {0}".format(self.batch)) logging.info("Xgboost Train get batch -> {0}".format(self.load_batch)) if self.train_batch == None: self.model_train_path = ''.join([self.model_path + '/' + self.batch + '.bin']) else: self.model_train_path = ''.join([self.model_path + '/' + self.train_batch + '.bin']) xgb_params = self.get_xgboost_paramter() num_rounds = self.conf.get("epoch") dtrain = xgb.DMatrix(x_train, y_train) # training data dvalid = xgb.DMatrix(x_test, y_test) # validation data eval_result= {} gbm = xgb.train(xgb_params, dtrain, num_rounds, [(dtrain, 'train'),(dvalid,"test")], evals_result= eval_result ) # stop if no improvement in 10 rounds gbm.save_model(self.model_train_path ) predictions = gbm.predict(dvalid) train_prediction = gbm.predict(dvalid) #trainprediction_xgb = pd.DataFrame({'id': test, # 'predict': train_prediction}) #trainprediction_xgb_merge = train_results_xgb.merge(trainprediction_xgb, how='left', on='id') # Todo Eval flag 보도록 고치고 # "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch} config = {"nn_id": self.nn_id, "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config) acc_result.loss_info["loss"].extend(eval_result['test']['rmse']) acc_result.acc_info["acc"].extend(eval_result['test']['rmse']) self.save_accloss_info(acc_result) config = {"type": self.model_type, "labels": _label_values, "nn_id":self.nn_id, "nn_wf_ver_id":self.wf_ver} eval_result = TrainSummaryInfo(conf=config) eval_result.set_nn_batch_ver_id(self.batch) eval_result.set_result_info(y_test, train_prediction) input_data = TrainSummaryInfo.save_result_info(self, eval_result) input_data['accuracy'] = eval_result.get_accuracy() return input_data except Exception as e: logging.info("NeuralNetNodeXgboost Run Exception : {0}".format(e)) raise Exception(e)
def eval(self, node_id, conf_data, data=None, result=None): """ Tensorflow Wide and Deep Network Eval Method :param node_id: :param parm: :return: None """ logging.info("eval_starting ------> {0}".format(node_id)) try: #self._init_node_parm(conf_data['node_id']) #conf_data['node_id'] #self._init_node_parm(conf_data['node_id']) self._init_node_parm(conf_data.get('nn_id') + "_" + conf_data.get('wf_ver')+ "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) config = {"type": self.model_type, "labels": self.label_values, "nn_id":conf_data.get('nn_id'), "nn_wf_ver_id":conf_data.get('wf_ver')} train = TrainSummaryInfo(conf=config) print(config) self.batch_eval = self.get_eval_batch(node_id) #print(train) self.model_eval_path = ''.join([self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag #conf_data['cls_pool'].get('nn00001_1_pre_feed_fr2wdnn_test') logging.info("model_path : {0}".format(self.model_path)) logging.info("hidden_layers : {0}".format(self.hidden_layers)) logging.info("activation_function : {0}".format(self.activation_function)) logging.info("batch_size : {0}".format(self.batch_size)) logging.info("epoch : {0}".format(self.epoch)) logging.info("model_type : {0}".format(self.model_type)) logging.info("auto_demension : {0}".format(self.auto_demension)) config_acc = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": conf_data.get('wf_ver'), "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config_acc) data_conf_info = self.data_conf #validation_monitor = _LossCheckerHook(acc_result) # make wide & deep modelnot wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_eval_path), self.train, self.auto_demension) #, self.train, self.auto_demension # feed # TODO file이 여러개면 어떻하지? # get prev node for load data #data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') #train_data_set = self.cls_pool[data_node_name[0]] # get filename train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): logging.info("Wdnn eval process from h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict #self.batch_size = 2 for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) #print("model fitting h5 " + str(data_set)) if self.model_type == 'regression': acc = eval_result['loss'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['loss'])) else: acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) # acc = eval_result['accuracy'] # loss = eval_result['loss'] # acc_result.loss_info["loss"].append(str(eval_result['loss'])) # acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) predict_value = wdnn_model.predict( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info)) data_set_count = len(data_set.index) predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error("wdnn eval error check : dataframe count({0}) predict count({1})".format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)') data_set['predict_label'] = predict_val_list #list(predict_value) predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting logging.info("wdnn eval ori list : {0}".format(len(ori_list)) ) logging.info("wdnn eval pre list : {0}".format(len(pre_list)) ) train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch_eval) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if (self.model_type == "category" or self.model_type == "deep"): # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] train.set_result_info(str(_ori), str(le.inverse_transform(pre_list[_i]))) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") except Exception as oe: logging.info(oe) raise Exception(e) return train