def eval(self, node_id, conf_data, data=None, result=None): """ Tensorflow Wide and Deep Network Eval Method :param node_id: :param parm: :return: None """ logging.info("eval_starting ------> {0}".format(node_id)) try: #self._init_node_parm(conf_data['node_id']) #conf_data['node_id'] #self._init_node_parm(conf_data['node_id']) self._init_node_parm(conf_data.get('nn_id') + "_" + conf_data.get('wf_ver')+ "_" + "netconf_node") self.cls_pool_all = conf_data['cls_pool'] # Data feeder graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) config = {"type": self.model_type, "labels": self.label_values, "nn_id":conf_data.get('nn_id'), "nn_wf_ver_id":conf_data.get('wf_ver')} train = TrainSummaryInfo(conf=config) print(config) self.batch_eval = self.get_eval_batch(node_id) #print(train) self.model_eval_path = ''.join([self.model_path + '/' + self.batch]) for _k, _v in self.cls_pool_all.items(): if 'test' in _k: self.cls_pool = _v if 'evaldata' in _k: self.multi_node_flag = _v.multi_node_flag #conf_data['cls_pool'].get('nn00001_1_pre_feed_fr2wdnn_test') logging.info("model_path : {0}".format(self.model_path)) logging.info("hidden_layers : {0}".format(self.hidden_layers)) logging.info("activation_function : {0}".format(self.activation_function)) logging.info("batch_size : {0}".format(self.batch_size)) logging.info("epoch : {0}".format(self.epoch)) logging.info("model_type : {0}".format(self.model_type)) logging.info("auto_demension : {0}".format(self.auto_demension)) config_acc = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": conf_data.get('wf_ver'), "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config_acc) data_conf_info = self.data_conf #validation_monitor = _LossCheckerHook(acc_result) # make wide & deep modelnot wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_eval_path), self.train, self.auto_demension) #, self.train, self.auto_demension # feed # TODO file이 여러개면 어떻하지? # get prev node for load data #data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') #train_data_set = self.cls_pool[data_node_name[0]] # get filename train_data_set = self.cls_pool # get filename file_queue = str(train_data_set.input_paths[0]) # get file_name # file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 # multi Feeder modified multi_read_flag = self.multi_read_flag # Todo H5 # train per files in folder h5용 # if multi_file flag = no이면 기본이 h5임 try: results = dict() ori_list = list() pre_list = list() while (train_data_set.has_next()): logging.info("Wdnn eval process from h5") # 파일이 하나 돌때마다 # for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. # 파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file # eval should be one line predict #self.batch_size = 2 for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) #print("model fitting h5 " + str(data_set)) if self.model_type == 'regression': acc = eval_result['loss'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['loss'])) else: acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) # acc = eval_result['accuracy'] # loss = eval_result['loss'] # acc_result.loss_info["loss"].append(str(eval_result['loss'])) # acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) predict_value = wdnn_model.predict( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info)) data_set_count = len(data_set.index) predict_val_list = [_pv for _pv in predict_value] predict_val_count = len(predict_val_list) if (data_set_count != predict_val_count): logging.error("wdnn eval error check : dataframe count({0}) predict count({1})".format(data_set_count, predict_val_count)) raise ValueError( 'eval data validation check error : dataframe and predict count is different(neuralnet_node_wdnn.eval)') data_set['predict_label'] = predict_val_list #list(predict_value) predict_y = list(data_set['predict_label']) ori_list.extend(data_set[self.label].values.tolist()) pre_list.extend(list(data_set['predict_label'])) # model fitting logging.info("wdnn eval ori list : {0}".format(len(ori_list)) ) logging.info("wdnn eval pre list : {0}".format(len(pre_list)) ) train_data_set.next() #TODO : 앞으로 옮기자 train.set_nn_batch_ver_id(self.batch_eval) if self.model_type == "regression": results['ori'] = ori_list results['pre'] = pre_list train.set_result_info(ori_list, pre_list) if (self.model_type == "category" or self.model_type == "deep"): # tfrecord는 여기서 Label을 변경한다. 나중에 꺼낼때 답이 없음 Tensor 객체로 추출되기 때문에 그러나 H5는 feeder에서 변환해주자 le = LabelEncoder() le.fit(self.label_values) for _i, _ori in enumerate(ori_list): #return_value = self.labels[np.argmax(model.predict(X_train))] train.set_result_info(str(_ori), str(le.inverse_transform(pre_list[_i]))) #return self.batch except Exception as e: print("eval error") print(e) raise Exception(e) logging.info("eval end") except Exception as oe: logging.info(oe) raise Exception(e) return train
def run(self, conf_data): """ Wide and Deep Network Training :param : conf_data :return: None """ logging.info("NeuralNetNodeWdnn Run called") try: self._init_node_parm(conf_data['node_id']) if self.train == False: #Train Value 가 False면 훈련 안함 return None graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) #Model Path #self.model_path = utils.get_model_path(nnid, ver, netconf_node) #Set Data Feeder self.cls_pool = conf_data['cls_pool'] # Data feeder # set batch self.load_batch = self.get_eval_batch( conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기 self.train_batch, self.batch = self.make_batch( conf_data['node_id']) #self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) # before train batch #if self.before_train_batch != None: # self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)]) logging.info("Wdnn Train get batch -> {0}".format(self.batch)) logging.info("Wdnn Train get batch -> {0}".format(self.load_batch)) if self.train_batch == None: self.model_train_path = ''.join( [self.model_path + '/' + self.batch]) else: self.model_train_path = ''.join( [self.model_path + '/' + self.train_batch]) #Todo Eval flag 보도록 고치고 # "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch} config = { "nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver, "nn_batch_ver_id": self.batch } acc_result = TrainSummaryAccLossInfo(config) if self.load_batch != self.batch: src = ''.join([self.model_path + '/' + self.load_batch]) dst = self.model_train_path utils.copy_all(src, dst) #Optimizer and learning rate #self._optimizer_type = self.optimizer_type #self._learning_rates = self.learning_rate logging.info("model_path : {0} ".format(self.model_path)) logging.info("hidden_layers : {0} ".format(self.hidden_layers)) logging.info("activation_function : {0} ".format( self.activation_function)) logging.info("batch_size : {0} ".format(self.batch_size)) logging.info("epoch : {0} ".format(self.epoch)) logging.info("model_type : {0} ".format(self.model_type)) logging.info("optimizer_type : {0} ".format(self.optimizer_type)) logging.info("learning_rates : {0} ".format(self.learning_rates)) data_conf_info = self.data_conf # make wide & deep model wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_train_path), self.train, self.auto_demension) #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type( conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name #file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 #multi Feeder modified multi_read_flag = self.multi_read_flag #validation_monitor = _LossCheckerHook(acc_result) train_cnt = 5 #customsMonitor = EarlyStoppingHook() if multi_read_flag == True: logging.info("Reading tfrecord") for index, fn in enumerate(train_data_set.input_paths): _num_tfrecords_files += self.generator_len( tf.python_io.tf_record_iterator( fn)) # get length of generators logging.info("total loop " + str(math.ceil(_num_tfrecords_files / _batch_size))) for index in range( int(math.ceil(_num_tfrecords_files / _batch_size))): for i in range(train_cnt): logging.info("number of for loop " + str(index)) train_result = wdnn_model.fit( input_fn=lambda: train_data_set.input_fn( tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=self.epoch) #train_result = wdnn_model.fit( # input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # _batch_size), steps=self.epoch) eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn( tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=200) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append( str(eval_result['loss'])) acc_result.acc_info["acc"].append( str(eval_result['accuracy'])) else: #Todo H5 # train per files in folder h5용 logging.info("Training Wide and Deep from Reading hdf5") while (train_data_set.has_next()): for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거 logging.info( "Training WDNN Total Count {0} out of {1}".format( i + self.batch_size, train_data_set.data_size())) data_set = train_data_set[i:i + self.batch_size] for t_i in range(train_cnt): logging.info( "Training WDNN Train Count {0} out of {1}". format(t_i, train_cnt)) #data_set = train_data_set[i:i + self.batch_size] if i == 0: eval_data_Set = data_set train_result = wdnn_model.fit( input_fn=lambda: train_data_set.input_fn2( tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=self.epoch) eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn2( tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) logging.info( "wdnn training complete count from h5 : {0} ". format(len(data_set))) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append( str(eval_result['loss'])) acc_result.acc_info["acc"].append( str(eval_result['accuracy'])) #train_result = wdnn_model.fit( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info), steps=self.epoch) logging.info( "Traing Result -> {0}".format(train_result)) train_data_set.next() print("end") self.save_accloss_info(acc_result) return None except Exception as e: logging.info("[Wide and Deep Train Process] : {0}".format(e)) raise Exception(e)