def run(self, conf_data): logging.info("NeuralNetNodeWdnn Run called") #return None #return None """ Wide & Deep Network Training :param nnid : network id in tfmsacore_nninfo :return: acturacy """ try: self._init_node_parm(conf_data['node_id']) if self.train == False: return None self.cls_pool = conf_data['cls_pool'] # Data feeder self.train_batch, self.batch = self.make_batch( conf_data['node_id']) #makebatch self.before_train_batch = self.get_before_make_batch( conf_data['node_id'], self.batch) #before train batch if self.before_train_batch != None: self.model_train_before_path = ''.join([ self.model_path + '/' + str(self.before_train_batch.nn_batch_ver_id) ]) if self.train_batch == None: self.model_train_path = ''.join( [self.model_path + '/' + self.batch]) else: self.model_train_path = ''.join( [self.model_path + '/' + self.train_batch]) #model file copy if self.before_train_batch != None: src = self.model_train_before_path dst = self.model_train_path utils.copy_all(src, dst) logging.info("model_path : {0} ".format(self.model_path)) logging.info("hidden_layers : {0} ".format(self.hidden_layers)) logging.info("activation_function : {0} ".format( self.activation_function)) logging.info("batch_size : {0} ".format(self.batch_size)) logging.info("epoch : {0} ".format(self.epoch)) logging.info("model_type : {0} ".format(self.model_type)) data_conf_info = self.data_conf # make wide & deep model wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'], self.hidden_layers, str(self.activation_function), data_conf_info, str(self.model_train_path), self.train, self.auto_demension) #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type( conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name #file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 #_batch_size = 2 #multi Feeder modified multi_read_flag = self.multi_read_flag if multi_read_flag == True: for index, fn in enumerate(train_data_set.input_paths): _num_tfrecords_files += self.generator_len( tf.python_io.tf_record_iterator( fn)) # get length of generators print("total loop " + str(math.ceil(_num_tfrecords_files / _batch_size))) for index in range( int(math.ceil(_num_tfrecords_files / _batch_size))): print("number of for loop " + str(index)) wdnn_model.fit(input_fn=lambda: train_data_set.input_fn( tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=self.epoch) results = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn( tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=1) else: #Todo H5 # train per files in folder h5용 #if multi_file flag = no이면 기본이 h5임 while (train_data_set.has_next()): print("h5") #파일이 하나 돌때마다 #for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. #파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 # # # Iteration is to improve for Model Accuracy # Per Line in file for i in range(0, train_data_set.data_size(), self.batch_size): data_set = train_data_set[i:i + self.batch_size] if i == 0: eval_data_Set = data_set #input_fn2(self, mode, data_file, df, nnid, dataconf): wdnn_model.fit( input_fn=lambda: train_data_set.input_fn2( tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) #model fitting print("model fitting h5 " + str(data_set)) # #Select Next file train_data_set.next() results = dict() #results = wdnn_model.evaluate( #input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # eval_data_Set, data_conf_info), steps=200) for key in sorted(results): print("%s: %s" % (key, results[key])) logging.info("train data eval result : {0} : {1}".format( key, results[key])) #feature_map, target = train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, 128) print("end") #with tf.Session() as sess: return None except Exception as e: logging.info("[Wide and Deep Train Process] : {0}".format(e)) raise Exception(e)
def run(self, conf_data): logging.info("NeuralNetNodeWdnn Run called") #return None #return None """ Wide & Deep Network Training :param nnid : network id in tfmsacore_nninfo :return: acturacy """ try: self._init_node_parm(conf_data['node_id']) self.cls_pool = conf_data['cls_pool'] # Data feeder self.train_batch, self.batch = self.make_batch(conf_data['node_id']) #makebatch self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) #before train batch if self.before_train_batch != None: self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)]) if self.train_batch == None : self.model_train_path = ''.join([self.model_path+'/'+self.batch]) else : self.model_train_path = ''.join([self.model_path + '/' + self.train_batch]) #model file copy if self.before_train_batch != None: src = self.model_train_before_path dst = self.model_train_path utils.copy_all(src, dst) logging.info("model_path : {0} ".format(self.model_path)) logging.info("hidden_layers : {0} ".format(self.hidden_layers)) logging.info("activation_function : {0} ".format(self.activation_function)) logging.info("batch_size : {0} ".format(self.batch_size)) logging.info("epoch : {0} ".format(self.epoch)) logging.info("model_type : {0} ".format(self.model_type)) data_conf_info = self.data_conf #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name #TODO 아 eval이 안되서 데이터가 안불러 _batch_size = self.batch_size _num_tfrecords_files = 0 #_batch_size = 2 data_set = train_data_set[0 : train_data_set.data_size() ] input_dims = len(data_set.columns)-1 # label 갯수 제거 # model = Sequential() # model.add(Dense(100, input_dim=input_dims, activation='relu')) # #model.add(Dense(50)) # model.add(Dense(50, activation='sigmoid')) # #model.add(Dense(50, activation='sigmoid')) # model.compile(optimizer='rmsprop', # loss='binary_crossentropy', # metrics=['accuracy']) model = Sequential() model.add(Dense(10, input_dim=input_dims,init='uniform')) model.add(BatchNormalization()) model.add(Activation('sigmoid')) #model.add(Dense(10, init='uniform')) #model.add(BatchNormalization()) #model.add(Activation('sigmoid')) model.add(Dense(1, init='uniform')) model.add(BatchNormalization()) model.add(Activation('sigmoid')) lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) early_stopper = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10) csv_logger = CSVLogger('resnet18_cifar10.csv') history = History() # Compile model model.compile(loss='binary_crossentropy', optimizer='sgd') self.batch_size = 4 #multi Feeder modified multi_read_flag = self.multi_read_flag if multi_read_flag == False: #Todo H5 # train per files in folder h5용 while(train_data_set.has_next()) : logging.info("start keras dnn") #파일이 하나 돌때마다 #for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. #파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 #for x in range(0, self.iter_size): for i in range(0, train_data_set.data_size(), self.batch_size): #Last batch size preprocessing if (i+self.batch_size > train_data_set.data_size()): i= i - (train_data_set.data_size()%self.batch_size) + 1 data_set = train_data_set[i:i + self.batch_size] #data_set = train_data_set[0:train_data_set.data_size()] logging.info(i) X_train, targets, = train_data_set.input_fn3( file_queue,data_set,data_conf_info) #loss, accuracy = model.fit(X_train, targets , epochs=10 , validation_data=(X_train, targets) , batch_size=self.batch_size , callbacks = [lr_reducer, csv_logger, history] ) #logging.info("keras training info loss : {0} , accuracy{1}".format(loss, accuracy)) # #Select Next file train_data_set.next() #os.makedirs(self.md_store_path + '/' + self.batch, exist_ok=True) # keras.models.save_model(model, ''.join([self.md_store_path + '/' + self.batch, '/model.bin'])) print("end") except Exception as e: logging.error("Error Message : {0}".format(e)) raise Exception(e) return None
def run(self, conf_data): """ Wide and Deep Network Training :param : conf_data :return: None """ logging.info("NeuralNetNodeWdnn Run called") try: self._init_node_parm(conf_data['node_id']) if self.train == False: #Train Value 가 False면 훈련 안함 return None graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) #Model Path #self.model_path = utils.get_model_path(nnid, ver, netconf_node) #Set Data Feeder self.cls_pool = conf_data['cls_pool'] # Data feeder # set batch self.load_batch = self.get_eval_batch(conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기 self.train_batch, self.batch = self.make_batch(conf_data['node_id']) #self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) # before train batch #if self.before_train_batch != None: # self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)]) logging.info("Wdnn Train get batch -> {0}".format(self.batch)) logging.info("Wdnn Train get batch -> {0}".format(self.load_batch)) if self.train_batch == None : self.model_train_path = ''.join([self.model_path+'/'+self.batch]) else : self.model_train_path = ''.join([self.model_path + '/' + self.train_batch]) #Todo Eval flag 보도록 고치고 # "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch} config = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver, "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config) if self.load_batch != self.batch: src = ''.join([self.model_path+'/'+self.load_batch]) dst = self.model_train_path utils.copy_all(src, dst) #Optimizer and learning rate #self._optimizer_type = self.optimizer_type #self._learning_rates = self.learning_rate logging.info("model_path : {0} ".format(self.model_path)) logging.info("hidden_layers : {0} ".format(self.hidden_layers)) logging.info("activation_function : {0} ".format(self.activation_function)) logging.info("batch_size : {0} ".format(self.batch_size)) logging.info("epoch : {0} ".format(self.epoch)) logging.info("model_type : {0} ".format(self.model_type)) logging.info("optimizer_type : {0} ".format(self.optimizer_type)) logging.info("learning_rates : {0} ".format(self.learning_rates)) data_conf_info = self.data_conf # make wide & deep model wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'],self.hidden_layers,str(self.activation_function),data_conf_info, str(self.model_train_path),self.train, self.auto_demension) #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name #file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 #multi Feeder modified multi_read_flag = self.multi_read_flag #validation_monitor = _LossCheckerHook(acc_result) train_cnt = 5 #customsMonitor = EarlyStoppingHook() if multi_read_flag == True: logging.info("Reading tfrecord") for index, fn in enumerate(train_data_set.input_paths): _num_tfrecords_files += self.generator_len( tf.python_io.tf_record_iterator(fn)) # get length of generators logging.info("total loop " + str(math.ceil(_num_tfrecords_files/_batch_size)) ) for index in range(int(math.ceil(_num_tfrecords_files/_batch_size))): for i in range(train_cnt): logging.info("number of for loop " + str(index)) train_result = wdnn_model.fit(input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,_batch_size), steps=self.epoch) #train_result = wdnn_model.fit( # input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # _batch_size), steps=self.epoch) eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=200) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) else: #Todo H5 # train per files in folder h5용 logging.info("Training Wide and Deep from Reading hdf5") while(train_data_set.has_next()) : for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거 logging.info("Training WDNN Total Count {0} out of {1}".format(i+self.batch_size, train_data_set.data_size())) data_set = train_data_set[i:i + self.batch_size] for t_i in range(train_cnt): logging.info( "Training WDNN Train Count {0} out of {1}".format(t_i, train_cnt)) #data_set = train_data_set[i:i + self.batch_size] if i == 0: eval_data_Set = data_set train_result = wdnn_model.fit( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set,data_conf_info), steps=self.epoch) eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) logging.info("wdnn training complete count from h5 : {0} ".format(len(data_set))) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) #train_result = wdnn_model.fit( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info), steps=self.epoch) logging.info("Traing Result -> {0}".format(train_result)) train_data_set.next() print("end") self.save_accloss_info(acc_result) return None except Exception as e: logging.info("[Wide and Deep Train Process] : {0}".format(e)) raise Exception(e)
def run(self, conf_data): """ ML Training :param : conf_data :return: None """ logging.info("ML Run called") try: self._init_node_parm(conf_data['node_id']) graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] if net['fields']['graph_node'] == 'eval_feed': eval_feed_name = conf_data['nn_id'] + "_" + conf_data['wf_ver'] + "_" + net['fields']['graph_node_name'] # Model Path self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) #Set Data Feeder self.cls_pool = conf_data['cls_pool'] # Data feeder # set batch self.load_batch = self.get_eval_batch(conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기 self.train_batch, self.batch = self.make_batch(conf_data['node_id']) logging.info("ML Train get batch -> {0}".format(self.batch)) logging.info("ML Train get batch -> {0}".format(self.load_batch)) if self.train_batch == None : self.model_train_path = ''.join([self.model_path+'/'+self.batch]) else : self.model_train_path = ''.join([self.model_path + '/' + self.train_batch]) config = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver, "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config) if self.load_batch != self.batch: src = ''.join([self.model_path+'/'+self.load_batch]) dst = self.model_train_path utils.copy_all(src, dst) logging.info("model_path : {0} ".format(self.model_path)) logging.info("ml_class : {0} ".format(self.ml_class)) logging.info("config : {0} ".format(self.config)) data_conf_info = self.data_conf # make ml_class if self.ml_class == 'DecisionTreeClassifier': max_depth = self.config["max_depth"] clf = tree.DecisionTreeClassifier(max_depth=max_depth) #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') #train_node_name = self._get_forward_node_with_type(conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name test_data_set = self.cls_pool[eval_feed_name] #file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 #multi Feeder modified multi_read_flag = False train_cnt = 1 if multi_read_flag == True: logging.info("Reading tfrecord") for index, fn in enumerate(train_data_set.input_paths): _num_tfrecords_files += self.generator_len( tf.python_io.tf_record_iterator(fn)) # get length of generators logging.info("total loop " + str(math.ceil(_num_tfrecords_files/_batch_size)) ) for index in range(int(math.ceil(_num_tfrecords_files/_batch_size))): for i in range(train_cnt): logging.info("number of for loop " + str(index)) train_result = clf.fit(input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,_batch_size), steps=self.epoch) eval_result = clf.evaluate( input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=200) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) else: #Todo H5 # train per files in folder h5용 logging.info("Training ML from Reading hdf5") while(train_data_set.has_next()) : for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거 logging.info("Training ML Total Count {0} out of {1}".format(i+self.batch_size, train_data_set.data_size())) data_set = train_data_set[i:i + self.batch_size] data_set_test = test_data_set[i:i +self.batch_size] for t_i in range(train_cnt): logging.info( "Training ML Train Count {0} out of {1}".format(t_i, train_cnt)) if i == 0: eval_data_Set = data_set iris = load_iris() keys = list(data_conf_info['cell_feature'].keys()) keys.remove(data_conf_info['label']) feature_names = keys keys = np.asarray(keys) data = data_set[keys].values test_data = data_set_test[keys].values label = data_set[data_conf_info['label']].values test_label = data_set_test[data_conf_info['label']].values clf = clf.fit(data,label) target_names = np.asarray(self.label_values) dot_data = tree.export_graphviz(clf,out_file=None, feature_names=feature_names, class_names=target_names, filled=True, rounded=True, special_characters=True) graph1 = graphviz.Source(dot_data) graph1.render(self.model_path+'/model') # eval_result = clf.evaluate( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info), steps=200) logging.info("ML training complete count from h5 : {0} ".format(len(data_set))) acc = cross_val_score(clf,test_data,test_label,scoring='accuracy').mean() loss = cross_val_score(clf,data,label,scoring='neg_log_loss').mean() # acc = eval_result['accuracy'] # loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(loss)) acc_result.acc_info["acc"].append(str(acc)) #logging.info("Traing Result -> {0}".format(train_result)) train_data_set.next() print("end") self.save_accloss_info(acc_result) joblib.dump(clf,self.model_path+'/model.pkl') return None except Exception as e: logging.info("[Wide and Deep Train Process] : {0}".format(e)) raise Exception(e)
def run(self, conf_data): """ ML Training :param : conf_data :return: None """ logging.info("ML Run called") try: self._init_node_parm(conf_data['node_id']) graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] if net['fields']['graph_node'] == 'eval_feed': eval_feed_name = conf_data['nn_id'] + "_" + conf_data[ 'wf_ver'] + "_" + net['fields']['graph_node_name'] # Model Path self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) #Set Data Feeder self.cls_pool = conf_data['cls_pool'] # Data feeder # set batch self.load_batch = self.get_eval_batch( conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기 self.train_batch, self.batch = self.make_batch( conf_data['node_id']) logging.info("ML Train get batch -> {0}".format(self.batch)) logging.info("ML Train get batch -> {0}".format(self.load_batch)) if self.train_batch == None: self.model_train_path = ''.join( [self.model_path + '/' + self.batch]) else: self.model_train_path = ''.join( [self.model_path + '/' + self.train_batch]) config = { "nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver, "nn_batch_ver_id": self.batch } acc_result = TrainSummaryAccLossInfo(config) if self.load_batch != self.batch: src = ''.join([self.model_path + '/' + self.load_batch]) dst = self.model_train_path utils.copy_all(src, dst) logging.info("model_path : {0} ".format(self.model_path)) logging.info("ml_class : {0} ".format(self.ml_class)) logging.info("config : {0} ".format(self.config)) data_conf_info = self.data_conf # make ml_class if self.ml_class == 'DecisionTreeClassifier': max_depth = self.config["max_depth"] clf = tree.DecisionTreeClassifier(max_depth=max_depth) #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type( conf_data['node_id'], 'preprocess') #train_node_name = self._get_forward_node_with_type(conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name test_data_set = self.cls_pool[eval_feed_name] #file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 #multi Feeder modified multi_read_flag = False train_cnt = 1 if multi_read_flag == True: logging.info("Reading tfrecord") for index, fn in enumerate(train_data_set.input_paths): _num_tfrecords_files += self.generator_len( tf.python_io.tf_record_iterator( fn)) # get length of generators logging.info("total loop " + str(math.ceil(_num_tfrecords_files / _batch_size))) for index in range( int(math.ceil(_num_tfrecords_files / _batch_size))): for i in range(train_cnt): logging.info("number of for loop " + str(index)) train_result = clf.fit( input_fn=lambda: train_data_set.input_fn( tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=self.epoch) eval_result = clf.evaluate( input_fn=lambda: train_data_set.input_fn( tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=200) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append( str(eval_result['loss'])) acc_result.acc_info["acc"].append( str(eval_result['accuracy'])) else: #Todo H5 # train per files in folder h5용 logging.info("Training ML from Reading hdf5") while (train_data_set.has_next()): for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거 logging.info( "Training ML Total Count {0} out of {1}".format( i + self.batch_size, train_data_set.data_size())) data_set = train_data_set[i:i + self.batch_size] data_set_test = test_data_set[i:i + self.batch_size] for t_i in range(train_cnt): logging.info( "Training ML Train Count {0} out of {1}". format(t_i, train_cnt)) if i == 0: eval_data_Set = data_set iris = load_iris() keys = list(data_conf_info['cell_feature'].keys()) keys.remove(data_conf_info['label']) feature_names = keys keys = np.asarray(keys) data = data_set[keys].values test_data = data_set_test[keys].values label = data_set[data_conf_info['label']].values test_label = data_set_test[ data_conf_info['label']].values clf = clf.fit(data, label) target_names = np.asarray(self.label_values) dot_data = tree.export_graphviz( clf, out_file=None, feature_names=feature_names, class_names=target_names, filled=True, rounded=True, special_characters=True) graph1 = graphviz.Source(dot_data) graph1.render(self.model_path + '/model') # eval_result = clf.evaluate( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info), steps=200) logging.info( "ML training complete count from h5 : {0} ". format(len(data_set))) acc = cross_val_score(clf, test_data, test_label, scoring='accuracy').mean() loss = cross_val_score( clf, data, label, scoring='neg_log_loss').mean() # acc = eval_result['accuracy'] # loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(loss)) acc_result.acc_info["acc"].append(str(acc)) #logging.info("Traing Result -> {0}".format(train_result)) train_data_set.next() print("end") self.save_accloss_info(acc_result) joblib.dump(clf, self.model_path + '/model.pkl') return None except Exception as e: logging.info("[Wide and Deep Train Process] : {0}".format(e)) raise Exception(e)
def run(self, conf_data): logging.info("NeuralNetNodeWdnn Run called") #return None #return None """ Wide & Deep Network Training :param nnid : network id in tfmsacore_nninfo :return: acturacy """ try: self._init_node_parm(conf_data['node_id']) self.cls_pool = conf_data['cls_pool'] # Data feeder self.train_batch, self.batch = self.make_batch( conf_data['node_id']) #makebatch self.before_train_batch = self.get_before_make_batch( conf_data['node_id'], self.batch) #before train batch if self.before_train_batch != None: self.model_train_before_path = ''.join([ self.model_path + '/' + str(self.before_train_batch.nn_batch_ver_id) ]) if self.train_batch == None: self.model_train_path = ''.join( [self.model_path + '/' + self.batch]) else: self.model_train_path = ''.join( [self.model_path + '/' + self.train_batch]) #model file copy if self.before_train_batch != None: src = self.model_train_before_path dst = self.model_train_path utils.copy_all(src, dst) logging.info("model_path : {0} ".format(self.model_path)) logging.info("hidden_layers : {0} ".format(self.hidden_layers)) logging.info("activation_function : {0} ".format( self.activation_function)) logging.info("batch_size : {0} ".format(self.batch_size)) logging.info("epoch : {0} ".format(self.epoch)) logging.info("model_type : {0} ".format(self.model_type)) data_conf_info = self.data_conf #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type( conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name #TODO 아 eval이 안되서 데이터가 안불러 _batch_size = self.batch_size _num_tfrecords_files = 0 #_batch_size = 2 data_set = train_data_set[0:train_data_set.data_size()] input_dims = len(data_set.columns) - 1 # label 갯수 제거 # model = Sequential() # model.add(Dense(100, input_dim=input_dims, activation='relu')) # #model.add(Dense(50)) # model.add(Dense(50, activation='sigmoid')) # #model.add(Dense(50, activation='sigmoid')) # model.compile(optimizer='rmsprop', # loss='binary_crossentropy', # metrics=['accuracy']) model = Sequential() model.add(Dense(10, input_dim=input_dims, init='uniform')) model.add(BatchNormalization()) model.add(Activation('sigmoid')) #model.add(Dense(10, init='uniform')) #model.add(BatchNormalization()) #model.add(Activation('sigmoid')) model.add(Dense(1, init='uniform')) model.add(BatchNormalization()) model.add(Activation('sigmoid')) lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) early_stopper = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10) csv_logger = CSVLogger('resnet18_cifar10.csv') history = History() # Compile model model.compile(loss='binary_crossentropy', optimizer='sgd') self.batch_size = 4 #multi Feeder modified multi_read_flag = self.multi_read_flag if multi_read_flag == False: #Todo H5 # train per files in folder h5용 while (train_data_set.has_next()): logging.info("start keras dnn") #파일이 하나 돌때마다 #for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다. #파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고 # Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?) # -> 그러면서 피팅 #for x in range(0, self.iter_size): for i in range(0, train_data_set.data_size(), self.batch_size): #Last batch size preprocessing if (i + self.batch_size > train_data_set.data_size()): i = i - (train_data_set.data_size() % self.batch_size) + 1 data_set = train_data_set[i:i + self.batch_size] #data_set = train_data_set[0:train_data_set.data_size()] logging.info(i) X_train, targets, = train_data_set.input_fn3( file_queue, data_set, data_conf_info) #loss, accuracy = model.fit(X_train, targets, epochs=10, validation_data=(X_train, targets), batch_size=self.batch_size, callbacks=[lr_reducer, csv_logger, history]) #logging.info("keras training info loss : {0} , accuracy{1}".format(loss, accuracy)) # #Select Next file train_data_set.next() #os.makedirs(self.md_store_path + '/' + self.batch, exist_ok=True) # keras.models.save_model(model, ''.join([self.md_store_path + '/' + self.batch, '/model.bin'])) print("end") except Exception as e: logging.error("Error Message : {0}".format(e)) raise Exception(e) return None
def run(self, conf_data): """ Wide and Deep Network Training :param : conf_data :return: None """ logging.info("NeuralNetNodeWdnn Run called") try: self._init_node_parm(conf_data['node_id']) if self.train == False: #Train Value 가 False면 훈련 안함 return None graph = NNCommonManager().get_nn_node_name(conf_data['nn_id']) for net in graph: if net['fields']['graph_node'] == 'netconf_node': netconf_node = net['fields']['graph_node_name'] self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node) #Model Path #self.model_path = utils.get_model_path(nnid, ver, netconf_node) #Set Data Feeder self.cls_pool = conf_data['cls_pool'] # Data feeder # set batch self.load_batch = self.get_eval_batch(conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기 self.train_batch, self.batch = self.make_batch(conf_data['node_id']) #self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) # before train batch #if self.before_train_batch != None: # self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)]) logging.info("Wdnn Train get batch -> {0}".format(self.batch)) logging.info("Wdnn Train get batch -> {0}".format(self.load_batch)) if self.train_batch == None : self.model_train_path = ''.join([self.model_path+'/'+self.batch]) else : self.model_train_path = ''.join([self.model_path + '/' + self.train_batch]) #Todo Eval flag 보도록 고치고 # "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch} config = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver, "nn_batch_ver_id": self.batch} acc_result = TrainSummaryAccLossInfo(config) if self.load_batch != self.batch: src = ''.join([self.model_path+'/'+self.load_batch]) dst = self.model_train_path utils.copy_all(src, dst) #Optimizer and learning rate #self._optimizer_type = self.optimizer_type #self._learning_rates = self.learning_rate logging.info("model_path : {0} ".format(self.model_path)) logging.info("hidden_layers : {0} ".format(self.hidden_layers)) logging.info("activation_function : {0} ".format(self.activation_function)) logging.info("batch_size : {0} ".format(self.batch_size)) logging.info("epoch : {0} ".format(self.epoch)) logging.info("model_type : {0} ".format(self.model_type)) logging.info("optimizer_type : {0} ".format(self.optimizer_type)) logging.info("learning_rates : {0} ".format(self.learning_rates)) data_conf_info = self.data_conf # make wide & deep model wdnn = NeuralCommonWdnn() wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'],self.hidden_layers,str(self.activation_function),data_conf_info, str(self.model_train_path),self.train, self.auto_demension) #feed # TODO file이 여러개면 어떻하지? # get prev node for load data data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess') train_data_set = self.cls_pool[data_node_name[0]] #get filename file_queue = str(train_data_set.input_paths[0]) #get file_name #file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요 _batch_size = self.batch_size _num_tfrecords_files = 0 #multi Feeder modified multi_read_flag = self.multi_read_flag #validation_monitor = _LossCheckerHook(acc_result) train_cnt = 5 #customsMonitor = EarlyStoppingHook() if multi_read_flag == True: logging.info("Reading tfrecord") for index, fn in enumerate(train_data_set.input_paths): _num_tfrecords_files += self.generator_len( tf.python_io.tf_record_iterator(fn)) # get length of generators logging.info("total loop " + str(math.ceil(_num_tfrecords_files/_batch_size)) ) for index in range(int(math.ceil(_num_tfrecords_files/_batch_size))): for i in range(train_cnt): logging.info("number of for loop " + str(index)) train_result = wdnn_model.fit(input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,_batch_size), steps=self.epoch) #train_result = wdnn_model.fit( # input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # _batch_size), steps=self.epoch) eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, _batch_size), steps=200) acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) else: #Todo H5 # train per files in folder h5용 logging.info("Training Wide and Deep from Reading hdf5") while(train_data_set.has_next()) : for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거 logging.info("Training WDNN Total Count {0} out of {1}".format(i+self.batch_size, train_data_set.data_size())) data_set = train_data_set[i:i + self.batch_size] for t_i in range(train_cnt): logging.info( "Training WDNN Train Count {0} out of {1}".format(t_i, train_cnt)) #data_set = train_data_set[i:i + self.batch_size] if i == 0: eval_data_Set = data_set train_result = wdnn_model.fit( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set,data_conf_info), steps=self.epoch) eval_result = wdnn_model.evaluate( input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, data_set, data_conf_info), steps=200) logging.info("wdnn training complete count from h5 : {0} ".format(len(data_set))) if self.model_type == 'regression': acc = eval_result['loss'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['loss'])) else: acc = eval_result['accuracy'] loss = eval_result['loss'] acc_result.loss_info["loss"].append(str(eval_result['loss'])) acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) #acc_result.loss_info["loss"].append(str(eval_result['loss'])) #acc_result.acc_info["acc"].append(str(eval_result['accuracy'])) #train_result = wdnn_model.fit( # input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue, # data_set, data_conf_info), steps=self.epoch) logging.info("Traing Result -> {0}".format(train_result)) train_data_set.next() print("end") self.save_accloss_info(acc_result) return None except Exception as e: logging.info("[Wide and Deep Train Process] : {0}".format(e)) raise Exception(e)