def run(self, conf_data):
logging.info("NeuralNetNodeWdnn Run called")
#return None
#return None
"""
Wide & Deep Network Training
:param nnid : network id in tfmsacore_nninfo
:return: acturacy
"""
try:
self._init_node_parm(conf_data['node_id'])
if self.train == False:
return None
self.cls_pool = conf_data['cls_pool'] # Data feeder
self.train_batch, self.batch = self.make_batch(
conf_data['node_id']) #makebatch
self.before_train_batch = self.get_before_make_batch(
conf_data['node_id'], self.batch) #before train batch
if self.before_train_batch != None:
self.model_train_before_path = ''.join([
self.model_path + '/' +
str(self.before_train_batch.nn_batch_ver_id)
])
if self.train_batch == None:
self.model_train_path = ''.join(
[self.model_path + '/' + self.batch])
else:
self.model_train_path = ''.join(
[self.model_path + '/' + self.train_batch])
#model file copy
if self.before_train_batch != None:
src = self.model_train_before_path
dst = self.model_train_path
utils.copy_all(src, dst)
logging.info("model_path : {0} ".format(self.model_path))
logging.info("hidden_layers : {0} ".format(self.hidden_layers))
logging.info("activation_function : {0} ".format(
self.activation_function))
logging.info("batch_size : {0} ".format(self.batch_size))
logging.info("epoch : {0} ".format(self.epoch))
logging.info("model_type : {0} ".format(self.model_type))
data_conf_info = self.data_conf
# make wide & deep model
wdnn = NeuralCommonWdnn()
wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'],
self.hidden_layers,
str(self.activation_function),
data_conf_info,
str(self.model_train_path),
self.train, self.auto_demension)
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(
conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
#file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요
_batch_size = self.batch_size
_num_tfrecords_files = 0
#_batch_size = 2
#multi Feeder modified
multi_read_flag = self.multi_read_flag
if multi_read_flag == True:
for index, fn in enumerate(train_data_set.input_paths):
_num_tfrecords_files += self.generator_len(
tf.python_io.tf_record_iterator(
fn)) # get length of generators
print("total loop " +
str(math.ceil(_num_tfrecords_files / _batch_size)))
for index in range(
int(math.ceil(_num_tfrecords_files / _batch_size))):
print("number of for loop " + str(index))
wdnn_model.fit(input_fn=lambda: train_data_set.input_fn(
tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size),
steps=self.epoch)
results = wdnn_model.evaluate(
input_fn=lambda: train_data_set.input_fn(
tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size),
steps=1)
else:
#Todo H5
# train per files in folder h5용
#if multi_file flag = no이면 기본이 h5임
while (train_data_set.has_next()):
print("h5")
#파일이 하나 돌때마다
#for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다.
#파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고
# Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?)
# -> 그러면서 피팅
#
# # Iteration is to improve for Model Accuracy
# Per Line in file
for i in range(0, train_data_set.data_size(),
self.batch_size):
data_set = train_data_set[i:i + self.batch_size]
if i == 0:
eval_data_Set = data_set
#input_fn2(self, mode, data_file, df, nnid, dataconf):
wdnn_model.fit(
input_fn=lambda: train_data_set.input_fn2(
tf.contrib.learn.ModeKeys.TRAIN, file_queue,
data_set, data_conf_info),
steps=200)
#model fitting
print("model fitting h5 " + str(data_set))
# #Select Next file
train_data_set.next()
results = dict()
#results = wdnn_model.evaluate(
#input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# eval_data_Set, data_conf_info), steps=200)
for key in sorted(results):
print("%s: %s" % (key, results[key]))
logging.info("train data eval result : {0} : {1}".format(
key, results[key]))
#feature_map, target = train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue, 128)
print("end")
#with tf.Session() as sess:
return None
except Exception as e:
logging.info("[Wide and Deep Train Process] : {0}".format(e))
raise Exception(e)
def run(self, conf_data):
logging.info("NeuralNetNodeWdnn Run called")
#return None
#return None
"""
Wide & Deep Network Training
:param nnid : network id in tfmsacore_nninfo
:return: acturacy
"""
try:
self._init_node_parm(conf_data['node_id'])
self.cls_pool = conf_data['cls_pool'] # Data feeder
self.train_batch, self.batch = self.make_batch(conf_data['node_id']) #makebatch
self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) #before train batch
if self.before_train_batch != None:
self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)])
if self.train_batch == None :
self.model_train_path = ''.join([self.model_path+'/'+self.batch])
else :
self.model_train_path = ''.join([self.model_path + '/' + self.train_batch])
#model file copy
if self.before_train_batch != None:
src = self.model_train_before_path
dst = self.model_train_path
utils.copy_all(src, dst)
logging.info("model_path : {0} ".format(self.model_path))
logging.info("hidden_layers : {0} ".format(self.hidden_layers))
logging.info("activation_function : {0} ".format(self.activation_function))
logging.info("batch_size : {0} ".format(self.batch_size))
logging.info("epoch : {0} ".format(self.epoch))
logging.info("model_type : {0} ".format(self.model_type))
data_conf_info = self.data_conf
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
#TODO 아 eval이 안되서 데이터가 안불러
_batch_size = self.batch_size
_num_tfrecords_files = 0
#_batch_size = 2
data_set = train_data_set[0 : train_data_set.data_size() ]
input_dims = len(data_set.columns)-1 # label 갯수 제거
# model = Sequential()
# model.add(Dense(100, input_dim=input_dims, activation='relu'))
# #model.add(Dense(50))
# model.add(Dense(50, activation='sigmoid'))
# #model.add(Dense(50, activation='sigmoid'))
# model.compile(optimizer='rmsprop',
# loss='binary_crossentropy',
# metrics=['accuracy'])
model = Sequential()
model.add(Dense(10, input_dim=input_dims,init='uniform'))
model.add(BatchNormalization())
model.add(Activation('sigmoid'))
#model.add(Dense(10, init='uniform'))
#model.add(BatchNormalization())
#model.add(Activation('sigmoid'))
model.add(Dense(1, init='uniform'))
model.add(BatchNormalization())
model.add(Activation('sigmoid'))
lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6)
early_stopper = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10)
csv_logger = CSVLogger('resnet18_cifar10.csv')
history = History()
# Compile model
model.compile(loss='binary_crossentropy', optimizer='sgd')
self.batch_size = 4
#multi Feeder modified
multi_read_flag = self.multi_read_flag
if multi_read_flag == False:
#Todo H5
# train per files in folder h5용
while(train_data_set.has_next()) :
logging.info("start keras dnn")
#파일이 하나 돌때마다
#for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다.
#파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고
# Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?)
# -> 그러면서 피팅
#for x in range(0, self.iter_size):
for i in range(0, train_data_set.data_size(), self.batch_size):
#Last batch size preprocessing
if (i+self.batch_size > train_data_set.data_size()):
i= i - (train_data_set.data_size()%self.batch_size) + 1
data_set = train_data_set[i:i + self.batch_size]
#data_set = train_data_set[0:train_data_set.data_size()]
logging.info(i)
X_train, targets, = train_data_set.input_fn3( file_queue,data_set,data_conf_info)
#loss, accuracy =
model.fit(X_train, targets
, epochs=10
, validation_data=(X_train, targets)
, batch_size=self.batch_size
, callbacks = [lr_reducer, csv_logger, history]
)
#logging.info("keras training info loss : {0} , accuracy{1}".format(loss, accuracy))
# #Select Next file
train_data_set.next()
#os.makedirs(self.md_store_path + '/' + self.batch, exist_ok=True)
# keras.models.save_model(model, ''.join([self.md_store_path + '/' + self.batch, '/model.bin']))
print("end")
except Exception as e:
logging.error("Error Message : {0}".format(e))
raise Exception(e)
return None
def run(self, conf_data):
"""
Wide and Deep Network Training
:param : conf_data
:return: None
"""
logging.info("NeuralNetNodeWdnn Run called")
try:
self._init_node_parm(conf_data['node_id'])
if self.train == False: #Train Value 가 False면 훈련 안함
return None
graph = NNCommonManager().get_nn_node_name(conf_data['nn_id'])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
netconf_node = net['fields']['graph_node_name']
self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node)
#Model Path
#self.model_path = utils.get_model_path(nnid, ver, netconf_node)
#Set Data Feeder
self.cls_pool = conf_data['cls_pool'] # Data feeder
# set batch
self.load_batch = self.get_eval_batch(conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기
self.train_batch, self.batch = self.make_batch(conf_data['node_id'])
#self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) # before train batch
#if self.before_train_batch != None:
# self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)])
logging.info("Wdnn Train get batch -> {0}".format(self.batch))
logging.info("Wdnn Train get batch -> {0}".format(self.load_batch))
if self.train_batch == None :
self.model_train_path = ''.join([self.model_path+'/'+self.batch])
else :
self.model_train_path = ''.join([self.model_path + '/' + self.train_batch])
#Todo Eval flag 보도록 고치고
# "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch}
config = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver,
"nn_batch_ver_id": self.batch}
acc_result = TrainSummaryAccLossInfo(config)
if self.load_batch != self.batch:
src = ''.join([self.model_path+'/'+self.load_batch])
dst = self.model_train_path
utils.copy_all(src, dst)
#Optimizer and learning rate
#self._optimizer_type = self.optimizer_type
#self._learning_rates = self.learning_rate
logging.info("model_path : {0} ".format(self.model_path))
logging.info("hidden_layers : {0} ".format(self.hidden_layers))
logging.info("activation_function : {0} ".format(self.activation_function))
logging.info("batch_size : {0} ".format(self.batch_size))
logging.info("epoch : {0} ".format(self.epoch))
logging.info("model_type : {0} ".format(self.model_type))
logging.info("optimizer_type : {0} ".format(self.optimizer_type))
logging.info("learning_rates : {0} ".format(self.learning_rates))
data_conf_info = self.data_conf
# make wide & deep model
wdnn = NeuralCommonWdnn()
wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'],self.hidden_layers,str(self.activation_function),data_conf_info, str(self.model_train_path),self.train, self.auto_demension)
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
#file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요
_batch_size = self.batch_size
_num_tfrecords_files = 0
#multi Feeder modified
multi_read_flag = self.multi_read_flag
#validation_monitor = _LossCheckerHook(acc_result)
train_cnt = 5
#customsMonitor = EarlyStoppingHook()
if multi_read_flag == True:
logging.info("Reading tfrecord")
for index, fn in enumerate(train_data_set.input_paths):
_num_tfrecords_files += self.generator_len(
tf.python_io.tf_record_iterator(fn)) # get length of generators
logging.info("total loop " + str(math.ceil(_num_tfrecords_files/_batch_size)) )
for index in range(int(math.ceil(_num_tfrecords_files/_batch_size))):
for i in range(train_cnt):
logging.info("number of for loop " + str(index))
train_result = wdnn_model.fit(input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,_batch_size), steps=self.epoch)
#train_result = wdnn_model.fit(
# input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# _batch_size), steps=self.epoch)
eval_result = wdnn_model.evaluate(
input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size), steps=200)
acc = eval_result['accuracy']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(eval_result['loss']))
acc_result.acc_info["acc"].append(str(eval_result['accuracy']))
else:
#Todo H5
# train per files in folder h5용
logging.info("Training Wide and Deep from Reading hdf5")
while(train_data_set.has_next()) :
for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거
logging.info("Training WDNN Total Count {0} out of {1}".format(i+self.batch_size, train_data_set.data_size()))
data_set = train_data_set[i:i + self.batch_size]
for t_i in range(train_cnt):
logging.info(
"Training WDNN Train Count {0} out of {1}".format(t_i, train_cnt))
#data_set = train_data_set[i:i + self.batch_size]
if i == 0:
eval_data_Set = data_set
train_result = wdnn_model.fit(
input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
data_set,data_conf_info), steps=self.epoch)
eval_result = wdnn_model.evaluate(
input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
data_set, data_conf_info), steps=200)
logging.info("wdnn training complete count from h5 : {0} ".format(len(data_set)))
acc = eval_result['accuracy']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(eval_result['loss']))
acc_result.acc_info["acc"].append(str(eval_result['accuracy']))
#train_result = wdnn_model.fit(
# input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# data_set, data_conf_info), steps=self.epoch)
logging.info("Traing Result -> {0}".format(train_result))
train_data_set.next()
print("end")
self.save_accloss_info(acc_result)
return None
except Exception as e:
logging.info("[Wide and Deep Train Process] : {0}".format(e))
raise Exception(e)
def run(self, conf_data):
"""
ML Training
:param : conf_data
:return: None
"""
logging.info("ML Run called")
try:
self._init_node_parm(conf_data['node_id'])
graph = NNCommonManager().get_nn_node_name(conf_data['nn_id'])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
netconf_node = net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'eval_feed':
eval_feed_name = conf_data['nn_id'] + "_" + conf_data['wf_ver'] + "_" + net['fields']['graph_node_name']
# Model Path
self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node)
#Set Data Feeder
self.cls_pool = conf_data['cls_pool'] # Data feeder
# set batch
self.load_batch = self.get_eval_batch(conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기
self.train_batch, self.batch = self.make_batch(conf_data['node_id'])
logging.info("ML Train get batch -> {0}".format(self.batch))
logging.info("ML Train get batch -> {0}".format(self.load_batch))
if self.train_batch == None :
self.model_train_path = ''.join([self.model_path+'/'+self.batch])
else :
self.model_train_path = ''.join([self.model_path + '/' + self.train_batch])
config = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver,
"nn_batch_ver_id": self.batch}
acc_result = TrainSummaryAccLossInfo(config)
if self.load_batch != self.batch:
src = ''.join([self.model_path+'/'+self.load_batch])
dst = self.model_train_path
utils.copy_all(src, dst)
logging.info("model_path : {0} ".format(self.model_path))
logging.info("ml_class : {0} ".format(self.ml_class))
logging.info("config : {0} ".format(self.config))
data_conf_info = self.data_conf
# make ml_class
if self.ml_class == 'DecisionTreeClassifier':
max_depth = self.config["max_depth"]
clf = tree.DecisionTreeClassifier(max_depth=max_depth)
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess')
#train_node_name = self._get_forward_node_with_type(conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
test_data_set = self.cls_pool[eval_feed_name]
#file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요
_batch_size = self.batch_size
_num_tfrecords_files = 0
#multi Feeder modified
multi_read_flag = False
train_cnt = 1
if multi_read_flag == True:
logging.info("Reading tfrecord")
for index, fn in enumerate(train_data_set.input_paths):
_num_tfrecords_files += self.generator_len(
tf.python_io.tf_record_iterator(fn)) # get length of generators
logging.info("total loop " + str(math.ceil(_num_tfrecords_files/_batch_size)) )
for index in range(int(math.ceil(_num_tfrecords_files/_batch_size))):
for i in range(train_cnt):
logging.info("number of for loop " + str(index))
train_result = clf.fit(input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,_batch_size), steps=self.epoch)
eval_result = clf.evaluate(
input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size), steps=200)
acc = eval_result['accuracy']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(eval_result['loss']))
acc_result.acc_info["acc"].append(str(eval_result['accuracy']))
else:
#Todo H5
# train per files in folder h5용
logging.info("Training ML from Reading hdf5")
while(train_data_set.has_next()) :
for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거
logging.info("Training ML Total Count {0} out of {1}".format(i+self.batch_size, train_data_set.data_size()))
data_set = train_data_set[i:i + self.batch_size]
data_set_test = test_data_set[i:i +self.batch_size]
for t_i in range(train_cnt):
logging.info(
"Training ML Train Count {0} out of {1}".format(t_i, train_cnt))
if i == 0:
eval_data_Set = data_set
iris = load_iris()
keys = list(data_conf_info['cell_feature'].keys())
keys.remove(data_conf_info['label'])
feature_names = keys
keys = np.asarray(keys)
data = data_set[keys].values
test_data = data_set_test[keys].values
label = data_set[data_conf_info['label']].values
test_label = data_set_test[data_conf_info['label']].values
clf = clf.fit(data,label)
target_names = np.asarray(self.label_values)
dot_data = tree.export_graphviz(clf,out_file=None,
feature_names=feature_names,
class_names=target_names,
filled=True, rounded=True,
special_characters=True)
graph1 = graphviz.Source(dot_data)
graph1.render(self.model_path+'/model')
# eval_result = clf.evaluate(
# input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# data_set, data_conf_info), steps=200)
logging.info("ML training complete count from h5 : {0} ".format(len(data_set)))
acc = cross_val_score(clf,test_data,test_label,scoring='accuracy').mean()
loss = cross_val_score(clf,data,label,scoring='neg_log_loss').mean()
# acc = eval_result['accuracy']
# loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(loss))
acc_result.acc_info["acc"].append(str(acc))
#logging.info("Traing Result -> {0}".format(train_result))
train_data_set.next()
print("end")
self.save_accloss_info(acc_result)
joblib.dump(clf,self.model_path+'/model.pkl')
return None
except Exception as e:
logging.info("[Wide and Deep Train Process] : {0}".format(e))
raise Exception(e)
def run(self, conf_data):
"""
ML Training
:param : conf_data
:return: None
"""
logging.info("ML Run called")
try:
self._init_node_parm(conf_data['node_id'])
graph = NNCommonManager().get_nn_node_name(conf_data['nn_id'])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
netconf_node = net['fields']['graph_node_name']
if net['fields']['graph_node'] == 'eval_feed':
eval_feed_name = conf_data['nn_id'] + "_" + conf_data[
'wf_ver'] + "_" + net['fields']['graph_node_name']
# Model Path
self.model_path = utils.get_model_path(conf_data['nn_id'],
conf_data['wf_ver'],
netconf_node)
#Set Data Feeder
self.cls_pool = conf_data['cls_pool'] # Data feeder
# set batch
self.load_batch = self.get_eval_batch(
conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기
self.train_batch, self.batch = self.make_batch(
conf_data['node_id'])
logging.info("ML Train get batch -> {0}".format(self.batch))
logging.info("ML Train get batch -> {0}".format(self.load_batch))
if self.train_batch == None:
self.model_train_path = ''.join(
[self.model_path + '/' + self.batch])
else:
self.model_train_path = ''.join(
[self.model_path + '/' + self.train_batch])
config = {
"nn_id": conf_data['node_id'],
"nn_wf_ver_id": self.net_ver,
"nn_batch_ver_id": self.batch
}
acc_result = TrainSummaryAccLossInfo(config)
if self.load_batch != self.batch:
src = ''.join([self.model_path + '/' + self.load_batch])
dst = self.model_train_path
utils.copy_all(src, dst)
logging.info("model_path : {0} ".format(self.model_path))
logging.info("ml_class : {0} ".format(self.ml_class))
logging.info("config : {0} ".format(self.config))
data_conf_info = self.data_conf
# make ml_class
if self.ml_class == 'DecisionTreeClassifier':
max_depth = self.config["max_depth"]
clf = tree.DecisionTreeClassifier(max_depth=max_depth)
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(
conf_data['node_id'], 'preprocess')
#train_node_name = self._get_forward_node_with_type(conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
test_data_set = self.cls_pool[eval_feed_name]
#file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요
_batch_size = self.batch_size
_num_tfrecords_files = 0
#multi Feeder modified
multi_read_flag = False
train_cnt = 1
if multi_read_flag == True:
logging.info("Reading tfrecord")
for index, fn in enumerate(train_data_set.input_paths):
_num_tfrecords_files += self.generator_len(
tf.python_io.tf_record_iterator(
fn)) # get length of generators
logging.info("total loop " +
str(math.ceil(_num_tfrecords_files /
_batch_size)))
for index in range(
int(math.ceil(_num_tfrecords_files / _batch_size))):
for i in range(train_cnt):
logging.info("number of for loop " + str(index))
train_result = clf.fit(
input_fn=lambda: train_data_set.input_fn(
tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size),
steps=self.epoch)
eval_result = clf.evaluate(
input_fn=lambda: train_data_set.input_fn(
tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size),
steps=200)
acc = eval_result['accuracy']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(
str(eval_result['loss']))
acc_result.acc_info["acc"].append(
str(eval_result['accuracy']))
else:
#Todo H5
# train per files in folder h5용
logging.info("Training ML from Reading hdf5")
while (train_data_set.has_next()):
for i in range(0, train_data_set.data_size(),
self.batch_size): #크게 한번 도는거
logging.info(
"Training ML Total Count {0} out of {1}".format(
i + self.batch_size,
train_data_set.data_size()))
data_set = train_data_set[i:i + self.batch_size]
data_set_test = test_data_set[i:i + self.batch_size]
for t_i in range(train_cnt):
logging.info(
"Training ML Train Count {0} out of {1}".
format(t_i, train_cnt))
if i == 0:
eval_data_Set = data_set
iris = load_iris()
keys = list(data_conf_info['cell_feature'].keys())
keys.remove(data_conf_info['label'])
feature_names = keys
keys = np.asarray(keys)
data = data_set[keys].values
test_data = data_set_test[keys].values
label = data_set[data_conf_info['label']].values
test_label = data_set_test[
data_conf_info['label']].values
clf = clf.fit(data, label)
target_names = np.asarray(self.label_values)
dot_data = tree.export_graphviz(
clf,
out_file=None,
feature_names=feature_names,
class_names=target_names,
filled=True,
rounded=True,
special_characters=True)
graph1 = graphviz.Source(dot_data)
graph1.render(self.model_path + '/model')
# eval_result = clf.evaluate(
# input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# data_set, data_conf_info), steps=200)
logging.info(
"ML training complete count from h5 : {0} ".
format(len(data_set)))
acc = cross_val_score(clf,
test_data,
test_label,
scoring='accuracy').mean()
loss = cross_val_score(
clf, data, label,
scoring='neg_log_loss').mean()
# acc = eval_result['accuracy']
# loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(loss))
acc_result.acc_info["acc"].append(str(acc))
#logging.info("Traing Result -> {0}".format(train_result))
train_data_set.next()
print("end")
self.save_accloss_info(acc_result)
joblib.dump(clf, self.model_path + '/model.pkl')
return None
except Exception as e:
logging.info("[Wide and Deep Train Process] : {0}".format(e))
raise Exception(e)
def run(self, conf_data):
logging.info("NeuralNetNodeWdnn Run called")
#return None
#return None
"""
Wide & Deep Network Training
:param nnid : network id in tfmsacore_nninfo
:return: acturacy
"""
try:
self._init_node_parm(conf_data['node_id'])
self.cls_pool = conf_data['cls_pool'] # Data feeder
self.train_batch, self.batch = self.make_batch(
conf_data['node_id']) #makebatch
self.before_train_batch = self.get_before_make_batch(
conf_data['node_id'], self.batch) #before train batch
if self.before_train_batch != None:
self.model_train_before_path = ''.join([
self.model_path + '/' +
str(self.before_train_batch.nn_batch_ver_id)
])
if self.train_batch == None:
self.model_train_path = ''.join(
[self.model_path + '/' + self.batch])
else:
self.model_train_path = ''.join(
[self.model_path + '/' + self.train_batch])
#model file copy
if self.before_train_batch != None:
src = self.model_train_before_path
dst = self.model_train_path
utils.copy_all(src, dst)
logging.info("model_path : {0} ".format(self.model_path))
logging.info("hidden_layers : {0} ".format(self.hidden_layers))
logging.info("activation_function : {0} ".format(
self.activation_function))
logging.info("batch_size : {0} ".format(self.batch_size))
logging.info("epoch : {0} ".format(self.epoch))
logging.info("model_type : {0} ".format(self.model_type))
data_conf_info = self.data_conf
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(
conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
#TODO 아 eval이 안되서 데이터가 안불러
_batch_size = self.batch_size
_num_tfrecords_files = 0
#_batch_size = 2
data_set = train_data_set[0:train_data_set.data_size()]
input_dims = len(data_set.columns) - 1 # label 갯수 제거
# model = Sequential()
# model.add(Dense(100, input_dim=input_dims, activation='relu'))
# #model.add(Dense(50))
# model.add(Dense(50, activation='sigmoid'))
# #model.add(Dense(50, activation='sigmoid'))
# model.compile(optimizer='rmsprop',
# loss='binary_crossentropy',
# metrics=['accuracy'])
model = Sequential()
model.add(Dense(10, input_dim=input_dims, init='uniform'))
model.add(BatchNormalization())
model.add(Activation('sigmoid'))
#model.add(Dense(10, init='uniform'))
#model.add(BatchNormalization())
#model.add(Activation('sigmoid'))
model.add(Dense(1, init='uniform'))
model.add(BatchNormalization())
model.add(Activation('sigmoid'))
lr_reducer = ReduceLROnPlateau(monitor='val_loss',
factor=np.sqrt(0.1),
cooldown=0,
patience=5,
min_lr=0.5e-6)
early_stopper = EarlyStopping(monitor='val_acc',
min_delta=0.001,
patience=10)
csv_logger = CSVLogger('resnet18_cifar10.csv')
history = History()
# Compile model
model.compile(loss='binary_crossentropy', optimizer='sgd')
self.batch_size = 4
#multi Feeder modified
multi_read_flag = self.multi_read_flag
if multi_read_flag == False:
#Todo H5
# train per files in folder h5용
while (train_data_set.has_next()):
logging.info("start keras dnn")
#파일이 하나 돌때마다
#for 배치사이즈와 파일의 총갯수를 가져다가 돌린다. -> 마지막에 뭐가 있을지 구분한다.
#파일에 iter를 넣으면 배치만큼 가져오는 fn이 있음 그걸 __itemd에 넣고
# Input 펑션에서 multi를 vk판단해서 col와 ca를 구분한다.(이걸 배치마다 할 필요가 있나?)
# -> 그러면서 피팅
#for x in range(0, self.iter_size):
for i in range(0, train_data_set.data_size(),
self.batch_size):
#Last batch size preprocessing
if (i + self.batch_size > train_data_set.data_size()):
i = i - (train_data_set.data_size() %
self.batch_size) + 1
data_set = train_data_set[i:i + self.batch_size]
#data_set = train_data_set[0:train_data_set.data_size()]
logging.info(i)
X_train, targets, = train_data_set.input_fn3(
file_queue, data_set, data_conf_info)
#loss, accuracy =
model.fit(X_train,
targets,
epochs=10,
validation_data=(X_train, targets),
batch_size=self.batch_size,
callbacks=[lr_reducer, csv_logger, history])
#logging.info("keras training info loss : {0} , accuracy{1}".format(loss, accuracy))
# #Select Next file
train_data_set.next()
#os.makedirs(self.md_store_path + '/' + self.batch, exist_ok=True)
# keras.models.save_model(model, ''.join([self.md_store_path + '/' + self.batch, '/model.bin']))
print("end")
except Exception as e:
logging.error("Error Message : {0}".format(e))
raise Exception(e)
return None
def run(self, conf_data):
"""
Wide and Deep Network Training
:param : conf_data
:return: None
"""
logging.info("NeuralNetNodeWdnn Run called")
try:
self._init_node_parm(conf_data['node_id'])
if self.train == False: #Train Value 가 False면 훈련 안함
return None
graph = NNCommonManager().get_nn_node_name(conf_data['nn_id'])
for net in graph:
if net['fields']['graph_node'] == 'netconf_node':
netconf_node = net['fields']['graph_node_name']
self.model_path = utils.get_model_path(conf_data['nn_id'], conf_data['wf_ver'], netconf_node)
#Model Path
#self.model_path = utils.get_model_path(nnid, ver, netconf_node)
#Set Data Feeder
self.cls_pool = conf_data['cls_pool'] # Data feeder
# set batch
self.load_batch = self.get_eval_batch(conf_data['node_id']) #Train이 Y인것 가져오기 Eval Flag가 Y인거 가져오기
self.train_batch, self.batch = self.make_batch(conf_data['node_id'])
#self.before_train_batch = self.get_before_make_batch(conf_data['node_id'], self.batch) # before train batch
#if self.before_train_batch != None:
# self.model_train_before_path = ''.join([self.model_path+'/'+str(self.before_train_batch.nn_batch_ver_id)])
logging.info("Wdnn Train get batch -> {0}".format(self.batch))
logging.info("Wdnn Train get batch -> {0}".format(self.load_batch))
if self.train_batch == None :
self.model_train_path = ''.join([self.model_path+'/'+self.batch])
else :
self.model_train_path = ''.join([self.model_path + '/' + self.train_batch])
#Todo Eval flag 보도록 고치고
# "nn_wf_ver_id": self.wf_ver, "nn_batch_ver_id": self.batch}
config = {"nn_id": conf_data['node_id'], "nn_wf_ver_id": self.net_ver,
"nn_batch_ver_id": self.batch}
acc_result = TrainSummaryAccLossInfo(config)
if self.load_batch != self.batch:
src = ''.join([self.model_path+'/'+self.load_batch])
dst = self.model_train_path
utils.copy_all(src, dst)
#Optimizer and learning rate
#self._optimizer_type = self.optimizer_type
#self._learning_rates = self.learning_rate
logging.info("model_path : {0} ".format(self.model_path))
logging.info("hidden_layers : {0} ".format(self.hidden_layers))
logging.info("activation_function : {0} ".format(self.activation_function))
logging.info("batch_size : {0} ".format(self.batch_size))
logging.info("epoch : {0} ".format(self.epoch))
logging.info("model_type : {0} ".format(self.model_type))
logging.info("optimizer_type : {0} ".format(self.optimizer_type))
logging.info("learning_rates : {0} ".format(self.learning_rates))
data_conf_info = self.data_conf
# make wide & deep model
wdnn = NeuralCommonWdnn()
wdnn_model = wdnn.wdnn_build(self.model_type, conf_data['node_id'],self.hidden_layers,str(self.activation_function),data_conf_info, str(self.model_train_path),self.train, self.auto_demension)
#feed
# TODO file이 여러개면 어떻하지?
# get prev node for load data
data_node_name = self._get_backward_node_with_type(conf_data['node_id'], 'preprocess')
train_data_set = self.cls_pool[data_node_name[0]] #get filename
file_queue = str(train_data_set.input_paths[0]) #get file_name
#file을 돌면서 최대 Row를 전부 들고 옴 tfrecord 총 record갯수 가져오는 방법필요
_batch_size = self.batch_size
_num_tfrecords_files = 0
#multi Feeder modified
multi_read_flag = self.multi_read_flag
#validation_monitor = _LossCheckerHook(acc_result)
train_cnt = 5
#customsMonitor = EarlyStoppingHook()
if multi_read_flag == True:
logging.info("Reading tfrecord")
for index, fn in enumerate(train_data_set.input_paths):
_num_tfrecords_files += self.generator_len(
tf.python_io.tf_record_iterator(fn)) # get length of generators
logging.info("total loop " + str(math.ceil(_num_tfrecords_files/_batch_size)) )
for index in range(int(math.ceil(_num_tfrecords_files/_batch_size))):
for i in range(train_cnt):
logging.info("number of for loop " + str(index))
train_result = wdnn_model.fit(input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,_batch_size), steps=self.epoch)
#train_result = wdnn_model.fit(
# input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# _batch_size), steps=self.epoch)
eval_result = wdnn_model.evaluate(
input_fn=lambda: train_data_set.input_fn(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
_batch_size), steps=200)
acc = eval_result['accuracy']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(eval_result['loss']))
acc_result.acc_info["acc"].append(str(eval_result['accuracy']))
else:
#Todo H5
# train per files in folder h5용
logging.info("Training Wide and Deep from Reading hdf5")
while(train_data_set.has_next()) :
for i in range(0, train_data_set.data_size(), self.batch_size): #크게 한번 도는거
logging.info("Training WDNN Total Count {0} out of {1}".format(i+self.batch_size, train_data_set.data_size()))
data_set = train_data_set[i:i + self.batch_size]
for t_i in range(train_cnt):
logging.info(
"Training WDNN Train Count {0} out of {1}".format(t_i, train_cnt))
#data_set = train_data_set[i:i + self.batch_size]
if i == 0:
eval_data_Set = data_set
train_result = wdnn_model.fit(
input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
data_set,data_conf_info), steps=self.epoch)
eval_result = wdnn_model.evaluate(
input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
data_set, data_conf_info), steps=200)
logging.info("wdnn training complete count from h5 : {0} ".format(len(data_set)))
if self.model_type == 'regression':
acc = eval_result['loss']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(eval_result['loss']))
acc_result.acc_info["acc"].append(str(eval_result['loss']))
else:
acc = eval_result['accuracy']
loss = eval_result['loss']
acc_result.loss_info["loss"].append(str(eval_result['loss']))
acc_result.acc_info["acc"].append(str(eval_result['accuracy']))
#acc_result.loss_info["loss"].append(str(eval_result['loss']))
#acc_result.acc_info["acc"].append(str(eval_result['accuracy']))
#train_result = wdnn_model.fit(
# input_fn=lambda: train_data_set.input_fn2(tf.contrib.learn.ModeKeys.TRAIN, file_queue,
# data_set, data_conf_info), steps=self.epoch)
logging.info("Traing Result -> {0}".format(train_result))
train_data_set.next()
print("end")
self.save_accloss_info(acc_result)
return None
except Exception as e:
logging.info("[Wide and Deep Train Process] : {0}".format(e))
raise Exception(e)