def summary(self, prefix='', include_total_time=False) -> str:
import operator
li = [(name, self.__watches[name].elapsed()) for name in self.__watches]
li = sorted(li, key=operator.itemgetter(1), reverse=True)
s = ''
# for name, total_milli_secs in li:
# s += 'total [%s] %s\n' % (DateUtil.millisecs_to_string(total_milli_secs), name)
if include_total_time:
for name, total_milli_secs in li:
if self.__cnt[name] > 0:
if len(prefix) > 0:
s += '%s total [%s] %s\n' % (
prefix, DateUtil.millisecs_to_string(float(total_milli_secs)), name)
else:
s += 'total [%s] %s\n' % (
DateUtil.millisecs_to_string(float(total_milli_secs)), name)
for name, total_milli_secs in li:
if self.__cnt[name] > 0:
if len(prefix) > 0:
s += '%s average [%s] %s\n' % (
prefix, DateUtil.millisecs_to_string(float(total_milli_secs) / float(self.__cnt[name])), name)
else:
s += 'average [%s] %s\n' % (
DateUtil.millisecs_to_string(float(total_milli_secs) / float(self.__cnt[name])), name)
return s
if not os.path.exists(test_file):
create_data4add(test_file, n_test, digit_max=99)
for training_mode in [True, False]: # training & testing
for batch_size in [1, 10, 100]:
tf.reset_default_graph() # Clears the default graph stack and resets the global default graph.
log.info('')
log.info('training_mode: %s, batch_size: %s, total_train_time: %s secs' % (training_mode, batch_size, total_train_time))
model_name = os.path.basename(__file__).replace('.py', '')
model_file = os.path.join(SAMPLE_MODELS_DIR, '%s.n_train_%s.batch_size_%s.total_train_time_%s/model' % (model_name, n_train, batch_size, total_train_time))
model_dir = os.path.dirname(model_file)
log.info('model_name: %s' % model_name)
log.info('model_file: %s' % model_file)
scope_name = '%s.%s.batch_size_%s.total_train_time_%s' % (model_name, DateUtil.current_yyyymmdd_hhmm(), batch_size, total_train_time)
log.info('scope_name: %s' % scope_name)
with tf.device('/gpu:0'):
with tf.Graph().as_default(): # for reusing graph
checkpoint = tf.train.get_checkpoint_state(model_dir)
is_training = True if training_mode or not checkpoint else False # learning or testing
config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
with tf.Session(config=config) as sess:
train_pipeline = input_pipeline([train_file], batch_size=batch_size, shuffle=True, delim='\t', splits=3)
valid_pipeline = input_pipeline([valid_file], batch_size=n_valid, shuffle=True, delim='\t', splits=3)
test_pipeline = input_pipeline([test_file], batch_size=n_test, shuffle=True, delim='\t', splits=3)
if is_training: # training
x, y, learning_rate, W1, b1, y_hat, cost, train_step, summary = create_graph(model_name, scope_name, verbose=False)
def elapsed_string(self, name=DEFAULT_WATCH_NAME) -> str:
return DateUtil.secs_to_string(self.elapsed(name))
log.info('')
log.info(
'training_mode: %s, batch_size: %s, total_train_time: %s secs'
% (training_mode, batch_size, total_train_time))
model_name = os.path.basename(__file__).replace('.py', '')
model_file = os.path.join(
MODELS_DIR,
'%s.n_train_%s.batch_size_%s.total_train_time_%s/model' %
(model_name, n_train, batch_size, total_train_time))
model_dir = os.path.dirname(model_file)
log.info('model_name: %s' % model_name)
log.info('model_file: %s' % model_file)
scope_name = '%s.%s.batch_size_%s.total_train_time_%s' % (
model_name, DateUtil.current_yyyymmdd_hhmm(), batch_size,
total_train_time)
log.info('scope_name: %s' % scope_name)
with tf.device('/gpu:0'):
with tf.Graph().as_default(): # for reusing graph
checkpoint = tf.train.get_checkpoint_state(model_dir)
is_training = True if training_mode or not checkpoint else False # learning or testing
config = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True, visible_device_list='0'))
with tf.Session(config=config) as sess:
train_pipeline = input_pipeline([train_file],
batch_size=batch_size,
shuffle=True,
delim='\t',
train_time = 10 * 60 # 6. 끝까지 학습 (cost: 4-5) (6분)
# # # 7. 좀더 줄이려면 어떻게 해야 할까요? (decay)
log.info('%s -> %s -> %s -> %s -> %s' % (x_train.shape[1], n_hiddens, activation.__name__, n_hiddens, 1))
log.info('weights_initializer: %s' % weights_initializer.__name__)
log.info('learning_rate: %.4f' % learning_rate)
log.info('train_time: %s' % train_time)
how_many_trains = 3 if train_time < 10 else 1
log.info('how_many_trains: %s' % how_many_trains)
for _ in range(how_many_trains):
time.sleep(1)
tf.reset_default_graph() # Clears the default graph stack and resets the global default graph.
tf.set_random_seed(7942) # 3. 결과를 규칙적으로 만들자. (cost: 600-700)
scope_name = '%s.%s' % (func.__name__, DateUtil.current_yyyymmdd_hhmmss())
x, y, y_hat, cost, rsme, train_step, summary = build_graph(scope_name, n_features, n_hiddens, n_classes, learning_rate, activation=activation, weights_initializer=weights_initializer,
bias_value=bias_value)
try:
watch = WatchUtil()
model_file_saved = False
model_file = os.path.join(MODELS_DIR, '%s_%s/model' % (os.path.basename(__file__.replace('.py', '')), func.__name__))
model_dir = os.path.dirname(model_file)
# log.info('model_file: %s' % model_file)
if not os.path.exists(model_dir):
# log.info('model_dir: %s' % model_dir)
os.makedirs(model_dir)
config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
saver = tf.train.Saver()
li.append((start_date, end_date))
start_date = end_date + datetime.timedelta(days=1)
else:
end_date = self.to_date
while end_date >= self.from_date:
start_date = end_date - datetime.timedelta(days=max_days)
if start_date < self.from_date:
start_date = self.from_date
li.append((start_date, end_date))
end_date = start_date - datetime.timedelta(days=1)
return li
if __name__ == '__main__':
# start_date, end_date = DateUtil.string_to_datetime('2015-11-29 00:00:00'), DateUtil.string_to_datetime('2016-01-01 00:00:00')
start_date_index, end_date = DateUtil.string_to_date(
'2015-11-01'), DateUtil.string_to_date('2016-01-05')
# between = DateBetweenUtil(start_date_index, end_date)
# for a, b in between.date_split(from_start=True):
# print(a, b)
# print()
# for a, b in between.date_split(from_start=False):
# print(a, b)
# between = DateBetweenUtil(datetime.datetime(2013, 1, 31, 1), datetime.datetime(2013, 2, 2, 4))
# print(between.days())
# for d in between.date_list():
# print(str(d), type(d))
# print(between.hour_list())
# check exception raised.
# assert Exception, DateBetweenUtil(10, datetime.datetime(2013, 2, 2, 4))
learning_rate = 0.001 # Adeam은 3승 이하 추천 / 590 -> 670으로 증가 | best/total 체크가 필요 : 431/431인 경우로 러닝 시간이 부족한 상황
train_time = 10 * 60 # best 10분 / cpu에서는 10배 정도 늦음
### 최적화 테스트 : 끝
print('%s -> %s -> %s -> %s -> %s' % (x_train.shape[1], n_hiddens, activation.__name__, n_hiddens, 1))
print('weights_initializer: %s' % weights_initializer.__name__)
print('learning_rate: %.4f' % learning_rate)
print('train_time: %s' % train_time)
how_many_trains = 3 if train_time <= 1 else 1 # 1초 실행하는 경우, 3번 실험 그 외에는 1번 실험.
for _ in range(how_many_trains) :
# time.sleep(1)
tf.reset_default_graph() # 기존 session을 초기화
tf.set_random_seed(7942) # tf.random_normal_initializer 사용하기 때문에 설정 필요
scope_name = '%s.%s' % (func.__name__,DateUtil.current_yyyymmdd_hhmmss()) # graph 겹치지 않게 하기 위해서, func + 날짜 이름으로 설정하는 것을 추천
x, y, y_hat, cost, rsme, train_step, summary = build_graph(scope_name, n_features, n_hiddens, n_classes, learning_rate, activation=activation, weights_initializer=weights_initializer, bias_value=bias_value)
try :
watch = WatchUtil()
model_file_saved = False
model_file = os.path.join('%s/workspace/nlp4kor/models/%s_%s/model' % (os.getcwd(), os.path.basename(__name__.replace('.py', '')), func.__name__))
model_dir = os.path.dirname(model_file)
# print('model_file: %s' % model_file)
if not os.path.exists(model_dir):
# print('model_dir: %s' % model_dir)
os.makedirs(model_dir)
config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
saver = tf.train.Saver() # 최근 5개만 남개 되어서 max_to_keep=None 해야함
