def summary(self, prefix='', include_total_time=False) -> str: import operator li = [(name, self.__watches[name].elapsed()) for name in self.__watches] li = sorted(li, key=operator.itemgetter(1), reverse=True) s = '' # for name, total_milli_secs in li: # s += 'total [%s] %s\n' % (DateUtil.millisecs_to_string(total_milli_secs), name) if include_total_time: for name, total_milli_secs in li: if self.__cnt[name] > 0: if len(prefix) > 0: s += '%s total [%s] %s\n' % ( prefix, DateUtil.millisecs_to_string(float(total_milli_secs)), name) else: s += 'total [%s] %s\n' % ( DateUtil.millisecs_to_string(float(total_milli_secs)), name) for name, total_milli_secs in li: if self.__cnt[name] > 0: if len(prefix) > 0: s += '%s average [%s] %s\n' % ( prefix, DateUtil.millisecs_to_string(float(total_milli_secs) / float(self.__cnt[name])), name) else: s += 'average [%s] %s\n' % ( DateUtil.millisecs_to_string(float(total_milli_secs) / float(self.__cnt[name])), name) return s
if not os.path.exists(test_file): create_data4add(test_file, n_test, digit_max=99) for training_mode in [True, False]: # training & testing for batch_size in [1, 10, 100]: tf.reset_default_graph() # Clears the default graph stack and resets the global default graph. log.info('') log.info('training_mode: %s, batch_size: %s, total_train_time: %s secs' % (training_mode, batch_size, total_train_time)) model_name = os.path.basename(__file__).replace('.py', '') model_file = os.path.join(SAMPLE_MODELS_DIR, '%s.n_train_%s.batch_size_%s.total_train_time_%s/model' % (model_name, n_train, batch_size, total_train_time)) model_dir = os.path.dirname(model_file) log.info('model_name: %s' % model_name) log.info('model_file: %s' % model_file) scope_name = '%s.%s.batch_size_%s.total_train_time_%s' % (model_name, DateUtil.current_yyyymmdd_hhmm(), batch_size, total_train_time) log.info('scope_name: %s' % scope_name) with tf.device('/gpu:0'): with tf.Graph().as_default(): # for reusing graph checkpoint = tf.train.get_checkpoint_state(model_dir) is_training = True if training_mode or not checkpoint else False # learning or testing config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) with tf.Session(config=config) as sess: train_pipeline = input_pipeline([train_file], batch_size=batch_size, shuffle=True, delim='\t', splits=3) valid_pipeline = input_pipeline([valid_file], batch_size=n_valid, shuffle=True, delim='\t', splits=3) test_pipeline = input_pipeline([test_file], batch_size=n_test, shuffle=True, delim='\t', splits=3) if is_training: # training x, y, learning_rate, W1, b1, y_hat, cost, train_step, summary = create_graph(model_name, scope_name, verbose=False)
def elapsed_string(self, name=DEFAULT_WATCH_NAME) -> str: return DateUtil.secs_to_string(self.elapsed(name))
log.info('') log.info( 'training_mode: %s, batch_size: %s, total_train_time: %s secs' % (training_mode, batch_size, total_train_time)) model_name = os.path.basename(__file__).replace('.py', '') model_file = os.path.join( MODELS_DIR, '%s.n_train_%s.batch_size_%s.total_train_time_%s/model' % (model_name, n_train, batch_size, total_train_time)) model_dir = os.path.dirname(model_file) log.info('model_name: %s' % model_name) log.info('model_file: %s' % model_file) scope_name = '%s.%s.batch_size_%s.total_train_time_%s' % ( model_name, DateUtil.current_yyyymmdd_hhmm(), batch_size, total_train_time) log.info('scope_name: %s' % scope_name) with tf.device('/gpu:0'): with tf.Graph().as_default(): # for reusing graph checkpoint = tf.train.get_checkpoint_state(model_dir) is_training = True if training_mode or not checkpoint else False # learning or testing config = tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True, visible_device_list='0')) with tf.Session(config=config) as sess: train_pipeline = input_pipeline([train_file], batch_size=batch_size, shuffle=True, delim='\t',
train_time = 10 * 60 # 6. 끝까지 학습 (cost: 4-5) (6분) # # # 7. 좀더 줄이려면 어떻게 해야 할까요? (decay) log.info('%s -> %s -> %s -> %s -> %s' % (x_train.shape[1], n_hiddens, activation.__name__, n_hiddens, 1)) log.info('weights_initializer: %s' % weights_initializer.__name__) log.info('learning_rate: %.4f' % learning_rate) log.info('train_time: %s' % train_time) how_many_trains = 3 if train_time < 10 else 1 log.info('how_many_trains: %s' % how_many_trains) for _ in range(how_many_trains): time.sleep(1) tf.reset_default_graph() # Clears the default graph stack and resets the global default graph. tf.set_random_seed(7942) # 3. 결과를 규칙적으로 만들자. (cost: 600-700) scope_name = '%s.%s' % (func.__name__, DateUtil.current_yyyymmdd_hhmmss()) x, y, y_hat, cost, rsme, train_step, summary = build_graph(scope_name, n_features, n_hiddens, n_classes, learning_rate, activation=activation, weights_initializer=weights_initializer, bias_value=bias_value) try: watch = WatchUtil() model_file_saved = False model_file = os.path.join(MODELS_DIR, '%s_%s/model' % (os.path.basename(__file__.replace('.py', '')), func.__name__)) model_dir = os.path.dirname(model_file) # log.info('model_file: %s' % model_file) if not os.path.exists(model_dir): # log.info('model_dir: %s' % model_dir) os.makedirs(model_dir) config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) saver = tf.train.Saver()
li.append((start_date, end_date)) start_date = end_date + datetime.timedelta(days=1) else: end_date = self.to_date while end_date >= self.from_date: start_date = end_date - datetime.timedelta(days=max_days) if start_date < self.from_date: start_date = self.from_date li.append((start_date, end_date)) end_date = start_date - datetime.timedelta(days=1) return li if __name__ == '__main__': # start_date, end_date = DateUtil.string_to_datetime('2015-11-29 00:00:00'), DateUtil.string_to_datetime('2016-01-01 00:00:00') start_date_index, end_date = DateUtil.string_to_date( '2015-11-01'), DateUtil.string_to_date('2016-01-05') # between = DateBetweenUtil(start_date_index, end_date) # for a, b in between.date_split(from_start=True): # print(a, b) # print() # for a, b in between.date_split(from_start=False): # print(a, b) # between = DateBetweenUtil(datetime.datetime(2013, 1, 31, 1), datetime.datetime(2013, 2, 2, 4)) # print(between.days()) # for d in between.date_list(): # print(str(d), type(d)) # print(between.hour_list()) # check exception raised. # assert Exception, DateBetweenUtil(10, datetime.datetime(2013, 2, 2, 4))
learning_rate = 0.001 # Adeam은 3승 이하 추천 / 590 -> 670으로 증가 | best/total 체크가 필요 : 431/431인 경우로 러닝 시간이 부족한 상황 train_time = 10 * 60 # best 10분 / cpu에서는 10배 정도 늦음 ### 최적화 테스트 : 끝 print('%s -> %s -> %s -> %s -> %s' % (x_train.shape[1], n_hiddens, activation.__name__, n_hiddens, 1)) print('weights_initializer: %s' % weights_initializer.__name__) print('learning_rate: %.4f' % learning_rate) print('train_time: %s' % train_time) how_many_trains = 3 if train_time <= 1 else 1 # 1초 실행하는 경우, 3번 실험 그 외에는 1번 실험. for _ in range(how_many_trains) : # time.sleep(1) tf.reset_default_graph() # 기존 session을 초기화 tf.set_random_seed(7942) # tf.random_normal_initializer 사용하기 때문에 설정 필요 scope_name = '%s.%s' % (func.__name__,DateUtil.current_yyyymmdd_hhmmss()) # graph 겹치지 않게 하기 위해서, func + 날짜 이름으로 설정하는 것을 추천 x, y, y_hat, cost, rsme, train_step, summary = build_graph(scope_name, n_features, n_hiddens, n_classes, learning_rate, activation=activation, weights_initializer=weights_initializer, bias_value=bias_value) try : watch = WatchUtil() model_file_saved = False model_file = os.path.join('%s/workspace/nlp4kor/models/%s_%s/model' % (os.getcwd(), os.path.basename(__name__.replace('.py', '')), func.__name__)) model_dir = os.path.dirname(model_file) # print('model_file: %s' % model_file) if not os.path.exists(model_dir): # print('model_dir: %s' % model_dir) os.makedirs(model_dir) config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) saver = tf.train.Saver() # 최근 5개만 남개 되어서 max_to_keep=None 해야함