def train_and_eval(model): for n in range(FLAGS.train_epochs): tf.compat.v1.logging.info('=' * 30 + ' START EPOCH {} '.format(n + 1) + '=' * 30 + '\n') train_data_list = list_files(FLAGS.train_data) # dir to file list for f in train_data_list: t0 = time.time() tf.compat.v1.logging.info('<EPOCH {}>: Start training {}'.format( n + 1, f)) model.train(input_fn=lambda: input_fn(f, FLAGS.image_train_data, 'train', FLAGS.batch_size), hooks=None, steps=None, max_steps=None, saving_listeners=None) tf.compat.v1.logging.info( '<EPOCH {}>: Finish training {}, take {} mins'.format( n + 1, f, elapse_time(t0))) print('-' * 80) tf.compat.v1.logging.info('<EPOCH {}>: Start evaluating {}'.format( n + 1, FLAGS.eval_data)) t0 = time.time() results = model.evaluate( input_fn=lambda: input_fn(FLAGS.eval_data, FLAGS.image_eval_data, 'eval', FLAGS. batch_size), steps=None, # Number of steps for which to evaluate model. hooks=None, checkpoint_path=None, # latest checkpoint in model_dir is used. name=None) tf.compat.v1.logging.info( '<EPOCH {}>: Finish evaluation {}, take {} mins'.format( n + 1, FLAGS.eval_data, elapse_time(t0))) print('-' * 80) # Display evaluation metrics for key in sorted(results): print('{}: {}'.format(key, results[key])) # every epochs_per_eval test the model (use larger test dataset) if (n + 1) % FLAGS.epochs_per_eval == 0: tf.compat.v1.logging.info('<EPOCH {}>: Start testing {}'.format( n + 1, FLAGS.test_data)) results = model.evaluate( input_fn=lambda: input_fn(FLAGS.test_data, FLAGS.image_test_data, 'pred', FLAGS. batch_size), steps=None, # Number of steps for which to evaluate model. hooks=None, checkpoint_path= None, # If None, the latest checkpoint in model_dir is used. name=None) tf.compat.v1.logging.info( '<EPOCH {}>: Finish testing {}, take {} mins'.format( n + 1, FLAGS.test_data, elapse_time(t0))) print('-' * 80) # Display evaluation metrics for key in sorted(results): print('{}: {}'.format(key, results[key]))
def dynamic_train(model): """Dynamic train mode. For example: train_data_files: [0301, 0302, 0303, ...] train mode: first take 0301 as train data, 0302 as test data; then keep training take 0302 as train data, 0303 as test data ... """ data_files = list_files(FLAGS.train_data) data_files.sort() assert len(data_files) > 1, 'Dynamic train mode need more than 1 data file' for i in range(len(data_files) - 1): train_data = data_files[i] test_data = data_files[i + 1] tf.compat.v1.logging.info( '=' * 30 + ' START TRAINING DATA: {} '.format(train_data) + '=' * 30 + '\n') for n in range(FLAGS.train_epochs): t0 = time.time() tf.compat.v1.logging.info( 'START TRAIN DATA <{}> <EPOCH {}>'.format(train_data, n + 1)) model.train(input_fn=lambda: input_fn( train_data, FLAGS.image_train_data, 'train', FLAGS.batch_size), hooks=None, steps=None, max_steps=None, saving_listeners=None) tf.compat.v1.logging.info( 'FINISH TRAIN DATA <{}> <EPOCH {}> take {} mins'.format( train_data, n + 1, elapse_time(t0))) print('-' * 80) tf.compat.v1.logging.info( 'START EVALUATE TEST DATA <{}> <EPOCH {}>'.format( test_data, n + 1)) t0 = time.time() results = model.evaluate( input_fn=lambda: input_fn(test_data, FLAGS.image_eval_data, 'eval', FLAGS.batch_size), steps=None, # Number of steps for which to evaluate model. hooks=None, checkpoint_path=None, # latest checkpoint in model_dir is used. name=None) tf.compat.v1.logging.info( 'FINISH EVALUATE TEST DATA <{}> <EPOCH {}>: take {} mins'. format(test_data, n + 1, elapse_time(t0))) print('-' * 80) # Display evaluation metrics for key in sorted(results): print('{}: {}'.format(key, results[key]))
def train(model): for n in range(FLAGS.train_epochs): tf.compat.v1.logging.info('=' * 30 + ' START EPOCH {} '.format(n + 1) + '=' * 30 + '\n') train_data_list = list_files(FLAGS.train_data) # dir to file list for f in train_data_list: t0 = time.time() tf.compat.v1.logging.info('<EPOCH {}>: Start training {}'.format(n + 1, f)) model.train( input_fn=lambda: input_fn(f, FLAGS.image_train_data, 'train', FLAGS.batch_size), hooks=None, steps=None, max_steps=None, saving_listeners=None) tf.compat.v1.logging.info('<EPOCH {}>: Finish training {}, take {} mins'.format(n + 1, f, elapse_time(t0)))