def experiment_fn(run_config, params): model = Model() estimator = tf.estimator.Estimator( model_fn=model.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) source_vocab = data_loader.load_vocab("source_vocab") target_vocab = data_loader.load_vocab("target_vocab") Config.data.rev_source_vocab = utils.get_rev_vocab(source_vocab) Config.data.rev_target_vocab = utils.get_rev_vocab(target_vocab) Config.data.source_vocab_size = len(source_vocab) Config.data.target_vocab_size = len(target_vocab) train_data, test_data = data_loader.make_train_and_test_set() train_input_fn, train_input_hook = data_loader.get_dataset_batch(train_data, batch_size=Config.model.batch_size, scope="train") test_input_fn, test_input_hook = data_loader.get_dataset_batch(test_data, batch_size=Config.model.batch_size, scope="test") train_hooks = [train_input_hook] if Config.train.print_verbose: train_hooks.append(hook.print_variables( variables=['train/enc_0'], rev_vocab=utils.get_rev_vocab(source_vocab), every_n_iter=Config.train.check_hook_n_iter)) train_hooks.append(hook.print_variables( variables=['train/target_0', 'train/pred_0'], rev_vocab=utils.get_rev_vocab(target_vocab), every_n_iter=Config.train.check_hook_n_iter)) if Config.train.debug: train_hooks.append(tf_debug.LocalCLIDebugHook()) eval_hooks = [test_input_hook] if Config.train.debug: eval_hooks.append(tf_debug.LocalCLIDebugHook()) experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, min_eval_frequency=Config.train.min_eval_frequency, train_monitors=train_hooks, eval_hooks=eval_hooks ) return experiment
def experiment_fn(run_config, params): """ Experiemnt API """ if params.model == "seq2seq": model = seq2seq_model.Seq2Seq() elif params.model == "s2vt": model = s2vt_model.S2VT() estimator = tf.estimator.Estimator(model_fn=model.model_fn, model_dir=params.logdir, params=params, config=run_config) vocab = load_vocab(params.vocab_path) train_videos, train_captions = dataset.data_reader( params.train_feature_path, params.train_vid_path, params.train_capid_path) test_videos, test_captions = dataset.data_reader(params.test_feature_path, params.test_vid_path, params.test_capid_path) train_input_fn, train_input_hook = dataset.get_train_inputs( train_videos, train_captions) test_input_fn, test_input_hook = dataset.get_test_inputs( test_videos, test_captions) experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=params.train_steps, min_eval_frequency=params.min_eval_frequency, train_monitors=[ train_input_hook, hook.print_variables( variables=["Train_Data/caption_0", "train/pred_0"], vocab=vocab, every_n_iter=params.check_hook_n_iter) ], eval_hooks=[ test_input_hook, hook.print_variables( variables=["Test_Data/caption_0", "train/pred_0"], vocab=vocab, every_n_iter=params.check_hook_n_iter) ]) return experiment
def experiment_fn(run_config, params): seq2seq = Seq2Seq() estimator = tf.estimator.Estimator(model_fn=seq2seq.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) vocab = data_loader.load_vocab("vocab") Config.data.vocab_size = len(vocab) train_X, test_X, train_y, test_y = data_loader.make_train_and_test_set() train_input_fn, train_input_hook = dataset.get_train_inputs( train_X, train_y) test_input_fn, test_input_hook = dataset.get_test_inputs(test_X, test_y) experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, min_eval_frequency=Config.train.min_eval_frequency, train_monitors=[ train_input_hook, hook.print_variables( variables=['train/enc_0', 'train/dec_0', 'train/pred_0'], vocab=vocab, every_n_iter=Config.train.check_hook_n_iter) ], eval_hooks=[test_input_hook]) return experiment
def experiment_fn(run_config, params): conversation = Conversation() estimator = tf.estimator.Estimator(model_fn=conversation.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) vocab = data_loader.load_vocab("vocab") Config.data.vocab_size = len(vocab) train_X, test_X, train_y, test_y = data_loader.make_train_and_test_set() train_input_fn, train_input_hook = data_loader.make_batch( (train_X, train_y), batch_size=Config.model.batch_size) test_input_fn, test_input_hook = data_loader.make_batch( (test_X, test_y), batch_size=Config.model.batch_size, scope="test") experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, min_eval_frequency=Config.train.min_eval_frequency, train_monitors=[ train_input_hook, hook.print_variables( variables=['train/enc_0', 'train/dec_0', 'train/pred_0'], vocab=vocab, every_n_iter=Config.train.check_hook_n_iter) ], eval_hooks=[test_input_hook], eval_delay_secs=0) return experiment
def experiment_fn(run_config, params): # 先定义estimator conversation = Conversation() estimator = tf.estimator.Estimator(model_fn=conversation.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) # 返回字典 vocab = data_loader.load_vocab("vocab") Config.data.vocab_size = len(vocab) # 定义训练数据 train_X, test_X, train_y, test_y = data_loader.make_train_and_test_set() train_input_fn, train_input_hook = data_loader.make_batch( (train_X, train_y), batch_size=Config.model.batch_size) test_input_fn, test_input_hook = data_loader.make_batch( (test_X, test_y), batch_size=Config.model.batch_size, scope="test") train_hooks = [train_input_hook] if Config.train.print_verbose: train_hooks.append( hook.print_variables( variables=['train/enc_0', 'train/dec_0', 'train/pred_0'], rev_vocab=utils.get_rev_vocab(vocab), every_n_iter=Config.train.check_hook_n_iter)) if Config.train.debug: train_hooks.append(tf_debug.LocalCLIDebugHook()) eval_hooks = [test_input_hook] if Config.train.debug: eval_hooks.append(tf_debug.LocalCLIDebugHook()) # 定义实验 experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, min_eval_frequency=Config.train.min_eval_frequency, train_monitors=train_hooks, eval_hooks=eval_hooks, eval_delay_secs=0) return experiment
def experiment_fn(run_config, params): model = Model() estimator = tf.estimator.Estimator(model_fn=model.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) train_data, test_data = data_loader.make_train_and_test_set() train_X, train_y = train_data test_X, test_y = train_data train_input_fn, train_input_hook = data_loader.make_batch( train_X, train_y, batch_size=Config.model.batch_size, scope="train") test_input_fn, test_input_hook = data_loader.make_batch( test_X, test_y, batch_size=Config.model.batch_size, scope="test") train_hooks = [train_input_hook] if Config.train.print_verbose: train_hooks.append( hook.print_variables(variables=['train/input_0', 'train/target_0'], every_n_iter=Config.train.check_hook_n_iter)) if Config.train.debug: train_hooks.append(tf_debug.LocalCLIDebugHook()) eval_hooks = [test_input_hook] if Config.train.debug: eval_hooks.append(tf_debug.LocalCLIDebugHook()) experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, min_eval_frequency=Config.train.min_eval_frequency, train_monitors=train_hooks, eval_hooks=eval_hooks) return experiment
def experiment_fn(run_config, params): char_rnn = CharRNN() estimator = tf.estimator.Estimator(model_fn=char_rnn.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) data_loader = TextLoader(Config.data.data_dir, batch_size=params.batch_size, seq_length=params.seq_length) Config.data.vocab_size = data_loader.vocab_size train_X, test_X, train_y, test_y = data_loader.make_train_and_test_set() train_input_fn, train_input_hook = dataset.get_train_inputs( train_X, train_y) test_input_fn, test_input_hook = dataset.get_test_inputs(test_X, test_y) experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, #min_eval_frequency=Config.train.min_eval_frequency, train_monitors=[ train_input_hook, hook.print_variables( variables=['training/output_0', 'prediction_0'], vocab=data_loader.vocab, every_n_iter=Config.train.check_hook_n_iter) ], eval_hooks=[test_input_hook], #eval_steps=None ) return experiment
def experiment_fn(run_config, params): if Config.model.type == 'ocr': from ocr import ocr_model model = ocr_model.Model() elif Config.model.type == 'text': model = Model() elif Config.model.type == 'text-dnn': from textdnn import text_dnn_model model = text_dnn_model.Model() elif Config.model.type == 'line-crf': from linecrf import crf_model model = crf_model.Model() elif Config.model.type == 'cell-merge': from cellMerge import cell_model model = cell_model.Model() else: raise KeyError('Unknown model type %s' % Config.model.type) estimator = tf.estimator.Estimator( model_fn=model.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config) vocab = data_loader.load_vocab() Config.data.vocab_size = len(vocab) print("vocab_size: %d" % Config.data.vocab_size) train_input_fn, train_input_hook, test_input_fn, test_input_hook = data_loader.make_train_and_test_input_fn() train_hooks = [] if train_input_hook is not None: train_hooks.append(train_input_hook) if Config.train.print_verbose and Config.model.type == 'text': train_hooks.append(hook.print_variables( variables=['train/input_0'], rev_vocab=vocab, every_n_iter=Config.train.check_hook_n_iter)) train_hooks.append(hook.print_target( variables=['train/target_0', 'train/pred_0'], rev_vocab=vocab, every_n_iter=Config.train.check_hook_n_iter)) if Config.train.print_verbose and Config.model.type == 'text-dnn': train_hooks.append(hook.print_variables( variables=['pre_processing/total_text_ids'], rev_vocab=vocab, every_n_iter=Config.train.check_hook_n_iter )) train_hooks.append(hook.print_variables( variables=['pre_processing/tags', 'text_graph/crf/crf_decode/text_pred_tags'], every_n_iter=Config.train.check_hook_n_iter)) if Config.train.debug: from tensorflow.python import debug as tf_debug train_hooks.append(tf_debug.LocalCLIDebugHook(ui_type='readline')) eval_hooks = [] if test_input_hook is not None: eval_hooks.append(test_input_hook) def serving_input_fn(): if Config.model.type == 'text': inputs = { "input_data": tf.placeholder( tf.int32, [None, Config.data.max_seq_length], name="input_data") } return tf.estimator.export.ServingInputReceiver(inputs, inputs) elif Config.model.type == 'ocr': def _preprocess_image(image_bytes): """Preprocess a single raw image.""" image = tf.image.decode_image(tf.reshape(image_bytes, shape=[]), channels=3) image.set_shape([None, None, None]) image = tf.image.convert_image_dtype(image, dtype=tf.float32) image = tf.image.resize_images(image, (64, 64)) image = tf.image.rgb_to_grayscale(image) return image use_uint8 = tf.placeholder(dtype=tf.bool, name='use_uint8') image_bytes_list = tf.placeholder( shape=[None], dtype=tf.string, name='image_bytes', ) uint8_images = tf.placeholder( shape=[None, 64, 64, 1], dtype=tf.uint8, name='uint8_images', ) def preprocess_uint8_image(): image = tf.image.convert_image_dtype(uint8_images, dtype=tf.float32) return image def preprocess_image(): return tf.map_fn(_preprocess_image, image_bytes_list, back_prop=False, dtype=tf.float32) features = { 'image': tf.cond(use_uint8, preprocess_uint8_image, preprocess_image, ) } receiver_tensors = {'image_bytes': image_bytes_list} return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) elif Config.model.type == 'text-dnn': input_dict = { 'margin': tf.placeholder(tf.float32, shape=[None, 4], name='margin'), 'indent': tf.placeholder(tf.float32, shape=[None, 4], name='indent'), 'font_size': tf.placeholder(tf.float32, shape=[None, 2], name='font_size'), 'text': tf.placeholder(tf.string, shape=[None,], name='text'), 'text_length': tf.placeholder(tf.int32, shape=[None,], name='text_length'), 'next_text': tf.placeholder(tf.string, shape=[None,], name='next_text'), 'next_text_length': tf.placeholder(tf.int32, shape=[None,], name='next_text_length'), } return tf.estimator.export.ServingInputReceiver( features=input_dict, receiver_tensors=input_dict) elif Config.model.type == 'line-crf': from linecrf import line_example_decoder example = tf.placeholder(dtype=tf.string, shape=[], name='serialized_example') input_dict, _ = line_example_decoder.parse_tfexample_fn(example, tf.estimator.ModeKeys.PREDICT) for key in input_dict: input_dict[key] = tf.expand_dims(input_dict[key], 0) return tf.estimator.export.ServingInputReceiver( features=input_dict, receiver_tensors={'serialized_example': example}) elif Config.model.type == 'cell-merge': # 此处是为导出模型的时候使用,features就是模型中的features input_dict = { 'x_l': tf.placeholder(tf.string, shape=[None], name='x_l'), 'x_r': tf.placeholder(tf.string, shape=[None], name='x_r'), 'l': tf.placeholder(tf.int64, shape=[None], name='l'), } return tf.estimator.export.ServingInputReceiver( features=input_dict, receiver_tensors=input_dict) else: return None experiment = tf.contrib.learn.Experiment( estimator=estimator, train_input_fn=train_input_fn, eval_input_fn=test_input_fn, train_steps=Config.train.train_steps, min_eval_frequency=Config.train.min_eval_frequency, eval_steps=Config.train.eval_steps, train_monitors=train_hooks, eval_hooks=eval_hooks, export_strategies=[saved_model_export_utils.make_export_strategy( serving_input_fn, default_output_alternative_key=None, exports_to_keep=1 )], ) return experiment