def __init__(self, is_training=True, is_predict=False): super(DecomposableNLI, self).__init__() self.is_training = is_training self.is_predict = is_predict #TODO move to melt.EmbeddingTrainerBase emb_dim = FLAGS.emb_dim init_width = 0.5 / emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size # cpu for adgrad optimizer self.emb = embedding.get_or_restore_embedding_cpu() melt.visualize_embedding(self.emb, FLAGS.vocab) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self._attention_output_size = 256 self._comparison_output_size = 128 self.scope = 'decomposable_nli' self.build_train_graph = self.build_graph
def __init__(self, is_training=True, is_predict=False): super(DiscriminantTrainer, self).__init__() self.is_training = is_training self.is_predict = is_predict self.gen_text_feature = None emb_dim = FLAGS.emb_dim init_width = 0.5 / emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size #if not cpu and on gpu run and using adagrad, will fail TODO check why #also this will be more safer, since emb is large might exceed gpu mem #with tf.device('/cpu:0'): # #NOTICE if using bidirectional rnn then actually emb_dim is emb_dim / 2, because will at last step depth-concatate output fw and bw vectors # self.emb = melt.variable.get_weights_uniform('emb', [vocab_size, emb_dim], -init_width, init_width) self.emb = embedding.get_embedding_cpu('emb') melt.visualize_embedding(self.emb, FLAGS.vocab) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self.activation = melt.activations[FLAGS.activation] self.weights_initializer = tf.random_uniform_initializer( -FLAGS.initializer_scale, FLAGS.initializer_scale) self.biases_initialzier = melt.slim.init_ops.zeros_initializer if FLAGS.bias else None if not FLAGS.pre_calc_image_feature: assert melt.apps.image_processing.image_processing_fn is not None, 'forget melt.apps.image_processing.init()' self.image_process_fn = functools.partial( melt.apps.image_processing.image_processing_fn, height=FLAGS.image_height, width=FLAGS.image_width)
def __init__(self, is_training=True, is_predict=False): self.is_training = is_training self.is_predict = is_predict vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size self.end_id = vocabulary.end_id() self.get_start_id() assert self.end_id != vocabulary.vocab.unk_id(), 'input vocab generated without end id' self.emb_dim = emb_dim = FLAGS.emb_dim #--- for perf problem here exchange w_t and w https://github.com/tensorflow/tensorflow/issues/4138 self.num_units = num_units = FLAGS.rnn_hidden_size with tf.variable_scope('output_projection'): self.w_t = melt.variable.get_weights_truncated('w', [vocab_size, num_units], stddev=FLAGS.weight_stddev) self.w = tf.transpose(self.w_t) self.v = melt.variable.get_weights_truncated('v', [vocab_size], stddev=FLAGS.weight_stddev) self.cell = melt.create_rnn_cell( num_units=num_units, is_training=is_training, keep_prob=FLAGS.keep_prob, num_layers=FLAGS.num_layers, cell_type=FLAGS.cell) num_sampled = FLAGS.num_sampled if not (is_predict and FLAGS.predict_no_sample) else 0 self.softmax_loss_function = self.gen_sampled_softmax_loss_function(num_sampled)
def __init__(self, encoder_type='bow', is_training=True, is_predict=False): super(DiscriminantTrainer, self).__init__() self.is_training = is_training self.is_predict = is_predict logging.info('emb_dim:{}'.format(FLAGS.emb_dim)) logging.info('margin:{}'.format(FLAGS.margin)) self.encoder = encoder_factory.get_encoder(encoder_type, is_training, is_predict) self.encoder_type = encoder_type emb_dim = FLAGS.emb_dim init_width = 0.5 / emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size self.emb = embedding.get_or_restore_embedding_cpu() melt.visualize_embedding(self.emb, vocabulary.vocab_path) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self.activation = melt.activations[FLAGS.activation] #TODO can consider global initiallizer like # with tf.variable_scope("Model", reuse=None, initializer=initializer) #https://github.com/tensorflow/models/blob/master/tutorials/rnn/ptb/ptb_word_lm.py self.weights_initializer = tf.random_uniform_initializer( -FLAGS.initializer_scale, FLAGS.initializer_scale) self.biases_initializer = melt.slim2.init_ops.zeros_initializer if FLAGS.bias else None self.image_process_fn = lambda x: x if not FLAGS.pre_calc_image_feature: assert melt.apps.image_processing.image_processing_fn is not None, 'forget melt.apps.image_processing.init()' self.image_process_fn = functools.partial( melt.apps.image_processing.image_processing_fn, height=FLAGS.image_height, width=FLAGS.image_width, trainable=FLAGS.finetune_image_model, is_training=is_training, random_crop=FLAGS.random_crop_image, finetune_end_point=FLAGS.finetune_end_point, distort=FLAGS.distort_image, feature_name=FLAGS.image_endpoint_feature_name) self.image_mlp_dims = [ int(x) for x in FLAGS.image_mlp_dims.split(',') ] if FLAGS.image_mlp_dims is not '0' else None self.text_mlp_dims = [int(x) for x in FLAGS.text_mlp_dims.split(',') ] if FLAGS.text_mlp_dims is not '0' else None self.scope = 'image_text_sim'
def train(): global vocab_size vocabulary.init() vocab_size = vocabulary.get_vocab_size() def seq2seq_criterion(vocabSize): weight = torch.ones(vocabSize) weight[0] = 0 crit = nn.NLLLoss(weight, size_average=False) if torch.cuda.is_available(): crit.cuda() return crit global criterion criterion = seq2seq_criterion(vocab_size) model = seq2seq.Seq2Seq(vocab_size, FLAGS.emb_dim, FLAGS.rnn_hidden_size, FLAGS.batch_size) if torch.cuda.is_available(): model.cuda() init_range = 0.08 model.init_weights(init_range) optimizer = optim.Adagrad(model.parameters(), lr=FLAGS.learning_rate) inputs, decode = input.get_decodes(FLAGS.shuffle_then_decode, FLAGS.dynamic_batch_length) inputs = functools.partial( inputs, decode=decode, num_epochs=FLAGS.num_epochs, num_threads=FLAGS.num_threads, batch_join=FLAGS.batch_join, shuffle_batch=FLAGS.shuffle_batch, shuffle=FLAGS.shuffle, allow_smaller_final_batch=True, ) ops = inputs(FLAGS.input, batch_size=FLAGS.batch_size) print(ops) eval_ops = None if FLAGS.valid_input: #eval_ops = inputs(FLAGS.valid_input, batch_size=FLAGS.batch_size*10) eval_ops = inputs(FLAGS.valid_input, batch_size=FLAGS.batch_size) timer = Timer() tf_flow(lambda sess, step: process_once(sess, step, ops, eval_ops, model, optimizer)) print(timer.elapsed())
def get_embedding(name='emb'): emb_dim = FLAGS.emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() ##NOTICE if using bidirectional rnn then actually emb_dim is emb_dim / 2, because will as last step depth-concatate output fw and bw vectors init_width = 0.5 / emb_dim emb = melt.variable.get_weights_uniform(name, [vocab_size, emb_dim], -init_width, init_width) #return to above code if this works not better #emb = melt.variable.get_weights_truncated(name, [vocab_size, emb_dim], stddev=FLAGS.weight_stddev) return emb
def get_embedding(name='emb', height=None, emb_dim=None, trainable=True): emb_dim = emb_dim or FLAGS.emb_dim if height is None: vocabulary.init() height = vocabulary.get_vocab_size() init_width = 0.5 / emb_dim emb = melt.variable.get_weights_uniform(name, [height, emb_dim], -init_width, init_width, trainable=trainable) #return to above code if this works not better #emb = melt.variable.get_weights_truncated(name, [vocab_size, emb_dim], stddev=FLAGS.weight_stddev) return emb
def __init__(self, is_training=True, is_predict=False): self.scope = 'rnn' self.is_training = is_training self.is_predict = is_predict assert not (FLAGS.decode_copy and FLAGS.decode_use_alignment) vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size self.end_id = vocabulary.end_id() self.get_start_id() assert self.end_id != vocabulary.vocab.unk_id( ), 'input vocab generated without end id' self.emb_dim = emb_dim = FLAGS.emb_dim #--- for perf problem here exchange w_t and w https://github.com/tensorflow/tensorflow/issues/4138 self.num_units = num_units = FLAGS.rnn_hidden_size with tf.variable_scope('output_projection'): self.w_t = melt.variable.get_weights_truncated( 'w_t', [vocab_size, num_units], stddev=FLAGS.weight_stddev) #weights self.w = tf.transpose(self.w_t) #biases self.v = melt.variable.get_weights_truncated( 'v', [vocab_size], stddev=FLAGS.weight_stddev) #TODO https://github.com/tensorflow/tensorflow/issues/6761 tf 1.0 will fail if not scope='rnn' the same as when using self.cell... self.cell = melt.create_rnn_cell(num_units=num_units, is_training=is_training, keep_prob=FLAGS.keep_prob, num_layers=FLAGS.num_layers, cell_type=FLAGS.cell) num_sampled = FLAGS.num_sampled if not ( is_predict and FLAGS.predict_no_sample) else 0 self.softmax_loss_function = melt.seq2seq.gen_sampled_softmax_loss_function( num_sampled, self.vocab_size, self.w_t, self.v, FLAGS.log_uniform_sample, is_predict=self.is_predict, sample_seed=FLAGS.predict_sample_seed, vocabulary=vocabulary)
def __init__(self, encoder_type='bow', is_training=True, is_predict=False): super(DualTextsim, self).__init__() self.is_training = is_training self.is_predict = is_predict self.encoder = encoder_factory.get_encoder(encoder_type, is_training, is_predict) self.encoder_type = encoder_type emb_dim = FLAGS.emb_dim init_width = 0.5 / emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size # cpu for adgrad optimizer self.emb = embedding.get_or_restore_embedding_cpu() self.pos_emb = embedding.get_position_embedding_cpu() melt.visualize_embedding(self.emb, FLAGS.vocab) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self.activation = melt.activations[FLAGS.activation] # TODO can consider global initiallizer like # with tf.variable_scope("Model", reuse=None, initializer=initializer) # https://github.com/tensorflow/models/blob/master/tutorials/rnn/ptb/ptb_word_lm.py self.weights_initializer = tf.random_uniform_initializer( -FLAGS.initializer_scale, FLAGS.initializer_scale) self.biases_initialzier = melt.slim.init_ops.zeros_initializer if FLAGS.bias else None self.mlp_dims = [int(x) for x in FLAGS.mlp_dims.split(',') ] if FLAGS.mlp_dims is not '0' else None #needed in build graph from PairwiseGraph self.scope = 'dual_textsim' self.build_train_graph = self.build_graph
def __init__(self, is_training=True, is_predict=False): super(MilTrainer, self).__init__() self.is_training = is_training self.is_predict = is_predict logging.info('emb_dim:{}'.format(FLAGS.emb_dim)) logging.info('margin:{}'.format(FLAGS.margin)) vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size self.emb = embedding.get_or_restore_embedding_cpu() melt.visualize_embedding(self.emb, FLAGS.vocab) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self.image_process_fn = lambda x: x if not FLAGS.pre_calc_image_feature: assert melt.apps.image_processing.image_processing_fn is not None, 'forget melt.apps.image_processing.init()' self.image_process_fn = functools.partial( melt.apps.image_processing.image_processing_fn, height=FLAGS.image_height, width=FLAGS.image_width, trainable=FLAGS.finetune_image_model, is_training=is_training, random_crop=FLAGS.random_crop_image, finetune_end_point=FLAGS.finetune_end_point, distort=FLAGS.distort_image, feature_name=FLAGS.image_endpoint_feature_name) ImageEncoder = deepiu.seq2seq.image_encoder.Encoders[ FLAGS.image_encoder] self.image_encoder = ImageEncoder(is_training, is_predict, FLAGS.emb_dim) self.using_attention = FLAGS.image_encoder != 'ShowAndTell' assert self.using_attention with tf.variable_scope('text_encoder'): if FLAGS.text_encoder: self.text_encoder = encoder_factory.get_encoder( FLAGS.text_encoder, is_training, is_predict) else: self.text_encoder = None self.weights_initializer = tf.random_uniform_initializer( -FLAGS.initializer_scale, FLAGS.initializer_scale) self.activation = melt.activations[FLAGS.activation] self.text_mlp_dims = [int(x) for x in FLAGS.text_mlp_dims.split(',') ] if FLAGS.text_mlp_dims is not '0' else None self.biases_initializer = melt.slim2.init_ops.zeros_initializer if FLAGS.bias else None logging.info('mil text_encoder:{}'.format(self.text_encoder)) if FLAGS.use_idf_weights: self.idf_weights = tf.constant(idf.get_idf()) else: self.idf_weights = tf.constant( [0.] * NUM_RESERVED_IDS + [1.0 for id in range(NUM_RESERVED_IDS, vocab_size)]) self.scope = FLAGS.trainer_scope or 'image_text_sim'
def __init__(self, encoder_type='bow', is_training=True, is_predict=False): super(DualTextsim, self).__init__() self.is_training = is_training self.is_predict = is_predict self.encoder = encoder_factory.get_encoder(encoder_type, is_training, is_predict) self.encoder_type = encoder_type emb_dim = FLAGS.emb_dim init_width = 0.5 / emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size # cpu for adgrad optimizer if (not FLAGS.word_embedding_file) or glob.glob(FLAGS.model_dir + '/model.ckpt*'): logging.info( 'Word embedding random init or from model_dir :{} and finetune=:{}' .format(FLAGS.model_dir, FLAGS.finetune_word_embedding)) self.emb = embedding.get_embedding_cpu( name='emb', trainable=FLAGS.finetune_word_embedding) else: # https://github.com/tensorflow/tensorflow/issues/1570 # still adgrad must cpu.. # if not fintue emb this will be ok if fintune restart will ok ? must not use word embedding file? os.path.exists(FLAGS.model_dir) ? judge? # or will still try to load from check point ? TODO for safe you could re run by setting word_embedding_file as None or '' logging.info( 'Loading word embedding from :{} and finetune=:{}'.format( FLAGS.word_embedding_file, FLAGS.finetune_word_embedding)) self.emb = melt.load_constant_cpu( FLAGS.word_embedding_file, name='emb', trainable=FLAGS.finetune_word_embedding) if FLAGS.position_embedding: logging.info('Using position embedding') self.pos_emb = embedding.get_embedding_cpu(name='pos_emb', height=TEXT_MAX_WORDS) else: self.pos_emb = None melt.visualize_embedding(self.emb, FLAGS.vocab) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self.activation = melt.activations[FLAGS.activation] # TODO can consider global initiallizer like # with tf.variable_scope("Model", reuse=None, initializer=initializer) # https://github.com/tensorflow/models/blob/master/tutorials/rnn/ptb/ptb_word_lm.py self.weights_initializer = tf.random_uniform_initializer( -FLAGS.initializer_scale, FLAGS.initializer_scale) self.biases_initialzier = melt.slim.init_ops.zeros_initializer if FLAGS.bias else None self.mlp_dims = [int(x) for x in FLAGS.mlp_dims.split(',') ] if FLAGS.mlp_dims is not '0' else None self.scope = 'dual_textsim' self.build_train_graph = self.build_graph
def __init__(self, is_training=True, is_predict=False): self.scope = 'rnn' self.is_training = is_training self.is_predict = is_predict assert not (FLAGS.decode_copy and FLAGS.decode_use_alignment) vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size self.end_id = vocabulary.end_id() self.start_id = None self.get_start_id() assert self.end_id != vocabulary.vocab.unk_id( ), 'input vocab generated without end id' self.emb_dim = emb_dim = FLAGS.emb_dim #--- for perf problem here exchange w_t and w https://github.com/tensorflow/tensorflow/issues/4138 self.num_units = num_units = FLAGS.rnn_hidden_size with tf.variable_scope('output_projection'): self.w_t = melt.variable.get_weights_truncated( 'w_t', [vocab_size, num_units], stddev=FLAGS.weight_stddev) #weights self.w = tf.transpose(self.w_t) #biases self.v = melt.variable.get_weights_truncated( 'v', [vocab_size], stddev=FLAGS.weight_stddev) #TODO https://github.com/tensorflow/tensorflow/issues/6761 tf 1.0 will fail if not scope='rnn' the same as when using self.cell... self.cell = melt.create_rnn_cell(num_units=num_units, is_training=is_training, keep_prob=FLAGS.keep_prob, num_layers=FLAGS.num_layers, cell_type=FLAGS.cell) self.num_sampled = num_sampled = FLAGS.num_sampled if not ( is_predict and FLAGS.predict_no_sample) else 0 #self.softmax_loss_function is None means not need sample self.softmax_loss_function = None if FLAGS.gen_only: self.softmax_loss_function = melt.seq2seq.gen_sampled_softmax_loss_function( num_sampled, self.vocab_size, weights=self.w_t, biases=self.v, log_uniform_sample=FLAGS.log_uniform_sample, is_predict=self.is_predict, sample_seed=FLAGS.predict_sample_seed, vocabulary=vocabulary) if FLAGS.use_attention: print('----attention_option:', FLAGS.attention_option) if FLAGS.gen_copy_switch or FLAGS.gen_copy or FLAGS.copy_only: assert FLAGS.use_attention is True, 'must use attention if not gen_only mode seq2seq' FLAGS.gen_only = False if FLAGS.gen_copy_switch: print('-------gen copy switch mode!') FLAGS.gen_copy = False FLAGS.copy_only = False elif FLAGS.gen_copy: print('-------gen copy mode !') FLAGS.copy_only = False else: print('-------copy only mode !') else: print('--------gen only mode') #if use copy mode use score as alignment(no softmax) self.score_as_alignment = False if FLAGS.gen_only else True #gen only output_fn self.output_fn = lambda cell_output: melt.dense( cell_output, self.w, self.v) def copy_output(indices, batch_size, cell_output, cell_state): alignments = cell_state.alignments updates = alignments return tf.scatter_nd(indices, updates, shape=[batch_size, self.vocab_size]) self.copy_output_fn = copy_output #one problem is big memory for large vocabulary def gen_copy_output(indices, batch_size, cell_output, cell_state): gen_logits = self.output_fn(cell_output) copy_logits = copy_output(indices, batch_size, cell_output, cell_state) if FLAGS.gen_copy_switch: gen_probability = cell_state.gen_probability #[batch_size, 1] * [batch_size, vocab_size] return gen_probability * tf.nn.softmax(gen_logits) + ( 1 - gen_probability) * tf.nn.softmax(copy_logits) else: return gen_logits + copy_logits self.gen_copy_output_fn = gen_copy_output def gen_copy_output_train(time, indices, targets, sampled_values, batch_size, cell_output, cell_state): if self.softmax_loss_function is not None: labels = tf.slice(targets, [0, time], [-1, 1]) sampled, true_expected_count, sampled_expected_count = sampled_values sampled_values = \ sampled, tf.slice(tf.reshape(true_expected_count, [batch_size, -1]), [0, time], [-1, 1]), sampled_expected_count sampled_ids, sampled_logits = melt.nn.compute_sampled_ids_and_logits( weights=self.w_t, biases=self.v, labels=labels, inputs=cell_output, num_sampled=self.num_sampled, num_classes=self.vocab_size, sampled_values=sampled_values, remove_accidental_hits=False) gen_indices = melt.batch_values_to_indices( tf.to_int32(sampled_ids)) gen_logits = tf.scatter_nd(gen_indices, sampled_logits, shape=[batch_size, self.vocab_size]) else: gen_logits = self.output_fn(cell_output) copy_logits = copy_output(indices, batch_size, cell_output, cell_state) if FLAGS.gen_copy_switch: #gen_copy_switch == True. gen_probability = cell_state.gen_probability return gen_probability * tf.nn.softmax(gen_logits) + ( 1 - gen_probability) * tf.nn.softmax(copy_logits) else: return gen_logits + copy_logits self.gen_copy_output_train_fn = gen_copy_output_train
def __init__(self, encoder_type='bow', is_training=True, is_predict=False): super(DiscriminantTrainer, self).__init__() self.is_training = is_training self.is_predict = is_predict logging.info('emb_dim:{}'.format(FLAGS.emb_dim)) logging.info('margin:{}'.format(FLAGS.margin)) self.encoder = encoder_factory.get_encoder(encoder_type, is_training, is_predict) self.encoder_type = encoder_type emb_dim = FLAGS.emb_dim init_width = 0.5 / emb_dim vocabulary.init() vocab_size = vocabulary.get_vocab_size() self.vocab_size = vocab_size #if not cpu and on gpu run and using adagrad, will fail TODO check why #also this will be more safer, since emb is large might exceed gpu mem #with tf.device('/cpu:0'): # self.emb = melt.variable.get_weights_uniform('emb', [vocab_size, emb_dim], -init_width, init_width) if (not FLAGS.word_embedding_file) or glob.glob(FLAGS.model_dir + '/model.ckpt*'): logging.info( 'Word embedding random init or from model_dir :{} and finetune=:{}' .format(FLAGS.model_dir, FLAGS.finetune_word_embedding)) self.emb = embedding.get_embedding_cpu( name='emb', trainable=FLAGS.finetune_word_embedding) else: #https://github.com/tensorflow/tensorflow/issues/1570 #still adgrad must cpu.. #if not fintue emb this will be ok if fintune restart will ok ? must not use word embedding file? os.path.exists(FLAGS.model_dir) ? judge? #or will still try to load from check point ? TODO for safe you could re run by setting word_embedding_file as None or '' logging.info( 'Loading word embedding from :{} and finetune=:{}'.format( FLAGS.word_embedding_file, FLAGS.finetune_word_embedding)) self.emb = melt.load_constant_cpu( FLAGS.word_embedding_file, name='emb', trainable=FLAGS.finetune_word_embedding) melt.visualize_embedding(self.emb, FLAGS.vocab) if is_training and FLAGS.monitor_level > 0: melt.monitor_embedding(self.emb, vocabulary.vocab, vocab_size) self.activation = melt.activations[FLAGS.activation] #TODO can consider global initiallizer like # with tf.variable_scope("Model", reuse=None, initializer=initializer) #https://github.com/tensorflow/models/blob/master/tutorials/rnn/ptb/ptb_word_lm.py self.weights_initializer = tf.random_uniform_initializer( -FLAGS.initializer_scale, FLAGS.initializer_scale) self.biases_initialzier = melt.slim.init_ops.zeros_initializer if FLAGS.bias else None if not FLAGS.pre_calc_image_feature: assert melt.apps.image_processing.image_processing_fn is not None, 'forget melt.apps.image_processing.init()' self.image_process_fn = functools.partial( melt.apps.image_processing.image_processing_fn, height=FLAGS.image_height, width=FLAGS.image_width) self.image_mlp_dims = [ int(x) for x in FLAGS.image_mlp_dims.split(',') ] if FLAGS.image_mlp_dims is not '0' else None self.text_mlp_dims = [int(x) for x in FLAGS.text_mlp_dims.split(',') ] if FLAGS.text_mlp_dims is not '0' else None self.scope = 'image_text_sim'