def transformer_encoder(x, src_mask, scope, num_heads, pdrop, scale=True, activation_type='relu', d_ff=None): with tf.variable_scope(scope): d_model = get_shape_as_list(x)[-1] if d_ff is None: d_ff = 4 * d_model x = layer_norm(x, 'ln_1') q, k, v = self_attention_qkv(x, d_model) a = multi_headed_attention(q, k, v, 'attn', d_model, num_heads, pdrop, scale=scale, mask=src_mask) x = x + tf.layers.dropout(a, pdrop, training=TRAIN_FLAG()) x = layer_norm(x, 'ln_2') m = ffn(x, 'ffn', pdrop, d_ff=d_ff, activation_type=activation_type) h = x + tf.layers.dropout(m, pdrop, training=TRAIN_FLAG()) return h
def pool(self, word_embeddings, dsz, init, **kwargs): """Do parallel convolutional filtering with varied receptive field widths, followed by max-over-time pooling :param word_embeddings: The word embeddings, which are inputs here :param dsz: The depth of the word embeddings :param init: The tensorflow initializer :param kwargs: See below :Keyword Arguments: * *cmotsz* -- (``int``) The number of convolutional feature maps for each filter These are MOT-filtered, leaving this # of units per parallel filter * *filtsz* -- (``list``) This is a list of filter widths to use :return: """ cmotsz = kwargs['cmotsz'] filtsz = kwargs['filtsz'] combine, _ = parallel_conv(word_embeddings, filtsz, dsz, cmotsz) # Definitely drop out with tf.name_scope("dropout"): combine = tf.layers.dropout(combine, self.pdrop_value, training=TRAIN_FLAG()) return combine
def encode(self, x=None): """Build a simple Lookup Table and set as input `x` if it exists, or `self.x` otherwise. :param x: An optional input sub-graph to bind to this operation or use `self.x` if `None` :return: The sub-graph output """ if x is None: x = LookupTableEmbeddings.create_placeholder(self.name) self.x = x with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): W = tf.get_variable("W", initializer=tf.constant_initializer( self.weights, dtype=tf.float32, verify_shape=True), shape=[self.vsz, self.dsz], trainable=self.finetune) e0 = tf.scatter_update(W, tf.constant(0, dtype=tf.int32, shape=[1]), tf.zeros(shape=[1, self.dsz])) with tf.control_dependencies([e0]): embedding_w_dropout = tf.layers.dropout(W, self.dropin, noise_shape=(self.vsz, 1), training=TRAIN_FLAG()) word_embeddings = tf.nn.embedding_lookup( embedding_w_dropout, self.x) return word_embeddings
def stacked(self, pooled, init, **kwargs): """Stack 1 or more hidden layers, optionally (forming an MLP) :param pooled: The fixed representation of the model :param init: The tensorflow initializer :param kwargs: See below :Keyword Arguments: * *hsz* -- (``int``) The number of hidden units (defaults to `100`) :return: The final layer """ hszs = listify(kwargs.get('hsz', [])) if len(hszs) == 0: return pooled in_layer = pooled for i, hsz in enumerate(hszs): fc = tf.layers.dense(in_layer, hsz, activation=tf.nn.relu, kernel_initializer=init, name='fc-{}'.format(i)) in_layer = tf.layers.dropout(fc, self.pdrop_value, training=TRAIN_FLAG(), name='fc-dropout-{}'.format(i)) return in_layer
def _test(self, loader, dataset=True, **kwargs): epoch_probs = [] epoch_y = [] epoch_loss = 0 epoch_norm = 0 pg = create_progress_bar(len(loader)) for batch_dict in pg(loader): if dataset: probs, lossv, y = self.sess.run( [self.probs, self.loss, self.y], feed_dict={TRAIN_FLAG(): 0} ) batchsz = len(y) else: feed_dict = self.model.make_input(batch_dict, False) probs, lossv, y = self.sess.run( [self.probs, self.loss, self.y], feed_dict=feed_dict ) batchsz = self._get_batchsz(batch_dict) epoch_loss += lossv * batchsz epoch_norm += batchsz epoch_probs.append(probs) epoch_y.append(y) probs = np.concatenate(epoch_probs, axis=0) y = np.argmax(np.concatenate(epoch_y, axis=0), axis=1) bins = multiclass_calibration_bins(y, probs, bins=int(kwargs.get('bins', 10))) metrics = { "ECE": expected_calibration_error(bins.accs, bins.confs, bins.counts), "MCE": maximum_calibration_error(bins.accs, bins.confs, bins.counts), "avg_loss": epoch_loss / float(epoch_norm) } return metrics
def ffn(x, scope, pdrop, d_ff, activation_type='relu'): with tf.variable_scope(scope): d_model = get_shape_as_list(x)[-1] act = tf_activation(activation_type) expansion = act(time_distributed_projection(x, name='ffn_ff', filters=d_ff)) dropped = tf.layers.dropout(expansion, pdrop, training=TRAIN_FLAG()) squeeze = time_distributed_projection(dropped, name='ffn_model', filters=d_model) return squeeze
def dot_product_attention(query, key, value, pdrop=0.0, mask=None, scale=True): w = tf.matmul(query, key, transpose_b=True) if scale: w *= tf.rsqrt(tf.to_float(tf.shape(query)[2])) if mask is not None: w = w * mask + -1e9 * (1 - mask) weights = tf.nn.softmax(w, name="attention_weights") weights = tf.layers.dropout(weights, pdrop, training=TRAIN_FLAG()) return tf.matmul(weights, value)
def transformer_decoder(tgt, src, src_mask, tgt_mask, scope, num_heads, pdrop, scale=True, activation_type='relu', d_ff=None): with tf.variable_scope(scope): d_model = get_shape_as_list(tgt)[-1] if d_ff is None: d_ff = 4 * d_model tgt = layer_norm(tgt, 'ln_1') q, k, v = self_attention_qkv(tgt, d_model) self_attn = multi_headed_attention(q, k, v, 'self_attn', d_model, num_heads, pdrop, scale=scale, mask=tgt_mask) tgt = tgt + tf.layers.dropout(self_attn, pdrop, training=TRAIN_FLAG()) tgt = layer_norm(tgt, 'ln_2') q, k, v = low_order_projection_qkv(tgt, src, src, d_model) # Mask at zeros??? src_attn = multi_headed_attention(q, k, v, "dual_attn", d_model, num_heads, pdrop, scale=scale, mask=src_mask) tgt = tgt + tf.layers.dropout(src_attn, pdrop, training=TRAIN_FLAG()) tgt = layer_norm(tgt, 'ln_3') m = ffn(tgt, 'ffn', pdrop, d_ff=d_ff, activation_type=activation_type) h = tgt + tf.layers.dropout(m, pdrop, training=TRAIN_FLAG()) return h
def _train(self, loader, dataset=True, **kwargs): """Train an epoch of data using either the input loader or using `tf.dataset` In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps to train. We do use a `feed_dict` for passing the `TRAIN_FLAG` in either case :param loader: A data feed :param kwargs: See below :Keyword Arguments: * *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True` * *reporting_fns* (`list`) A list of reporting hooks to use :return: Metrics """ if self.ema: self.sess.run(self.ema_restore) reporting_fns = kwargs.get('reporting_fns', []) epoch_loss = 0 epoch_div = 0 steps = len(loader) pg = create_progress_bar(steps) for batch_dict in pg(loader): if dataset: _, step, lossv = self.sess.run( [self.train_op, self.global_step, self.loss], feed_dict={TRAIN_FLAG(): 1}) else: feed_dict = self.model.make_input(batch_dict, True) _, step, lossv = self.sess.run( [self.train_op, self.global_step, self.loss], feed_dict=feed_dict) batchsz = self._get_batchsz(batch_dict) report_lossv = lossv * batchsz epoch_loss += report_lossv epoch_div += batchsz self.nstep_agg += report_lossv self.nstep_div += batchsz if (step + 1) % self.nsteps == 0: metrics = self.calc_metrics(self.nstep_agg, self.nstep_div) self.report(step + 1, metrics, self.nstep_start, 'Train', 'STEP', reporting_fns, self.nsteps) self.reset_nstep() metrics = self.calc_metrics(epoch_loss, epoch_div) return metrics
def fit(model_params, _, ts, vs=None, **kwargs): model_file = get_model_file('classify', 'tf', kwargs.get('basedir')) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) test_metrics = trainer.test(vs, reporting_fns, phase="Test-Before", dataset=False) trainer.train(ts, reporting_fns, dataset=False) test_metrics = trainer.test(vs, reporting_fns, phase="Test", dataset=False) trainer.checkpoint() trainer.model.save(model_file)
def train(self, ts, reporting_fns, dataset=True): """Train by looping over the steps For a `tf.dataset`-backed `fit_func`, we are using the previously wired `dataset`s in the model (and `dataset` is `True`). For `feed_dict`, we convert the ts samples to `feed_dict`s and hand them in one-by-one :param ts: The training set :param reporting_fns: A list of reporting hooks :param dataset: (`bool`) Are we using `tf.dataset`s :return: Metrics """ epoch_loss = 0 epoch_toks = 0 start = time.time() self.nstep_start = start for batch_dict in ts: if dataset: _, global_step, lossv = self.sess.run( [self.train_op, self.global_step, self.loss], feed_dict={TRAIN_FLAG(): 1}) else: feed_dict = self.model.make_input(batch_dict, True) _, global_step, lossv = self.sess.run( [self.train_op, self.global_step, self.loss], feed_dict=feed_dict) # ?? How to get this cleaner? toks = self._num_toks(batch_dict['tgt_lengths']) report_loss = lossv * toks epoch_loss += report_loss epoch_toks += toks self.nstep_agg += report_loss self.nstep_div += toks if (global_step + 1) % self.nsteps == 0: metrics = self.calc_metrics(self.nstep_agg, self.nstep_div) self.report(global_step + 1, metrics, self.nstep_start, 'Train', 'STEP', reporting_fns, self.nsteps) self.reset_nstep() metrics = self.calc_metrics(epoch_loss, epoch_toks) self.train_epochs += 1 self.report(self.train_epochs, metrics, start, 'Train', 'EPOCH', reporting_fns) return metrics
def encode(self, x=None): """Create a really large embedding matrix in tensorflow. Tensorflow has a limit on the size that a op can be (2GB). When we have very large embedding lookuptables (for example when we don't prune the vocab) we hit this limit and can't have the embeddings in the graph. This is due to a limit in the size that a thing can be in a protocol buffer (how tensorflow serializes the graph). Here we get around it with a placeholder. The place holder will be in the graph and it will know it needs to have a size of [vsz, dsz] but it doesn't have the actual values so it can be serialized into a protocol buffer since it is small. We then have a variable that is initialized with the value of the placeholder. This is filled in with value during the `sess.run` of `tf.global_variables_initialzier` with a feed_dict. Values are then saved into the checkpoint and can be reloaded from there. ``` sess.run(tf.global_variables_initializer(), {e.W_place: e.weights}) ``` """ if x is None: x = LookupTableEmbeddings.create_placeholder(self.name) self.x = x with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): self.W_place = tf.placeholder(tf.float32, shape=(self.vsz, self.dsz)) W = tf.get_variable("W", initializer=self.W_place) e0 = tf.scatter_update(W, tf.constant(0, dtype=tf.int32, shape=[1]), tf.zeros(shape=[1, self.dsz])) with tf.control_dependencies([e0]): embedding_w_dropout = tf.layers.dropout(W, self.dropin, noise_shape=(self.vsz, 1), training=TRAIN_FLAG()) word_embeddings = tf.nn.embedding_lookup( embedding_w_dropout, self.x) return word_embeddings
def fit(model_params, _, ts, vs=None, **kwargs): """Calibrate a model with temperature scaling""" model_file = get_model_file('classify', 'tf', kwargs.get('basedir')) batchsz = kwargs['batchsz'] lengths_key = model_params.get('lengths_key') ## First, make tf.datasets for ts, vs and es # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md # effective_batch_sz = args.batchsz*args.gpus train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts, lengths_key)) train_dataset = train_dataset.batch(batchsz, drop_remainder=False) train_dataset = train_dataset.repeat(2) train_dataset = train_dataset.prefetch(NUM_PREFETCH) valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs, lengths_key)) valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False) valid_dataset = valid_dataset.repeat(2) valid_dataset = valid_dataset.prefetch(NUM_PREFETCH) iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset), tf.compat.v1.data.get_output_shapes(train_dataset)) features, y = iter.get_next() # Add features to the model params model_params.update(features) model_params['y'] = tf.one_hot(tf.reshape(y, [-1]), len(model_params['labels'])) # create the initialisation operations train_init_op = iter.make_initializer(train_dataset) valid_init_op = iter.make_initializer(valid_dataset) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) last_improved = 0 trainer.sess.run(train_init_op) trainer.train(ts, reporting_fns) trainer.sess.run(valid_init_op) test_metrics = trainer.test(vs, reporting_fns, phase='Test') trainer.checkpoint() trainer.model.save(model_file)
def encode(self, x=None): if x is None: x = CharConvEmbeddings.create_placeholder(self.name) self.x = x ech0 = tf.scatter_update(self.Wch, tf.constant(0, dtype=tf.int32, shape=[1]), tf.zeros(shape=[1, self.dsz])) mxlen = tf.shape(self.x)[1] gating_fn = highway_conns if self.gating.startswith( 'highway') else skip_conns with tf.variable_scope("Chars2Word"): with tf.control_dependencies([ech0]): mxwlen = tf.shape(self.x)[-1] char_bt_x_w = tf.reshape(self.x, [-1, mxwlen]) # The ablation table (4) in https://arxiv.org/pdf/1708.02182.pdf shows this has a massive impact embedding_w_dropout = tf.layers.dropout(self.Wch, self.dropin, noise_shape=(self.vsz, 1), training=TRAIN_FLAG()) cembed = tf.nn.embedding_lookup(embedding_w_dropout, char_bt_x_w, name="embeddings") cmot, num_filts = char_word_conv_embeddings( cembed, self.filtsz, self.dsz, self.nfeats, activation_fn=tf_activation(self.activation), gating=gating_fn, num_gates=self.num_gates) if self.projsz: cmot = tf.matmul(cmot, self.Wp) + self.bp word_char = tf.reshape(cmot, [-1, mxlen, self.outsz]) return word_char
def _train(self, loader, dataset=True, **kwargs): reporting_fns = kwargs.get('reporting_fns', []) pg = create_progress_bar(len(loader)) epoch_loss = 0 epoch_norm = 0 epoch_logits = [] epoch_probs = [] epoch_y = [] start = time.time() for batch_dict in pg(loader): if dataset: logits, probs, lossv, y = self.sess.run( [self.logits, self.probs, self.loss, self.y], feed_dict={TRAIN_FLAG(): 0} ) batchsz = len(y) else: feed_dict = self.model.make_input(batch_dict, False) logits, probs, lossv, y = self.sess.run( [self.logits, self.probs, self.loss, self.y], feed_dict=feed_dict ) batchsz = self._get_batchsz(batch_dict) epoch_loss += lossv * batchsz epoch_norm += batchsz epoch_logits.append(logits) epoch_probs.append(probs) epoch_y.append(y) logits = np.concatenate(epoch_logits, axis=0) probs = np.concatenate(epoch_probs, axis=0) y = np.argmax(np.concatenate(epoch_y, axis=0), axis=1) bins = multiclass_calibration_bins(y, probs, bins=int(kwargs.get('bins', 10))) metrics = { "ECE": expected_calibration_error(bins.accs, bins.confs, bins.counts), "MCE": maximum_calibration_error(bins.accs, bins.confs, bins.counts), "avg_loss": epoch_loss / float(epoch_norm) } self.report( 0, metrics, start, "Train-Before", "EPOCH", reporting_fns, 1 ) # Fit import tensorflow_probability as tfp x = tf.constant(logits) y = tf.constant(y) def scale(p): return tfp.math.value_and_gradient( lambda v: tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits( logits=x / v, labels=y )), p ) results = tfp.optimizer.lbfgs_minimize( value_and_gradients_function=scale, initial_position=self.model.trainable_variables[0], max_iterations=100 ) self.sess.run(self.model.trainable_variables[0].assign(results.position)) metrics = self._test(loader, dataset, phase="Train-After", **kwargs) return metrics
def pool(self, word_embeddings, dsz, init, **kwargs): """LSTM with dropout yielding a final-state as output :param word_embeddings: The input word embeddings :param dsz: The input word embedding depth :param init: The tensorflow initializer to use (currently ignored) :param kwargs: See below :Keyword Arguments: * *rnnsz* -- (``int``) The number of hidden units (defaults to `hsz`) * *hsz* -- (``int``) backoff for `rnnsz`, typically a result of stacking params. This keeps things simple so its easy to do things like residual connections between LSTM and post-LSTM stacking layers :return: """ hsz = kwargs.get('rnnsz', kwargs.get('hsz', 100)) vdrop = bool(kwargs.get('variational_dropout', False)) if type(hsz) is list: hsz = hsz[0] rnntype = kwargs.get('rnn_type', kwargs.get('rnntype', 'lstm')) nlayers = int(kwargs.get('layers', 1)) if rnntype == 'blstm': rnnfwd = stacked_lstm(hsz // 2, self.pdrop_value, nlayers, variational=vdrop, training=TRAIN_FLAG()) rnnbwd = stacked_lstm(hsz // 2, self.pdrop_value, nlayers, variational=vdrop, training=TRAIN_FLAG()) ((_, _), (fw_final_state, bw_final_state)) = tf.nn.bidirectional_dynamic_rnn( rnnfwd, rnnbwd, word_embeddings, sequence_length=self.lengths, dtype=tf.float32) # The output of the BRNN function needs to be joined on the H axis output_state = tf.concat( [fw_final_state[-1].h, bw_final_state[-1].h], -1) out_hsz = hsz else: rnnfwd = stacked_lstm(hsz, self.pdrop_value, nlayers, variational=vdrop, training=TRAIN_FLAG()) (_, (output_state)) = tf.nn.dynamic_rnn(rnnfwd, word_embeddings, sequence_length=self.lengths, dtype=tf.float32) output_state = output_state[-1].h out_hsz = hsz combine = tf.reshape(output_state, [-1, out_hsz]) return combine
def train(self, ts, reporting_fns, dataset=True): """Train by looping over the steps For a `tf.dataset`-backed `fit_func`, we are using the previously wired `dataset`s in the model (and `dataset` is `True`). For `feed_dict`, we convert the ts samples to `feed_dict`s and hand them in one-by-one :param ts: The training set :param reporting_fns: A list of reporting hooks :param dataset: (`bool`) Are we using `tf.dataset`s :return: Metrics """ epoch_loss = 0.0 epoch_toks = 0 if self.model.requires_state: state = self.model.sess.run(self.model.initial_state, self.model.make_input(ts[0], True)) fetches = { "loss": self.loss, "train_op": self.train_op, "global_step": self.global_step } if self.model.requires_state: fetches["final_state"] = self.model.final_state start = time.time() self.nstep_start = start for batch_dict in ts: if dataset: feed_dict = {TRAIN_FLAG(): 1} else: feed_dict = self.model.make_input(batch_dict, True) _, global_step, lossv = self.sess.run( [self.train_op, self.global_step, self.loss], feed_dict=feed_dict) # In Keras LSTM, the order is h first, c second, its the opposite in TF 1, however I dont think it # ends up mattering here if self.model.requires_state: for i, (s1, s2) in enumerate(self.model.initial_state): feed_dict[s1] = state[i][0] #.c # 0 feed_dict[s2] = state[i][1] #.h # 1 vals = self.model.sess.run(fetches, feed_dict) loss = vals["loss"] if self.model.requires_state: state = vals["final_state"] global_step = vals["global_step"] toks = self._num_toks(batch_dict) report_loss = loss * toks epoch_loss += report_loss epoch_toks += toks self.nstep_agg += report_loss self.nstep_div += toks if (global_step + 1) % self.nsteps == 0: metrics = self.calc_metrics(self.nstep_agg, self.nstep_div) self.report(global_step + 1, metrics, self.nstep_start, 'Train', 'STEP', reporting_fns, self.nsteps) self.reset_nstep() metrics = self.calc_metrics(epoch_loss, epoch_toks) self.train_epochs += 1 self.report(self.train_epochs, metrics, start, 'Train', 'EPOCH', reporting_fns) return metrics
def create(cls, embeddings, labels, **kwargs): """The main method for creating all :class:`WordBasedModel` types. This method instantiates a model with pooling and optional stacking layers. Many of the arguments provided are reused by each implementation, but some sub-classes need more information in order to properly initialize. For this reason, the full list of keyword args are passed to the :method:`pool` and :method:`stacked` methods. :param embeddings: This is a dictionary of embeddings, mapped to their numerical indices in the lookup table :param labels: This is a list of the `str` labels :param kwargs: There are sub-graph specific Keyword Args allowed for e.g. embeddings. See below for known args: :Keyword Arguments: * *gpus* -- (``int``) How many GPUs to split training across. If called this function delegates to another class `ClassifyParallelModel` which creates a parent graph and splits its inputs across each sub-model, by calling back into this exact method (w/o this argument), once per GPU * *model_type* -- The string name for the model (defaults to `default`) * *sess* -- An optional tensorflow session. If not passed, a new session is created * *lengths_key* -- (``str``) Specifies which `batch_dict` property should be used to determine the temporal length if this is not set, it defaults to either `word`, or `x` if `word` is also not a feature * *finetune* -- Are we doing fine-tuning of word embeddings (defaults to `True`) * *mxlen* -- The maximum signal (`x` tensor temporal) length (defaults to `100`) * *dropout* -- This indicates how much dropout should be applied to the model when training. * *filtsz* -- This is actually a top-level param due to an unfortunate coupling between the pooling layer and the input, which, for convolution, requires input padding. :return: A fully-initialized tensorflow classifier """ TRAIN_FLAG() gpus = kwargs.get('gpus', 1) if gpus == -1: gpus = len( os.getenv('CUDA_VISIBLE_DEVICES', os.getenv('NV_GPU', '0')).split(',')) kwargs['gpus'] = gpus if gpus > 1: return ClassifyParallelModel(cls.create, embeddings, labels, **kwargs) sess = kwargs.get('sess', tf.Session()) model = cls() model.embeddings = embeddings model._record_state(**kwargs) model.lengths_key = kwargs.get('lengths_key') if model.lengths_key is not None: model.lengths = kwargs.get( 'lengths', tf.placeholder(tf.int32, [None], name="lengths")) else: model.lengths = None model.labels = labels nc = len(labels) model.y = kwargs.get('y', tf.placeholder(tf.int32, [None, nc], name="y")) # This only exists to make exporting easier model.pdrop_value = kwargs.get('dropout', 0.5) # This only exists to make exporting easier with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): seed = np.random.randint(10e8) init = tf.random_uniform_initializer(-0.05, 0.05, dtype=tf.float32, seed=seed) word_embeddings = model.embed(**kwargs) input_sz = word_embeddings.shape[-1] pooled = model.pool(word_embeddings, input_sz, init, **kwargs) stacked = model.stacked(pooled, init, **kwargs) # For fully connected layers, use xavier (glorot) transform with tf.variable_scope("output"): model.logits = tf.identity(tf.layers.dense( stacked, nc, activation=None, kernel_initializer=tf.glorot_uniform_initializer(seed)), name="logits") model.best = tf.argmax(model.logits, 1, name="best") model.probs = tf.nn.softmax(model.logits, name="probs") model.sess = sess # writer = tf.summary.FileWriter('blah', sess.graph) return model
def fit(model_params, ts, vs, es, **kwargs): """ Train a classifier using TensorFlow with a `feed_dict`. This is the previous default behavior for training. To use this, you need to pass `fit_func: feed_dict` in your MEAD config :param model_params: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 :return: None """ epochs = int(kwargs.get('epochs', 5)) patience = int(kwargs.get('patience', epochs)) conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = bool(kwargs.get('verbose', False)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: trainer.recover_last_checkpoint() # What to do about overloading this?? evaluator = TaggerEvaluatorTf(trainer.model, span_type, verbose) timer = Timer() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = timer.elapsed() for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration})
def fit_datasets(model_params, ts, vs, es=None, **kwargs): """ Train a tagger using TensorFlow with `tf.dataset`. This is the default behavior for training. :param model_params: The model (or parameters to create the model) to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 :return: None """ conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)}) epochs = int(kwargs.get('epochs', 20)) batchsz = kwargs['batchsz'] ## First, make tf.datasets for ts, vs and es # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md # effective_batch_sz = args.batchsz*args.gpus test_batchsz = kwargs.get('test_batchsz', batchsz) # This is a little awkward: lengths_key = model_params.get('lengths_key') train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts, lengths_key)) train_dataset = train_dataset.shuffle(buffer_size=SHUF_BUF_SZ) train_dataset = train_dataset.batch(batchsz, drop_remainder=False) train_dataset = train_dataset.repeat(epochs + 1) train_dataset = train_dataset.prefetch(NUM_PREFETCH) valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs, lengths_key)) valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False) valid_dataset = valid_dataset.repeat(epochs + 1) valid_dataset = valid_dataset.prefetch(NUM_PREFETCH) iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset), tf.compat.v1.data.get_output_shapes(train_dataset)) features, y = iter.get_next() # Add features to the model params model_params.update(features) model_params['y'] = y # create the initialisation operations train_init_op = iter.make_initializer(train_dataset) valid_init_op = iter.make_initializer(valid_dataset) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.sess.run(train_init_op) trainer.train(ts, reporting_fns) trainer.sess.run(valid_init_op) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: print('Reloading best checkpoint') trainer.recover_last_checkpoint() test_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(es, lengths_key)) test_dataset = test_dataset.batch(test_batchsz, drop_remainder=False) test_dataset = test_dataset.repeat(epochs + 1) test_dataset = test_dataset.prefetch(NUM_PREFETCH) test_init_op = iter.make_initializer(test_dataset) trainer.sess.run(test_init_op) # What to do about overloading this?? evaluator = TaggerEvaluatorTf(trainer.model, span_type, verbose) start = time.time() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = time.time() - start for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration})
def fit_datasets(model_params, ts, vs, es=None, **kwargs): """ Train an language model using TensorFlow with `tf.dataset`. This is the default behavior for training. :param model_params: The model (or parameters to create the model) to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 :return: None """ epochs = int(kwargs.get('epochs', 5)) patience = int(kwargs.get('patience', epochs)) model_file = get_model_file('lm', 'tf', kwargs.get('basedir')) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) batchsz = kwargs['batchsz'] test_batchsz = kwargs.get('test_batchsz', batchsz) tgt_key = model_params.get('tgt_key') train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts)) train_dataset = train_dataset.shuffle(buffer_size=SHUF_BUF_SZ) train_dataset = train_dataset.batch(batchsz, drop_remainder=False) train_dataset = train_dataset.repeat(epochs + 1) train_dataset = train_dataset.prefetch(NUM_PREFETCH) valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs)) valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False) valid_dataset = valid_dataset.repeat(epochs + 1) valid_dataset = valid_dataset.prefetch(NUM_PREFETCH) iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset), tf.compat.v1.data.get_output_shapes(train_dataset)) features, tgt = iter.get_next() # Add features to the model params model_params.update(features) model_params.update({'y': tgt}) # create the initialization operations train_init_op = iter.make_initializer(train_dataset) valid_init_op = iter.make_initializer(valid_dataset) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.sess.run(train_init_op) trainer.train(ts, reporting_fns) trainer.sess.run(valid_init_op) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: print('Reloading best checkpoint') trainer.recover_last_checkpoint() test_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(es)) test_dataset = test_dataset.batch(test_batchsz, drop_remainder=False) test_dataset = test_dataset.repeat(epochs + 1) test_dataset = test_dataset.prefetch(NUM_PREFETCH) test_init_op = iter.make_initializer(test_dataset) trainer.sess.run(test_init_op) trainer.test(es, reporting_fns, phase='Test')