def test_get_optimizer(self): sm_1 = SequenceModel() opt_1 = sm_1.get_optimizer(name="adam", learning_rate=0.2) self.assertEqual(type(opt_1).__name__, "AdamOptimizer") opt_2 = sm_1.get_optimizer(name="adadelta", learning_rate=0.3) self.assertEqual(type(opt_2).__name__, "AdadeltaOptimizer") with self.assertRaises(Exception) as context: _ = sm_1.get_optimizer(name="unknown") self.assertTrue("unknown" in str(context.exception))
def predict(self, dataset): """ Predict the output from input. Args: dataset: the input dataset Returns: predictions array """ output = self.model[-1] output = tf.Print(output, [tf.shape(output)], summarize=4) # Initialize variables init_g = tf.global_variables_initializer() init_l = tf.local_variables_initializer() # Initialize states initial_state_pred = np.zeros(shape=(len(dataset), self.units_per_cell), dtype=np.float32) # load validation set input_pred, label_pred = self.load_batch(dataset) with tf.Session() as sess: sess.run(init_g) sess.run(init_l) # Load existing model SequenceModel.load(self, sess) print(sess.run(self.global_step)) prediction = sess.run( [output], feed_dict={ self.input: input_pred, self.label: label_pred, self.initial_state: initial_state_pred }) return prediction[0]
def fit(self, training_set, validation_set): """ Fit the model Args: training_set: validation_set: Returns: """ outputs = self.model outputs[-1] = tf.Print(outputs[-1], [self.label], summarize=self.batch_size * self.n_output, message="Label: ") outputs[-1] = tf.Print(outputs[-1], [outputs[-1]], summarize=self.batch_size * self.n_output, message="Prediction: ") metrics = SequenceModel.compute_metrics( outputs[-1], self.label, self.time_steps if self.is_sequence_output else 1) loss = SequenceModel.compute_loss( outputs[-1], self.label, self.loss_name, self.time_steps if self.is_sequence_output else 1) train_op = SequenceModel.compute_gradient(self, loss, self.global_step) # Merge summaries summaries = tf.summary.merge_all() # Initialize variables init_g = tf.global_variables_initializer() init_l = tf.local_variables_initializer() self.global_step = tf.add(self.global_step, tf.constant(1)) with tf.Session() as sess: run_opts = tf.RunOptions(report_tensor_allocations_upon_oom=True) sess.run(init_g) sess.run(init_l) self.train_writer.add_graph(sess.graph) # Load existing model SequenceModel.load(self, sess) if self.from_pretrained else None for epoch in range(self.epochs): for i in range(self.batch_size, len(training_set), self.batch_size): time0 = time() batch_input, batch_label = self.load_batch( training_set[i - self.batch_size:i]) initial_state = np.zeros(shape=(self.batch_size, self.units_per_cell), dtype=np.float32) _, loss_value, summaries_value, step = sess.run( [train_op, loss, summaries, self.global_step], feed_dict={ self.input: batch_input, self.label: batch_label, self.initial_state: initial_state }, options=run_opts, ) self.train_writer.add_summary(summaries_value, step) time1 = time() self.logger.info( "Cost = {0} for batch {1} in {2:.2f} seconds".format( loss_value, i / self.batch_size, time1 - time0)) if self.logger else None if i % self.validation_step == 0: SequenceModel.memory() self.validation_eval(sess, summaries, validation_set, metrics, step) gc.collect() SequenceModel.memory() if i % self.checkpoint_step == 0: SequenceModel.save(self, sess, step=self.global_step)
def __init__(self, units_per_cell, batch_size, time_steps, n_features, n_layers=1, is_sequence_output=True, with_prev_output=True, n_output=1, f_out="identity", optimizer_name="rmsprop", learning_rate=0.1, loss_name="mse", epochs=1, from_pretrained=False, validation_step=1000, checkpoint_step=1000, summary_path="", checkpoint_path="", name="rnn", debug=0): """ Initialization of RNN model Args: units_per_cell: the number of units per RNN Cell batch_size: the size of batch time_steps: the length of sequence n_features: the number of features in the sequence n_layers: the number of layers stacked is_sequence_output: whether the model outputs a sequence with_prev_output: whether the model uses the previous cell output n_output: the output dimension f_out: the activation function used as output of cell optimizer_name: the optimizer name used learning_rate: the learning rate loss_name: the name of the metric to use for optimization epochs: the number of epochs for training from_pretrained: whether to use the pre trained model validation_step: model evaluation frequency checkpoint_step: model checkpoint frequency summary_path: the path to the summary folder checkpoint_path: the path to the checkpoint path name: the name of the variable scope debug: whether debug mode is activate """ tf.reset_default_graph() SequenceModel.__init__(self, name, debug) self.units_per_cell = units_per_cell self.n_layers = n_layers self.n_cells = time_steps # the number of RNN cells is equivalent to the length of the sequence self.batch_size = batch_size self.time_steps = time_steps self.n_features = n_features self.is_sequence_output = is_sequence_output self.with_prev_output = with_prev_output self.n_output = n_output self.loss_name = loss_name self.epochs = epochs self.from_pretrained = from_pretrained self.validation_step = validation_step self.checkpoint_step = checkpoint_step self.summary_path = summary_path self.checkpoint_path = checkpoint_path self.input_keep_prob = 1 self.state_keep_prob = 1 self.output_keep_prob = 1 self.seed = 42 self.input = tf.placeholder(tf.float32, [None, self.time_steps, self.n_features]) self.initial_state = tf.placeholder(tf.float32, [None, self.units_per_cell]) if self.is_sequence_output: self.label = tf.placeholder(tf.float32, [None, self.time_steps, self.n_output]) else: self.label = tf.placeholder(tf.float32, [None, self.n_output]) # Global step self.global_step = tf.Variable(0, dtype=tf.int32, name="global_step") # Optimizer self.optimizer = self.get_optimizer(optimizer_name, learning_rate) # Output activation self.f_out = self.get_activation(f_out) # Summary writers self.train_writer, self.validation_writer = self.get_writer() # Model saver self.saver = tf.train.Saver() # Build model self.model = self.build_model(name)
def fit(self, training_set, validation_set, stop_at_step=None): """ Fit the model. Args: training_set: set of data for training validation_set: set of data for evaluation stop_at_step: step from which to stop """ outputs = self.model outputs[-1] = tf.Print(outputs[-1], [self.label], summarize=self.batch_size * self.n_output, message="Label: ") outputs[-1] = tf.Print(outputs[-1], [outputs[-1]], summarize=self.batch_size * self.n_output, message="Prediction: ") metrics = SequenceModel.compute_metrics( outputs[-1], self.label, self.time_steps if self.is_sequence_output else 1) loss = SequenceModel.compute_loss( outputs[-1], self.label, self.loss_name, self.time_steps if self.is_sequence_output else 1) train_op = SequenceModel.compute_gradient(self, loss, self.global_step) self.global_step = tf.add(self.global_step, tf.constant(1)) # Merge summaries summaries = tf.summary.merge_all() # Initialize variables init_g = tf.global_variables_initializer() init_l = tf.local_variables_initializer() # Initialize states initial_state = np.zeros(shape=(self.batch_size, self.units_per_cell), dtype=np.float32) initial_state_val = np.zeros(shape=(len(validation_set), self.units_per_cell), dtype=np.float32) # load validation set input_val, label_val = self.load_batch(validation_set) with tf.Session() as sess: run_opts = tf.RunOptions(report_tensor_allocations_upon_oom=True) sess.run(init_g) sess.run(init_l) self.train_writer.add_graph(sess.graph) # Load existing model SequenceModel.load(self, sess) if self.from_pretrained else None for epoch in range(self.epochs): for i in range(self.batch_size, len(training_set), self.batch_size): time0 = time() batch_input, batch_label = self.load_batch( training_set[i - self.batch_size:i]) _, loss_value, summaries_value, step = sess.run( [train_op, loss, summaries, self.global_step], feed_dict={ self.input: batch_input, self.label: batch_label, self.initial_state: initial_state }, options=run_opts, ) self.train_writer.add_summary(summaries_value, step) time1 = time() self.logger.info( "Cost = {0} for batch {1} in {2:.2f} seconds".format( loss_value, i / self.batch_size, time1 - time0)) if self.logger else None if i % self.validation_step == 0: if i % self.validation_step == 0: self.validation_eval(sess, summaries, input_val, label_val, initial_state_val, metrics, step) if i % self.checkpoint_step == 0: # SequenceModel.save(self, sess, step=self.global_step) import os checkpoint_path = os.path.join(self.checkpoint_path, self.name) self.saver.save(sess, checkpoint_path, global_step=step) if stop_at_step and step >= stop_at_step: break predictions = sess.run( [outputs[-1]], feed_dict={ self.input: input_val, self.label: label_val, self.initial_state: initial_state_val }) np.save("predictions.npy", predictions[0])