def _run_train_step_DBM(self, train_set, hidden_data): """ Run a training step. A training step is made by randomly shuffling the training set, divide into batches and run the variable update nodes for each batch. :param train_set: training set :return: self """ np.random.shuffle(train_set) batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)] batches_2 = [ _ for _ in utils.gen_batches(hidden_data, self.batch_size) ] updates = [ self.w_1_upd8_3, self.w_2_upd8_3, self.bh_1_upd8_3, self.bh_2_upd8_3, self.bv_1_upd8_3, self.bv_2_upd8_3 ] for batch_1, batch_2 in zip(batches, batches_2): upd = self.tf_session.run(updates, feed_dict=self._create_feed_dict( batch_1, True, pos='dbm', status='train', hidden_data=batch_2))
def train(self, training_x, validation_x): sess = tf.Session() sess.run(tf.global_variables_initializer()) corruption_ratio = np.round(self.corruption_fraction * training_x.shape[1]).astype(np.int) for epoch in range(self.epochs): print("Epoch: {}".format(epoch)) x_corrupted = self.corrupt_input(training_x, corruption_ratio) shuff = list(zip(training_x, x_corrupted)) np.random.shuffle(shuff) batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)] for batch in batches: x_batch, x_corrupted_batch = zip(*batch) sess.run(self.train_step, feed_dict={self.input_data: x_batch, self.corrupted_input_data: x_corrupted_batch}) if epoch % 5 == 0: if validation_x is not None: error = sess.run(self.cost, feed_dict={self.input_data: validation_x, self.corrupted_input_data: validation_x}) print("Validation cost is {}".format(error)) if validation_x is not None: error = sess.run(self.cost, feed_dict={self.input_data: validation_x, self.corrupted_input_data: validation_x}) print("Validation cost is {}".format(error))
def main(_): ## # Load Data ## with open(FLAGS.data_path, 'r') as f: reader = csv.reader(f) # data is a list of tuples (img path, steering angle) data = np.array([row for row in reader]) # Split train and validation data np.random.shuffle(data) split_i = int(len(data) * 0.9) X_train, y_train = list(zip(*data[:split_i])) X_val, y_val = list(zip(*data[split_i:])) X_train, y_train = np.array(X_train), np.array(y_train) X_val, y_val = np.array(X_val), np.array(y_val) gen = gen_batches(X_train, y_train, 10) imgs, labels = gen.__next__() print(imgs[0]) for i, img in enumerate(imgs): imsave('test/' + str(i) + '.png', np.squeeze(img))
def _run_train_step(self, train_set, corruption_ratio, corrupted_train_set): ''' Run a training step. A training step is made by randomly corrupting the training set, randomly shuffling it, and dividing it into batches and running the optimizer for each batch. Parameters ---------- train_set : training set corruption_ratio : fraction of elements to corrupt Returns ------- self ''' if self.corr_type == 'id': x_corrupted = corrupted_train_set else: x_corrupted = self._corrupt_input(train_set, corruption_ratio) shuff = zip(train_set, x_corrupted) np.random.shuffle(shuff) batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)] for idx, batch in enumerate(batches): x_batch, x_corr_batch = zip(*batch) tr_feed = {self.input_data: x_batch, self.input_data_corr: x_corr_batch} self.tf_session.run(self.train_step, feed_dict=tr_feed)
def _run_train_step(self, train_set, validation_set, pos='first'): """ Run a training step. A training step is made by randomly shuffling the training set, divide into batches and run the variable update nodes for each batch. :param train_set: training set :return: self """ np.random.shuffle(train_set) batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)] if pos == 'first': updates = [ self.w_upd8_1, self.bh_upd8_1, self.bv_upd8_1, self.last_s_1 ] else: updates = [ self.w_upd8_2, self.bh_upd8_2, self.bv_upd8_2, self.last_s_2 ] i = 0 start = True for batch in batches: upd = self.tf_session.run(updates, feed_dict=self._create_feed_dict( batch, start, pos)) ### feed to next batch self.last_s = upd[3] start = False
def eval(self, val_tup, verbose=True, write_summary=False): """ Evaluates the model on given data. Writes out a validation loss summary. :param val_tup: The data to evaluate the model on; tuple of (images, labels, feedback) :param verbose: Whether to print the error :param write_summary: whether to write a summary at current timestep to disk (for training) """ if verbose: print("validating...") error = 0.0 loss = 0.0 #note, if you are not interested in the loss (but only the error metric), you can set only_positive=True for faster results for imgs, labels, feedback in utils.gen_batches( val_tup, shuffle=False, only_positive=False, batch_sz=c.EVAL_BATCH_SIZE): processed_imgs = self._process_images(imgs) sess_args = [self.abs_err, self.loss] feed_dict = { self.img_input: processed_imgs, self.labels: labels, self.feedback: feedback } batch_abs_err, batch_loss = self.sess.run(sess_args, feed_dict=feed_dict) error += len( imgs ) * batch_abs_err #batches can be different lengths, so weight them loss += len(imgs) * batch_loss error /= len(val_tup[0]) loss /= len(val_tup[0]) if write_summary: #make sumary mannually becuase its too large for one batch step = self.sess.run(self.global_step) #error error_summary = tf.Summary(value=[ tf.Summary.Value(tag='val_error' + c.TRIAL_STR, simple_value=error) ]) self.summary_writer.add_summary(error_summary, global_step=step) #loss loss_summary = tf.Summary(value=[ tf.Summary.Value(tag='val_loss' + c.TRIAL_STR, simple_value=loss) ]) self.summary_writer.add_summary(loss_summary, global_step=step) if verbose: print("") print("Validation Error:", error)
def _run_train_step(self, train_set): """ Run a training step. A training step is made by randomly shuffling the training set, divide into batches and run the variable update nodes for each batch. :param train_set: training set :return: self """ np.random.shuffle(train_set) batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)] updates = [self.w_upd8, self.bh_upd8, self.bv_upd8] for batch in batches: self.tf_session.run(updates, feed_dict=self._create_feed_dict(batch))
def train(self, trX, valX, restore_previous_model=False): """ Train function, take the input train and validation set, fit it to the model and output the RMSE score :param trX: Training set :param valX: Validation set :param restore_previous_model: Do we restore the pre-trained model or not? """ ops.reset_default_graph() self._create_graph() merged = tf.summary.merge_all() init_op = tf.initialize_all_variables() with tf.Session() as self.sess: self.sess.run(init_op) if restore_previous_model: self.saver.restore(self.sess, self.models_dir + self.model_name) self.model_name += '-restored{}'.format(self.N_EPOCH) writer = tf.summary.FileWriter(self.summary_dir, self.sess.graph_def) for epoch in range(self.N_EPOCH): np.random.shuffle(trX) batches = [_ for _ in utils.gen_batches(trX, self.BATCH_SIZE)] total_error = 0.0 for batch in batches: result = self.sess.run([self.run_update, self.cost], feed_dict={self.X: batch}) total_error += result[1] print("Cost at epoch %s of training: %s" % (epoch, total_error / len(batches))) # if epoch % 5 == 0: # result = self.sess.run([merged, self.cost], feed_dict={self.X: valX}) # summary_str = result[0] # err = result[1] # # writer.add_summary(summary_str, 1) # print("Cost at epoch %s on validation set: %s" % (epoch, err)) self.saver.save(self.sess, self.models_dir + self.model_name)
def train(self, train_tup, val_tup): """ Training loop. Trains & validates for the given number of epochs on given data. :param train_tup: All the training data; tuple of (images, labels, feedback) :param val_tup: All the validation data; tuple of (images, labels, feedback) """ #caclulate number of steps steps_per_epc = int(np.ceil(len(train_tup[0])/c.BATCH_SIZE)) num_steps = c.NUM_EPOCHS * steps_per_epc initial_step = self.sess.run(self.global_step) print(num_steps, "steps total in this train session...") #start training self.eval(val_tup, write_summary=True) #eval always at start for i in range(c.NUM_EPOCHS): print("\nEpoch", i+1, "out of", c.NUM_EPOCHS) for imgs, labels, feedback in utils.gen_batches(train_tup, shuffle=True): self._train_step(imgs, labels, feedback, initial_step, num_steps, val_tup)
def _train_model(self, train_X, validation_X): """ This function perform the training process """ feed_train = { self.input_data: train_X, self.keep_prob: self.finetune_dropout } feed_validation = {self.input_data: validation_X, self.keep_prob: 1} shuff = train_X old_iter, old_loss = 0, 10000 for iteration in range(1, self.finetune_num_epochs + 1): np.random.shuffle(shuff) batches = [_ for _ in gen_batches(shuff, self.finetune_batch_size)] for batch in batches: feed_batch = { self.input_data: batch, self.keep_prob: self.finetune_dropout } self.sess.run(self.train_step, feed_dict=feed_batch) print('Iter %d' % iteration, end=': ') loss_train = self.sess.run(self.reconstruction_loss, feed_dict=feed_train) if validation_X is not None: loss_validation = self.sess.run(self.reconstruction_loss, feed_dict=feed_validation) reconstructed_error = self.sess.run(self.reconstructed_error, feed_dict=feed_validation) self.loss_summary.append(loss_validation) print('Training set: current loss %f' % loss_train, end=' || ') print('Validation set: current loss %f' % loss_validation) print('Validation: ', np.argsort(-reconstructed_error)[:25]) if loss_validation > old_loss and (iteration - old_iter) > 5: return if loss_validation < old_loss: old_loss = loss_validation old_iter = iteration else: print('Training set: current loss %f' % loss_train)
def _train_model(self, train_X, train_y, validation_X, validation_y): """ This function perform the training process """ feed_train = { self.input_data: train_X, self.input_labels: train_y, self.keep_prob: self.finetune_dropout } feed_validation = { self.input_data: validation_X, self.input_labels: validation_y, self.keep_prob: 1 } shuff = list(zip(train_X, train_y)) for iteration in range(1, self.finetune_num_epochs + 1): np.random.shuffle(shuff) batches = [_ for _ in gen_batches(shuff, self.finetune_batch_size)] for batch in batches: X_batch, y_batch = zip(*batch) feed_batch = { self.input_data: X_batch, self.input_labels: y_batch, self.keep_prob: self.finetune_dropout } self.sess.run(self.train_step, feed_dict=feed_batch) if validation_X is not None: updates = [self.loss, self.accuracy] loss_train, accuracy_train = self.sess.run( updates, feed_dict=feed_train) print('Iter: %d' % iteration) print('....Training: current loss %f' % loss_train) print('....Training: current accuracy %f' % accuracy_train) loss_val, accuracy_val = self.sess.run( updates, feed_dict=feed_validation) print('****Validation: current loss %f' % loss_val) print('****Validation: current accuracy %f' % accuracy_val)
def _run_train_step(self, train_set): """ Run a training step. A training step is made by randomly shuffling the training set, divide into batches and run the variable update nodes for each batch. If self.plot_training_loss is true, will record training loss after each batch. :param train_set: training set :return: self """ np.random.shuffle(train_set) batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)] updates = [self.w_upd8, self.bh_upd8, self.bv_upd8] for batch in batches: if self.plot_training_loss: _, loss = self.tf_session.run([updates, self.loss_function], feed_dict=self._create_feed_dict(batch)) self.training_losses.append(loss) else: self.tf_session.run(updates, feed_dict=self._create_feed_dict(batch))
def _run_train_step(self, train_set, corruption_ratio): """ Run a training step. A training step is made by randomly corrupting the training set, randomly shuffling it, divide it into batches and run the optimizer for each batch. :param train_set: training set :param corruption_ratio: fraction of elements to corrupt :return: self """ x_corrupted = self._corrupt_input(train_set, corruption_ratio) shuff = zip(train_set, x_corrupted) np.random.shuffle(shuff) batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)] for batch in batches: x_batch, x_corr_batch = zip(*batch) tr_feed = { self.input_data: x_batch, self.input_data_corr: x_corr_batch } self.tf_session.run(self.train_step, feed_dict=tr_feed)
def eval(self, val_tup, verbose=True, write_summary=False, evaluate_angle=False): """ Evaluates the model on given data. Writes out a validation loss summary. :param val_tup: The data to evaluate the model on; tuple of (images, labels, feedback) :param verbose: Whether to print. E.g. the error (and the predicted angles if evaluate_angle) :param evaluate_angle: Wether to evaluate the anlge (implied by our net), as opposed to the feedback predicted :param write_summary: whether to write a summary at current timestep to disk (for training) """ if verbose: print("validating...") #evalualt# error = 0.0 for imgs, labels, feedback in utils.gen_batches(val_tup, shuffle=True, only_positive=evaluate_angle, batch_sz=c.EVAL_BATCH_SIZE): processed_imgs = self._process_images(imgs) if evaluate_angle: batch_abs_err = self._eval_angle_on_batch(processed_imgs, labels, verbose) else: sess_args = self.abs_err feed_dict = {self.img_input: processed_imgs, self.labels: labels, self.feedback: feedback} batch_abs_err = self.sess.run(sess_args, feed_dict=feed_dict) error += len(imgs) * batch_abs_err #batches can be different lengths, so weight them error /= len(val_tup[0]) ## if write_summary: #make sumary mannually becuase its too large for one batch step = self.sess.run(self.global_step) error_summary = tf.Summary(value=[tf.Summary.Value(tag='val_feedback_error'+c.TRIAL_STR, simple_value=error)]) self.summary_writer.add_summary(error_summary, global_step=step) if verbose: print("") string = "ANGLE" if evaluate_angle else "FEEDBACK" print("Validation Error [" + string + " predicition]:", error)
def main(_): # Main model-building process print('Starting data load...') print(time.strftime('%H:%M:%S')) print() start_time = time.time() ## # Load Data ## ## # Will have centre_count main driving files - each of which are frames taken when driving centrally # Naming driving_log_centre_X.csv where X is sequential numbering # Will have left_recover_count files - each of which represents sets of frames taken when recovering from the left # When processing these files will just take +ve steering angle imags, ignoring all others # Will have right_recover_count files - each of which represents set of frames taken when recovering from the right # When processing these files will just take -ve steering angle images, ignoring all others # Note that both the left_ and right_ recovery files will have some normal driving frames with +ve/-ve steering angles # and these will just add to the set of data ## # Initialise variables X_train = [] # our training data image paths y_train = [] # our training data labels n_min_y = 0.0 # will use these two variables to track the min/max steering angles found in the data n_max_y = 0.0 # Read in the centre images data for file_num in range(1, FLAGS.centre_count + 1): sFilename = FLAGS.data_file + '_centre_' + str(file_num) + '.csv' with open(sFilename, 'r') as f: reader = csv.reader(f) # data is a list of tuples (img path, steering angle) data = np.array([row for row in reader]) # Provided data has form of: # data[i][0] = centre image # data[i][1] = left image # data[i][2] = right image # data[i][3] = steering angle # data[i][4] = throttle # data[i][5] = brake # data[i][6] = speed # Parse out the centre camera view and steering angle - this is our training input and label for i in range(len(data)): if data[i][0] != 'center': # ignore the header row # sort out the image path, if using the Udacity provided data s = str(data[i][0]) # get the filename from the csv data s = s[s.rfind('/') + 1:] # trim off any paths s = FLAGS.imgs_dir + s # put back the path to the images location X_train.append(s) y_label = float(data[i][3]) y_train.append(y_label) if y_label < n_min_y: n_min_y = y_label if y_label > n_max_y: n_max_y = y_label print('Imported ', len(X_train), ' centre frames') print('Total frames so far :', len(X_train)) print('Range of label values : ', n_min_y, ' to ', n_max_y) print() n_centre_frames = len(X_train) # Read in the left-of-centre recovery images data for file_num in range(1, FLAGS.left_recover_count + 1): sFilename = FLAGS.data_file + '_left_' + str(file_num) + '.csv' with open(sFilename, 'r') as f: reader = csv.reader(f) # data is a list of tuples (img path, steering angle) data = np.array([row for row in reader]) for i in range(len(data)): if data[i][0] != 'center': # ignore the header row, if there is one if float( data[i] [3]) > 0.0: # only interested in +ve steering angle frames # sort out the image path, if using the Udacity provided data s = str(data[i][0]) # get the filename from the csv data s = s[s.rfind('/') + 1:] # trim off any paths s = FLAGS.imgs_dir + s # put back the path to the images location X_train.append(s) y_label = float(data[i][3]) y_train.append(y_label) if y_label < n_min_y: n_min_y = y_label if y_label > n_max_y: n_max_y = y_label n_left_frames = len(X_train) - n_centre_frames print('Imported ', n_left_frames, ' left frames') print('Total frames so far :', len(X_train)) print('Range of label values : ', n_min_y, ' to ', n_max_y) print() # Read in the right-of-centre recovery images data for file_num in range(1, FLAGS.right_recover_count + 1): sFilename = FLAGS.data_file + '_right_' + str(file_num) + '.csv' with open(sFilename, 'r') as f: reader = csv.reader(f) # data is a list of tuples (img path, steering angle) data = np.array([row for row in reader]) for i in range(len(data)): if data[i][0] != 'center': # ignore the header row, if there is one if float( data[i] [3]) < 0.0: # only interested in -ve steering angle frames # sort out the image path, different if local or if using the Udacity provided data s = str(data[i][0]) # get the filename from the csv data s = s[s.rfind('/') + 1:] # trim off any paths s = FLAGS.imgs_dir + s # put back the path to the images location X_train.append(s) y_label = float(data[i][3]) y_train.append(y_label) if y_label < n_min_y: n_min_y = y_label if y_label > n_max_y: n_max_y = y_label n_right_frames = len(X_train) - n_left_frames - n_centre_frames print('Imported ', n_right_frames, ' right frames') print('Total frames so far :', len(X_train)) print('Range of label values : ', n_min_y, ' to ', n_max_y) print() print('Total frames : ', len(X_train)) print() # Split into training and validation data X_train, y_train, X_valid, y_valid = train_validation_split( X_train, y_train, training_percent) X_train = np.array(X_train) y_train = np.array(y_train) X_valid = np.array(X_valid) y_valid = np.array(y_valid) print('Split training/validation sets (', training_percent, '/', (1.0 - training_percent), ')') print('Training set size : ', len(X_train)) print('Validation set size : ', len(X_valid)) print() ## # Define Model ## print('Creating model...') model = Sequential([ Conv2D(32, 3, 3, input_shape=(32, 16, 1), border_mode='same', activation='relu'), Conv2D(64, 3, 3, border_mode='same', activation='relu'), Dropout(0.5), Conv2D(128, 3, 3, border_mode='same', activation='relu'), Conv2D(256, 3, 3, border_mode='same', activation='relu'), Dropout(0.5), Flatten(), Dense(1024, activation='relu'), Dense(512, activation='relu'), Dense(128, activation='relu'), Dense(1, name='output', activation='tanh'), ]) model.compile(optimizer=Adam(lr=FLAGS.lrate), loss='mse') model.summary() print('Model created') print() ## # Train ## # When using a generator, can say how may samples to use per epoch # The generator will be creating random batches, so this value can be other than the total samples value... n_samples_per_epoch = len(X_train) #* 2 n_validation_size = len(X_valid) #* 2 history = model.fit_generator(gen_batches(X_train, y_train, FLAGS.batch_size), n_samples_per_epoch, FLAGS.num_epochs, validation_data=gen_batches( X_valid, y_valid, FLAGS.batch_size), nb_val_samples=n_validation_size) ## # Save model ## json = model.to_json() model.save_weights('save/model.h5') with open('save/model.json', 'w') as f: f.write(json) end_time = time.time() print() print('Training time : ', (end_time - start_time) / 60, ' minutes')
def _fit_stochastic(self, X, y, activations, deltas, coef_grads, intercept_grads, layer_units): params = self.coefs_ + self.intercepts_ if self.solver == 'sgd': self._optimizer = SGDOptimizer( params, self.learning_rate, self.momentum, self.nesterovs_momentum) elif self.solver == 'adam': self._optimizer = AdamOptimizer( params, self.learning_rate, self.beta_1, self.beta_2, self.epsilon) # early_stopping early_stopping = self.early_stopping if self.evaluation: X_val = self._X_eval y_val = self._y_eval else: X_val = None y_val = None # number of samples n_samples = X.shape[0] # batch size batch_size = np.clip(self.batch_size, 1, n_samples) # training process try: # every epoch for it in range(self.epoches): # loss accumulated_loss = 0.0 # every batches for batch_slice in gen_batches(n_samples, batch_size): activations[0] = X[batch_slice] # backpropogation batch_loss, coef_grads, intercept_grads = self._backprop( X[batch_slice], y[batch_slice], activations, deltas, coef_grads, intercept_grads) # loss accumulated_loss += batch_loss * (batch_slice.stop - batch_slice.start) # update weights grads = coef_grads + intercept_grads self._optimizer.update_params(grads) self.n_iter_ += 1 self.loss_ = accumulated_loss / X.shape[0] # append loss self.loss_curve_.append(self.loss_) # append training acore self.train_scores_.append(self.score(X, np.argmax(y, axis = 1))) if self.evaluation == False: if self.verbose: print("Iteration %d, loss = %.8f Training score: %f" % (self.n_iter_,self.loss_, self.train_scores_[-1])) else: # compute validation score, use that for stopping self.validation_scores_.append(self.score(X_val, y_val)) if self.verbose: print("Iteration %d, loss = %.8f Training score: %f Validation score: %f" % (self.n_iter_,self.loss_, self.train_scores_[-1], self.validation_scores_[-1])) # update no_improvement_count based on training loss or # validation score according to early_stopping self._update_no_improvement_count(early_stopping, X_val, y_val) if self._no_improvement_count > self.n_iter_no_change: # not better than last `n_iter_no_change` iterations by tol # stop or decrease learning rate if early_stopping: msg = ("Early stop. Validation score did not improve more than " "tol=%f for %d consecutive epochs." % ( self.tol, self.n_iter_no_change)) print(msg) break self._no_improvement_count = 0 except KeyboardInterrupt: warnings.warn("Training interrupted by user.") # restore best weights self.coefs_ = self._best_coefs self.intercepts_ = self._best_intercepts
def fit(self, trX, vlX=None, restore_previous_model=False): if vlX is not None: self.validation_size = vlX.shape[0] # Reset tensorflow's default graph ops.reset_default_graph() self._create_graph() merged = tf.merge_all_summaries() init_op = tf.initialize_all_variables() self.saver = tf.train.Saver() with tf.Session() as self.sess: self.sess.run(init_op) if restore_previous_model: # Restore previous model self.saver.restore(self.sess, self.models_dir + self.model_name) # Change model name self.model_name += '-restored{}'.format(self.n_iter) # Write tensorflow summaries to summary dir writer = tf.train.SummaryWriter(self.summary_dir, self.sess.graph_def) for i in range(self.n_iter): # Randomly shuffle the input np.random.shuffle(trX) batches = [_ for _ in utils.gen_batches(trX, self.batch_size)] for batch in batches: self.sess.run( [self.w_upd8, self.bh_upd8, self.bv_upd8], feed_dict={ self.x: batch, self.hrand: np.random.rand(batch.shape[0], self.nhid), self.vrand: np.random.rand(batch.shape[0], self.nvis) }) if i % 5 == 0: # Record summary data if vlX is not None: feed = { self.x: vlX, self.hrand: np.random.rand(self.validation_size, self.nhid), self.vrand: np.random.rand(self.validation_size, self.nvis) } result = self.sess.run([merged, self.cost], feed_dict=feed) summary_str = result[0] err = result[1] writer.add_summary(summary_str, 1) if self.verbose == 1: print("Validation cost at step %s: %s" % (i, err)) # Save trained model self.saver.save(self.sess, self.models_dir + self.model_name)
def fit(self, trX, vlX=None, restore_previous_model=False): """ Fit the model to the data. :type trX: array_like, shape (n_samples, n_features). :param trX: Training data. :type vlX: array_like, shape (n_validation_samples, n_features). :param vlX: optional, default None. Validation data. :return: self """ n_features = trX.shape[1] self._create_graph(n_features) # Merge all the summaries merged = tf.merge_all_summaries() # Initialize variables init_op = tf.initialize_all_variables() # Add ops to save and restore all the variables self.saver = tf.train.Saver() with tf.Session() as self.sess: self.sess.run(init_op) if restore_previous_model: # Restore previous model self.saver.restore(self.sess, self.models_dir + self.model_name) # Change model name self.model_name += '-restored{}'.format(self.n_iter) # ################## # # Training phase # # ################## # v = np.round(self.corr_frac * n_features).astype(np.int) # Write the summaries to summary_dir writer = tf.train.SummaryWriter(self.summary_dir, self.sess.graph_def) for i in range(self.n_iter): # #################### # # Input Corruption # # #################### # if self.corr_type == 'masking': x_corrupted = utils.masking_noise(trX, v) elif self.corr_type == 'salt_and_pepper': x_corrupted = utils.salt_and_pepper_noise(trX, v) else: # none, normal autoencoder x_corrupted = trX # Randomly shuffle the input shuff = zip(trX, x_corrupted) np.random.shuffle(shuff) # # Divide dataset into mini-batches batches = [ _ for _ in utils.gen_batches(shuff, self.batch_size) ] # All the batches for each epoch for batch in batches: x_batch, x_corr_batch = zip(*batch) tr_feed = { self.x: x_batch, self.x_corr: x_corr_batch, self.keep_prob: self.dropout } self.sess.run(self.train_step, feed_dict=tr_feed) # Record summary data if vlX is not None: vl_feed = { self.x: vlX, self.x_corr: vlX, self.keep_prob: 1. } result = self.sess.run([merged, self.cost], feed_dict=vl_feed) summary_str = result[0] err = result[1] writer.add_summary(summary_str, i) if self.verbose == 1: print("Validation cost at step %s: %s" % (i, err)) # Save trained model self.saver.save(self.sess, self.models_dir + self.model_name)
EPOCHES = 200 SAMPLE_LEN = 64 modi_adj = loadADJ() DAD = cal_DAD(modi_adj).to(device) load_mat_samples = LoadMatSamples() feature, label, SCADA = load_mat_samples.load('pf') ava_idx = load_mat_samples.ava_idx if SAVE: feature_train = feature[:-TEST_SIZE] feature_test = feature[-TEST_SIZE:] label_train = label[:-TEST_SIZE] label_test = label[-TEST_SIZE:] features_batches_train, labels_batches_train = gen_batches( feature_train, label_train, BATCH_SIZE) gcn = PF_GCN(DAD).to(device) # get object of the GCN with matrix DAD. criterion = nn.MSELoss() # print(features_batches_train.shape) loss = 0.1 pf_loss = [] for iepoch in range(EPOCHES): for ibatch in range(len(features_batches_train)): step = float(loss / 500) optimizer = optim.Adam(gcn.parameters(), lr=step) optimizer.zero_grad() output = gcn(features_batches_train[ibatch].to(device)) loss = criterion(output, labels_batches_train[ibatch].to(device)) loss.backward() optimizer.step()
def main(_): ## # Load Data ## with open(FLAGS.data_path, 'r') as f: reader = csv.reader(f) # data is a list of tuples (img path, steering angle) data = np.array([row for row in reader]) # Split train and validation data np.random.shuffle(data) split_i = int(len(data) * 0.9) X_train, y_train = list(zip(*data[:split_i])) X_val, y_val = list(zip(*data[split_i:])) X_train, y_train = np.array(X_train), np.array(y_train) X_val, y_val = np.array(X_val), np.array(y_val) ## # Define Model ## model = Sequential([ Conv2D(32, (3, 3), input_shape=(64, 32, 1), padding='same', activation='relu'), Conv2D(64, (3, 3), padding='same', activation='relu'), MaxPooling2D(), Dropout(0.5), Conv2D(128, (3, 3), padding='same', activation='relu'), Conv2D(256, (3, 3), padding='same', activation='relu'), MaxPooling2D(), Dropout(0.5), Flatten(), Dense(1024, activation='relu'), # Dense(512, activation='relu'), # Dense(128, activation='relu'), Dense(1, name='output', activation='tanh'), ]) model.compile(optimizer=Adam(lr=FLAGS.lrate), loss='mse') ## # Train ## history = model.fit_generator( gen_batches(X_train, y_train, FLAGS.batch_size), len(X_train), FLAGS.num_epochs, validation_data=gen_batches(X_val, y_val, FLAGS.batch_size), validation_steps=(len(X_val) / FLAGS.batch_size)) ## # Save model ## json = model.to_json() model.save_weights('save/sdc-mjc+mrg2/model.h5') with open('save/sdc-mjc+mrg2/model.json', 'w') as f: f.write(json)
# Some information about the data after splitting print("Splitting with split rate: ", SPLIT) print("Training Data Size: ", X_train.shape, y_train.shape) print("Validation Data Size: ", X_valid.shape, y_valid.shape) # Freeing the memory block cause you know, it needs to be free. data = None X = None y = None ans = str(input("Continue? ([Y]/N) --- ")) if (ans == 'N' or ans == 'n'): exit() # Generate training and validation data for the model compilation train = utils.gen_batches(X_train, y_train, True, BATCH_SIZE) valid = utils.gen_batches(X_valid, y_valid, False, BATCH_SIZE) ######################### Model Training ########################### def nvidia(LR=1e-4, inputshape=(64, 64, 1), comp=False, summary=False): model = Sequential() model.add(Lambda(lambda x: x/127.5-1.0, input_shape=inputshape)) model.add(Conv2D(24, 5, 5, subsample=(2, 2), border_mode='valid', W_regularizer=l2(0.001))) model.add(ELU()) model.add(Conv2D(36, 5, 5, subsample=(2, 2), border_mode='valid', W_regularizer=l2(0.001))) model.add(ELU()) model.add(Conv2D(48, 5, 5, subsample=(2, 2), border_mode='valid', W_regularizer=l2(0.001))) model.add(ELU()) model.add(Conv2D(64, 3, 3, border_mode='valid', W_regularizer=l2(0.001)))
def main(_): tf.logging.set_verbosity(tf.logging.INFO) parser = argparse.ArgumentParser() parser.add_argument("--train_file", type=str, default="", help="Input train file.") parser.add_argument("--vocab_file", type=str, default="", help="Input vocab file.") parser.add_argument("--model_save_dir", type=str, default="", help="Specified the directory in which the model should stored.") parser.add_argument("--lstm_dim", type=int, default=100, help="Dimension of LSTM cell.") parser.add_argument("--embedding_dim", type=int, default=100, help="Dimension of word embedding.") parser.add_argument("--layer_num", type=int, default=2, help="LSTM layer num.") parser.add_argument("--batch_size", type=int, default=32, help="Batch size.") parser.add_argument("--train_step", type=int, default=10000, help="Number of training steps.") parser.add_argument("--warmup_step", type=int, default=1000, help="Number of warmup steps.") parser.add_argument("--learning_rate", type=float, default=0.001, help="The initial learning rate") parser.add_argument("--dropout_rate", type=float, default=0.5, help="Dropout rate") parser.add_argument("--seed", type=int, default=0, help="Random seed value.") parser.add_argument("--print_step", type=int, default=1000, help="Print log every x step.") parser.add_argument("--max_predictions_per_seq", type=int, default=10, help="For each sequence, predict x words at most.") parser.add_argument("--weight_decay", type=float, default=0, help='Weight decay rate') parser.add_argument("--clip_norm", type=float, default=1, help='Clip normalization rate.') parser.add_argument("--max_seq_len", type=int, default=100, help="Max seqence length.") parser.add_argument("--use_queue", type=int, default=0, help="Whether or not using a queue for input.") parser.add_argument("--init_checkpoint", type=str, default="", help="Initial checkpoint") parser.add_argument("--enqueue_thread_num", type=int, default=5, help="Enqueue thread count.") args = parser.parse_args() np.random.seed(args.seed) tf.set_random_seed(args.seed) tf.logging.info(args) if not os.path.exists(args.model_save_dir): os.mkdir(args.model_save_dir) # load data word2id, id2word = utils.load_vocab_file(args.vocab_file) training_sens = utils.load_pretraining_data(args.train_file, args.max_seq_len) if not args.use_queue: utils.to_ids(training_sens, word2id, args, id2word) other_arg_dict = {} other_arg_dict['token_num'] = len(word2id) # load model model = models.create_bidirectional_lm_training_op(args, other_arg_dict) gc.collect() saver = tf.train.Saver(max_to_keep=2) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) if args.init_checkpoint: tf.logging.info('restore the checkpoint : ' + str(args.init_checkpoint)) saver.restore(sess, args.init_checkpoint) total_loss = 0 num = 0 global_step = 0 while global_step < args.train_step: if not args.use_queue: iterator = utils.gen_batches(training_sens, args.batch_size) else: iterator = utils.queue_gen_batches(training_sens, args, word2id, id2word) for batch_data in iterator: feed_dict = {model.ph_tokens: batch_data[0], model.ph_length: batch_data[1], model.ph_labels: batch_data[2], model.ph_positions: batch_data[3], model.ph_weights: batch_data[4], model.ph_dropout_rate: args.dropout_rate} _, global_step, loss, learning_rate = sess.run([model.train_op, \ model.global_step, model.loss_op, model.learning_rate_op], feed_dict=feed_dict) total_loss += loss num += 1 if global_step % args.print_step == 0: tf.logging.info("\nglobal step : " + str(global_step) + ", avg loss so far : " + str(total_loss / num) + ", instant loss : " + str(loss) + ", learning_rate : " + str(learning_rate) + ", time :" + str(time.strftime('%Y-%m-%d %H:%M:%S'))) tf.logging.info("save model ...") saver.save(sess, args.model_save_dir + '/lm_pretrain.ckpt', global_step=global_step) gc.collect() if not args.use_queue: utils.to_ids(training_sens, word2id, args, id2word) # MUST run this for randomization for each sentence gc.collect()
import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import numpy as np import tensorflow as tf from tensorflow import keras import threading from network import AGAN from utils import to_array, gen_batches BATCH_SIZE = 25 paths = os.listdir('assets/') training_data = gen_batches(paths, BATCH_SIZE) print('Done preparing training data') aGAN = AGAN(noise_size=200, batch_size=BATCH_SIZE) aGAN.restore() try: aGAN.train(dataset=training_data, epochs=12000, example_interval=5, save_interval=100) except KeyboardInterrupt: try: print('Saving...') aGAN.save() print('Clearing session...')
def _fit(self, X, y): hidden_layer_sizes = self.hidden_layer_sizes if not hasattr(hidden_layer_sizes, '__iter__'): hidden_layer_sizes = [hidden_layer_sizes] hidden_layer_sizes = list(hidden_layer_sizes) self._validate_hyperparams(X, y) X, y = self._validate_input(X, y) # make y 2D if len(y.shape) == 1: y.reshape(y.shape[0], 1) # tell the size of every layers layer_sizes = [X.shape[1]] layer_sizes.extend(hidden_layer_sizes) layer_sizes.append(y.shape[1]) self._initilize(layer_sizes) packed_params = self._pack_W_b(self.W_, self.b_) # trainning params assignment batch_size = self.batch_size learning_rate = self.learning_rate momentum = self.momentum tol = self.tol verbose = self.verbose velocities = np.zeros_like(packed_params) self.loss_ = [] for i in range(self.max_iter): epoch_loss = 0 for batch_slice in gen_batches(X.shape[0], batch_size): _X, _y = X[batch_slice], y[batch_slice] activations = self._forward_pass(_X, self.W_, self.b_) loss, grad_W, grad_b = self._backprop(_X, _y, self.W_, self.b_, activations) packed_grad = self._pack_W_b(grad_W, grad_b) packed_params = self._pack_W_b(self.W_, self.b_) # apply gradients updates = momentum * velocities - learning_rate * packed_grad velocities = updates packed_params = packed_params + updates self.W_, self.b_ = self._unpack_W_b(packed_params, self.W_, self.b_) epoch_loss += loss * (batch_slice.stop - batch_slice.start) epoch_loss /= X.shape[0] self.loss_.append(epoch_loss) # two continuous epoch get loss changes lower than tol if i > 2 and \ (self.loss_[i] - self.loss_[i-1] > -tol) and \ (self.loss_[i-1] - self.loss_[i-2] > -tol): break if verbose: print('Iteration {:d}, loss = {:f}'.format(i + 1, epoch_loss)) return self
def slice_into_batches(self, Input_list): produced_batches = [] length = len(Input_list) for each in gen_batches(length, 100): produced_batches.append(Input_list[each]) return produced_batches
def _train_model(self, train_X, train_y, validation_X, validation_y, test_X, test_y): """ This function perform the training process """ feed_train = { self.input_data: train_X, self.input_labels: train_y, self.keep_prob: self.finetune_dropout } feed_validation = { self.input_data: validation_X, self.input_labels: validation_y, self.keep_prob: 1 } feed_test = { self.input_data: test_X, self.input_labels: test_y, self.keep_prob: 1 } shuff = list(zip(train_X, train_y)) old_iter, old_accuracy = 0, 0 for iteration in range(1, self.finetune_num_epochs + 1): np.random.shuffle(shuff) batches = [_ for _ in gen_batches(shuff, self.finetune_batch_size)] for batch in batches: X_batch, y_batch = zip(*batch) feed_batch = { self.input_data: X_batch, self.input_labels: y_batch, self.keep_prob: self.finetune_dropout } self.sess.run(self.train_step, feed_dict=feed_batch) updates = [self.loss, self.accuracy] loss_train, accuracy_train = self.sess.run(updates, feed_dict=feed_train) print('Iter %d: ' % iteration, end='') if validation_X is not None: loss_val, accuracy_val = self.sess.run( updates, feed_dict=feed_validation) loss_test, accuracy_test = self.sess.run(updates, feed_dict=feed_test) print('Training: current loss %f' % loss_train, 'current accuracy %f' % accuracy_train, sep=' | ', end=' || ') print('Validation: current loss %f' % loss_val, 'current accuracy %f' % accuracy_val, sep=' | ', end=' || ') print('Test: accuracy %f' % accuracy_test) self.accuracy_summary.append(accuracy_test) self.loss_summary.append(loss_test) if old_accuracy > accuracy_test and (iteration - old_iter) > 5: return if accuracy_test > old_accuracy: old_accuracy = accuracy_test old_iter = iteration else: print('Training: current loss %f' % loss_train, 'current accuracy %f' % accuracy_train, sep=' | ')