def _run_train_step_DBM(self, train_set, hidden_data):
""" Run a training step. A training step is made by randomly shuffling the training set,
divide into batches and run the variable update nodes for each batch.
:param train_set: training set
:return: self
"""
np.random.shuffle(train_set)
batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)]
batches_2 = [
_ for _ in utils.gen_batches(hidden_data, self.batch_size)
]
updates = [
self.w_1_upd8_3, self.w_2_upd8_3, self.bh_1_upd8_3,
self.bh_2_upd8_3, self.bv_1_upd8_3, self.bv_2_upd8_3
]
for batch_1, batch_2 in zip(batches, batches_2):
upd = self.tf_session.run(updates,
feed_dict=self._create_feed_dict(
batch_1,
True,
pos='dbm',
status='train',
hidden_data=batch_2))
def train(self, training_x, validation_x):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
corruption_ratio = np.round(self.corruption_fraction * training_x.shape[1]).astype(np.int)
for epoch in range(self.epochs):
print("Epoch: {}".format(epoch))
x_corrupted = self.corrupt_input(training_x, corruption_ratio)
shuff = list(zip(training_x, x_corrupted))
np.random.shuffle(shuff)
batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)]
for batch in batches:
x_batch, x_corrupted_batch = zip(*batch)
sess.run(self.train_step, feed_dict={self.input_data: x_batch,
self.corrupted_input_data: x_corrupted_batch})
if epoch % 5 == 0:
if validation_x is not None:
error = sess.run(self.cost, feed_dict={self.input_data: validation_x,
self.corrupted_input_data: validation_x})
print("Validation cost is {}".format(error))
if validation_x is not None:
error = sess.run(self.cost, feed_dict={self.input_data: validation_x,
self.corrupted_input_data: validation_x})
print("Validation cost is {}".format(error))
def main(_):
##
# Load Data
##
with open(FLAGS.data_path, 'r') as f:
reader = csv.reader(f)
# data is a list of tuples (img path, steering angle)
data = np.array([row for row in reader])
# Split train and validation data
np.random.shuffle(data)
split_i = int(len(data) * 0.9)
X_train, y_train = list(zip(*data[:split_i]))
X_val, y_val = list(zip(*data[split_i:]))
X_train, y_train = np.array(X_train), np.array(y_train)
X_val, y_val = np.array(X_val), np.array(y_val)
gen = gen_batches(X_train, y_train, 10)
imgs, labels = gen.__next__()
print(imgs[0])
for i, img in enumerate(imgs):
imsave('test/' + str(i) + '.png', np.squeeze(img))
def _run_train_step(self, train_set, corruption_ratio, corrupted_train_set):
''' Run a training step. A training step is made by randomly corrupting
the training set, randomly shuffling it, and dividing it into batches and
running the optimizer for each batch.
Parameters
----------
train_set : training set
corruption_ratio : fraction of elements to corrupt
Returns
-------
self
'''
if self.corr_type == 'id':
x_corrupted = corrupted_train_set
else:
x_corrupted = self._corrupt_input(train_set, corruption_ratio)
shuff = zip(train_set, x_corrupted)
np.random.shuffle(shuff)
batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)]
for idx, batch in enumerate(batches):
x_batch, x_corr_batch = zip(*batch)
tr_feed = {self.input_data: x_batch, self.input_data_corr: x_corr_batch}
self.tf_session.run(self.train_step, feed_dict=tr_feed)
def _run_train_step(self, train_set, validation_set, pos='first'):
""" Run a training step. A training step is made by randomly shuffling the training set,
divide into batches and run the variable update nodes for each batch.
:param train_set: training set
:return: self
"""
np.random.shuffle(train_set)
batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)]
if pos == 'first':
updates = [
self.w_upd8_1, self.bh_upd8_1, self.bv_upd8_1, self.last_s_1
]
else:
updates = [
self.w_upd8_2, self.bh_upd8_2, self.bv_upd8_2, self.last_s_2
]
i = 0
start = True
for batch in batches:
upd = self.tf_session.run(updates,
feed_dict=self._create_feed_dict(
batch, start, pos))
### feed to next batch
self.last_s = upd[3]
start = False
def eval(self, val_tup, verbose=True, write_summary=False):
""" Evaluates the model on given data. Writes out a validation loss summary.
:param val_tup: The data to evaluate the model on; tuple of (images, labels, feedback)
:param verbose: Whether to print the error
:param write_summary: whether to write a summary at current timestep to disk (for training)
"""
if verbose:
print("validating...")
error = 0.0
loss = 0.0
#note, if you are not interested in the loss (but only the error metric), you can set only_positive=True for faster results
for imgs, labels, feedback in utils.gen_batches(
val_tup,
shuffle=False,
only_positive=False,
batch_sz=c.EVAL_BATCH_SIZE):
processed_imgs = self._process_images(imgs)
sess_args = [self.abs_err, self.loss]
feed_dict = {
self.img_input: processed_imgs,
self.labels: labels,
self.feedback: feedback
}
batch_abs_err, batch_loss = self.sess.run(sess_args,
feed_dict=feed_dict)
error += len(
imgs
) * batch_abs_err #batches can be different lengths, so weight them
loss += len(imgs) * batch_loss
error /= len(val_tup[0])
loss /= len(val_tup[0])
if write_summary:
#make sumary mannually becuase its too large for one batch
step = self.sess.run(self.global_step)
#error
error_summary = tf.Summary(value=[
tf.Summary.Value(tag='val_error' + c.TRIAL_STR,
simple_value=error)
])
self.summary_writer.add_summary(error_summary, global_step=step)
#loss
loss_summary = tf.Summary(value=[
tf.Summary.Value(tag='val_loss' + c.TRIAL_STR,
simple_value=loss)
])
self.summary_writer.add_summary(loss_summary, global_step=step)
if verbose:
print("")
print("Validation Error:", error)
def _run_train_step(self, train_set):
""" Run a training step. A training step is made by randomly shuffling the training set,
divide into batches and run the variable update nodes for each batch.
:param train_set: training set
:return: self
"""
np.random.shuffle(train_set)
batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)]
updates = [self.w_upd8, self.bh_upd8, self.bv_upd8]
for batch in batches:
self.tf_session.run(updates,
feed_dict=self._create_feed_dict(batch))
def train(self, trX, valX, restore_previous_model=False):
"""
Train function, take the input train and validation set, fit it to the model and output the RMSE score
:param trX: Training set
:param valX: Validation set
:param restore_previous_model: Do we restore the pre-trained model or not?
"""
ops.reset_default_graph()
self._create_graph()
merged = tf.summary.merge_all()
init_op = tf.initialize_all_variables()
with tf.Session() as self.sess:
self.sess.run(init_op)
if restore_previous_model:
self.saver.restore(self.sess,
self.models_dir + self.model_name)
self.model_name += '-restored{}'.format(self.N_EPOCH)
writer = tf.summary.FileWriter(self.summary_dir,
self.sess.graph_def)
for epoch in range(self.N_EPOCH):
np.random.shuffle(trX)
batches = [_ for _ in utils.gen_batches(trX, self.BATCH_SIZE)]
total_error = 0.0
for batch in batches:
result = self.sess.run([self.run_update, self.cost],
feed_dict={self.X: batch})
total_error += result[1]
print("Cost at epoch %s of training: %s" %
(epoch, total_error / len(batches)))
# if epoch % 5 == 0:
# result = self.sess.run([merged, self.cost], feed_dict={self.X: valX})
# summary_str = result[0]
# err = result[1]
#
# writer.add_summary(summary_str, 1)
# print("Cost at epoch %s on validation set: %s" % (epoch, err))
self.saver.save(self.sess, self.models_dir + self.model_name)
def train(self, train_tup, val_tup):
""" Training loop. Trains & validates for the given number of epochs
on given data.
:param train_tup: All the training data; tuple of (images, labels, feedback)
:param val_tup: All the validation data; tuple of (images, labels, feedback)
"""
#caclulate number of steps
steps_per_epc = int(np.ceil(len(train_tup[0])/c.BATCH_SIZE))
num_steps = c.NUM_EPOCHS * steps_per_epc
initial_step = self.sess.run(self.global_step)
print(num_steps, "steps total in this train session...")
#start training
self.eval(val_tup, write_summary=True) #eval always at start
for i in range(c.NUM_EPOCHS):
print("\nEpoch", i+1, "out of", c.NUM_EPOCHS)
for imgs, labels, feedback in utils.gen_batches(train_tup, shuffle=True):
self._train_step(imgs, labels, feedback, initial_step, num_steps, val_tup)
def _train_model(self, train_X, validation_X):
"""
This function perform the training process
"""
feed_train = {
self.input_data: train_X,
self.keep_prob: self.finetune_dropout
}
feed_validation = {self.input_data: validation_X, self.keep_prob: 1}
shuff = train_X
old_iter, old_loss = 0, 10000
for iteration in range(1, self.finetune_num_epochs + 1):
np.random.shuffle(shuff)
batches = [_ for _ in gen_batches(shuff, self.finetune_batch_size)]
for batch in batches:
feed_batch = {
self.input_data: batch,
self.keep_prob: self.finetune_dropout
}
self.sess.run(self.train_step, feed_dict=feed_batch)
print('Iter %d' % iteration, end=': ')
loss_train = self.sess.run(self.reconstruction_loss,
feed_dict=feed_train)
if validation_X is not None:
loss_validation = self.sess.run(self.reconstruction_loss,
feed_dict=feed_validation)
reconstructed_error = self.sess.run(self.reconstructed_error,
feed_dict=feed_validation)
self.loss_summary.append(loss_validation)
print('Training set: current loss %f' % loss_train,
end=' || ')
print('Validation set: current loss %f' % loss_validation)
print('Validation: ', np.argsort(-reconstructed_error)[:25])
if loss_validation > old_loss and (iteration - old_iter) > 5:
return
if loss_validation < old_loss:
old_loss = loss_validation
old_iter = iteration
else:
print('Training set: current loss %f' % loss_train)
def _train_model(self, train_X, train_y, validation_X, validation_y):
"""
This function perform the training process
"""
feed_train = {
self.input_data: train_X,
self.input_labels: train_y,
self.keep_prob: self.finetune_dropout
}
feed_validation = {
self.input_data: validation_X,
self.input_labels: validation_y,
self.keep_prob: 1
}
shuff = list(zip(train_X, train_y))
for iteration in range(1, self.finetune_num_epochs + 1):
np.random.shuffle(shuff)
batches = [_ for _ in gen_batches(shuff, self.finetune_batch_size)]
for batch in batches:
X_batch, y_batch = zip(*batch)
feed_batch = {
self.input_data: X_batch,
self.input_labels: y_batch,
self.keep_prob: self.finetune_dropout
}
self.sess.run(self.train_step, feed_dict=feed_batch)
if validation_X is not None:
updates = [self.loss, self.accuracy]
loss_train, accuracy_train = self.sess.run(
updates, feed_dict=feed_train)
print('Iter: %d' % iteration)
print('....Training: current loss %f' % loss_train)
print('....Training: current accuracy %f' % accuracy_train)
loss_val, accuracy_val = self.sess.run(
updates, feed_dict=feed_validation)
print('****Validation: current loss %f' % loss_val)
print('****Validation: current accuracy %f' % accuracy_val)
def _run_train_step(self, train_set):
""" Run a training step. A training step is made by randomly shuffling the training set,
divide into batches and run the variable update nodes for each batch. If self.plot_training_loss
is true, will record training loss after each batch.
:param train_set: training set
:return: self
"""
np.random.shuffle(train_set)
batches = [_ for _ in utils.gen_batches(train_set, self.batch_size)]
updates = [self.w_upd8, self.bh_upd8, self.bv_upd8]
for batch in batches:
if self.plot_training_loss:
_, loss = self.tf_session.run([updates, self.loss_function], feed_dict=self._create_feed_dict(batch))
self.training_losses.append(loss)
else:
self.tf_session.run(updates, feed_dict=self._create_feed_dict(batch))
def _run_train_step(self, train_set, corruption_ratio):
""" Run a training step. A training step is made by randomly corrupting the training set,
randomly shuffling it, divide it into batches and run the optimizer for each batch.
:param train_set: training set
:param corruption_ratio: fraction of elements to corrupt
:return: self
"""
x_corrupted = self._corrupt_input(train_set, corruption_ratio)
shuff = zip(train_set, x_corrupted)
np.random.shuffle(shuff)
batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)]
for batch in batches:
x_batch, x_corr_batch = zip(*batch)
tr_feed = {
self.input_data: x_batch,
self.input_data_corr: x_corr_batch
}
self.tf_session.run(self.train_step, feed_dict=tr_feed)
def eval(self, val_tup, verbose=True, write_summary=False, evaluate_angle=False):
""" Evaluates the model on given data. Writes out a validation loss summary.
:param val_tup: The data to evaluate the model on; tuple of (images, labels, feedback)
:param verbose: Whether to print. E.g. the error (and the predicted angles if evaluate_angle)
:param evaluate_angle: Wether to evaluate the anlge (implied by our net), as opposed to the feedback predicted
:param write_summary: whether to write a summary at current timestep to disk (for training)
"""
if verbose:
print("validating...")
#evalualt#
error = 0.0
for imgs, labels, feedback in utils.gen_batches(val_tup, shuffle=True, only_positive=evaluate_angle, batch_sz=c.EVAL_BATCH_SIZE):
processed_imgs = self._process_images(imgs)
if evaluate_angle:
batch_abs_err = self._eval_angle_on_batch(processed_imgs, labels, verbose)
else:
sess_args = self.abs_err
feed_dict = {self.img_input: processed_imgs,
self.labels: labels,
self.feedback: feedback}
batch_abs_err = self.sess.run(sess_args, feed_dict=feed_dict)
error += len(imgs) * batch_abs_err #batches can be different lengths, so weight them
error /= len(val_tup[0])
##
if write_summary:
#make sumary mannually becuase its too large for one batch
step = self.sess.run(self.global_step)
error_summary = tf.Summary(value=[tf.Summary.Value(tag='val_feedback_error'+c.TRIAL_STR, simple_value=error)])
self.summary_writer.add_summary(error_summary, global_step=step)
if verbose:
print("")
string = "ANGLE" if evaluate_angle else "FEEDBACK"
print("Validation Error [" + string + " predicition]:", error)
def main(_):
# Main model-building process
print('Starting data load...')
print(time.strftime('%H:%M:%S'))
print()
start_time = time.time()
##
# Load Data
##
##
# Will have centre_count main driving files - each of which are frames taken when driving centrally
# Naming driving_log_centre_X.csv where X is sequential numbering
# Will have left_recover_count files - each of which represents sets of frames taken when recovering from the left
# When processing these files will just take +ve steering angle imags, ignoring all others
# Will have right_recover_count files - each of which represents set of frames taken when recovering from the right
# When processing these files will just take -ve steering angle images, ignoring all others
# Note that both the left_ and right_ recovery files will have some normal driving frames with +ve/-ve steering angles
# and these will just add to the set of data
##
# Initialise variables
X_train = [] # our training data image paths
y_train = [] # our training data labels
n_min_y = 0.0 # will use these two variables to track the min/max steering angles found in the data
n_max_y = 0.0
# Read in the centre images data
for file_num in range(1, FLAGS.centre_count + 1):
sFilename = FLAGS.data_file + '_centre_' + str(file_num) + '.csv'
with open(sFilename, 'r') as f:
reader = csv.reader(f)
# data is a list of tuples (img path, steering angle)
data = np.array([row for row in reader])
# Provided data has form of:
# data[i][0] = centre image
# data[i][1] = left image
# data[i][2] = right image
# data[i][3] = steering angle
# data[i][4] = throttle
# data[i][5] = brake
# data[i][6] = speed
# Parse out the centre camera view and steering angle - this is our training input and label
for i in range(len(data)):
if data[i][0] != 'center': # ignore the header row
# sort out the image path, if using the Udacity provided data
s = str(data[i][0]) # get the filename from the csv data
s = s[s.rfind('/') + 1:] # trim off any paths
s = FLAGS.imgs_dir + s # put back the path to the images location
X_train.append(s)
y_label = float(data[i][3])
y_train.append(y_label)
if y_label < n_min_y:
n_min_y = y_label
if y_label > n_max_y:
n_max_y = y_label
print('Imported ', len(X_train), ' centre frames')
print('Total frames so far :', len(X_train))
print('Range of label values : ', n_min_y, ' to ', n_max_y)
print()
n_centre_frames = len(X_train)
# Read in the left-of-centre recovery images data
for file_num in range(1, FLAGS.left_recover_count + 1):
sFilename = FLAGS.data_file + '_left_' + str(file_num) + '.csv'
with open(sFilename, 'r') as f:
reader = csv.reader(f)
# data is a list of tuples (img path, steering angle)
data = np.array([row for row in reader])
for i in range(len(data)):
if data[i][0] != 'center': # ignore the header row, if there is one
if float(
data[i]
[3]) > 0.0: # only interested in +ve steering angle frames
# sort out the image path, if using the Udacity provided data
s = str(data[i][0]) # get the filename from the csv data
s = s[s.rfind('/') + 1:] # trim off any paths
s = FLAGS.imgs_dir + s # put back the path to the images location
X_train.append(s)
y_label = float(data[i][3])
y_train.append(y_label)
if y_label < n_min_y:
n_min_y = y_label
if y_label > n_max_y:
n_max_y = y_label
n_left_frames = len(X_train) - n_centre_frames
print('Imported ', n_left_frames, ' left frames')
print('Total frames so far :', len(X_train))
print('Range of label values : ', n_min_y, ' to ', n_max_y)
print()
# Read in the right-of-centre recovery images data
for file_num in range(1, FLAGS.right_recover_count + 1):
sFilename = FLAGS.data_file + '_right_' + str(file_num) + '.csv'
with open(sFilename, 'r') as f:
reader = csv.reader(f)
# data is a list of tuples (img path, steering angle)
data = np.array([row for row in reader])
for i in range(len(data)):
if data[i][0] != 'center': # ignore the header row, if there is one
if float(
data[i]
[3]) < 0.0: # only interested in -ve steering angle frames
# sort out the image path, different if local or if using the Udacity provided data
s = str(data[i][0]) # get the filename from the csv data
s = s[s.rfind('/') + 1:] # trim off any paths
s = FLAGS.imgs_dir + s # put back the path to the images location
X_train.append(s)
y_label = float(data[i][3])
y_train.append(y_label)
if y_label < n_min_y:
n_min_y = y_label
if y_label > n_max_y:
n_max_y = y_label
n_right_frames = len(X_train) - n_left_frames - n_centre_frames
print('Imported ', n_right_frames, ' right frames')
print('Total frames so far :', len(X_train))
print('Range of label values : ', n_min_y, ' to ', n_max_y)
print()
print('Total frames : ', len(X_train))
print()
# Split into training and validation data
X_train, y_train, X_valid, y_valid = train_validation_split(
X_train, y_train, training_percent)
X_train = np.array(X_train)
y_train = np.array(y_train)
X_valid = np.array(X_valid)
y_valid = np.array(y_valid)
print('Split training/validation sets (', training_percent, '/',
(1.0 - training_percent), ')')
print('Training set size : ', len(X_train))
print('Validation set size : ', len(X_valid))
print()
##
# Define Model
##
print('Creating model...')
model = Sequential([
Conv2D(32,
3,
3,
input_shape=(32, 16, 1),
border_mode='same',
activation='relu'),
Conv2D(64, 3, 3, border_mode='same', activation='relu'),
Dropout(0.5),
Conv2D(128, 3, 3, border_mode='same', activation='relu'),
Conv2D(256, 3, 3, border_mode='same', activation='relu'),
Dropout(0.5),
Flatten(),
Dense(1024, activation='relu'),
Dense(512, activation='relu'),
Dense(128, activation='relu'),
Dense(1, name='output', activation='tanh'),
])
model.compile(optimizer=Adam(lr=FLAGS.lrate), loss='mse')
model.summary()
print('Model created')
print()
##
# Train
##
# When using a generator, can say how may samples to use per epoch
# The generator will be creating random batches, so this value can be other than the total samples value...
n_samples_per_epoch = len(X_train) #* 2
n_validation_size = len(X_valid) #* 2
history = model.fit_generator(gen_batches(X_train, y_train,
FLAGS.batch_size),
n_samples_per_epoch,
FLAGS.num_epochs,
validation_data=gen_batches(
X_valid, y_valid, FLAGS.batch_size),
nb_val_samples=n_validation_size)
##
# Save model
##
json = model.to_json()
model.save_weights('save/model.h5')
with open('save/model.json', 'w') as f:
f.write(json)
end_time = time.time()
print()
print('Training time : ', (end_time - start_time) / 60, ' minutes')
def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
intercept_grads, layer_units):
params = self.coefs_ + self.intercepts_
if self.solver == 'sgd':
self._optimizer = SGDOptimizer(
params, self.learning_rate,
self.momentum, self.nesterovs_momentum)
elif self.solver == 'adam':
self._optimizer = AdamOptimizer(
params, self.learning_rate, self.beta_1, self.beta_2,
self.epsilon)
# early_stopping
early_stopping = self.early_stopping
if self.evaluation:
X_val = self._X_eval
y_val = self._y_eval
else:
X_val = None
y_val = None
# number of samples
n_samples = X.shape[0]
# batch size
batch_size = np.clip(self.batch_size, 1, n_samples)
# training process
try:
# every epoch
for it in range(self.epoches):
# loss
accumulated_loss = 0.0
# every batches
for batch_slice in gen_batches(n_samples, batch_size):
activations[0] = X[batch_slice]
# backpropogation
batch_loss, coef_grads, intercept_grads = self._backprop(
X[batch_slice], y[batch_slice], activations, deltas,
coef_grads, intercept_grads)
# loss
accumulated_loss += batch_loss * (batch_slice.stop -
batch_slice.start)
# update weights
grads = coef_grads + intercept_grads
self._optimizer.update_params(grads)
self.n_iter_ += 1
self.loss_ = accumulated_loss / X.shape[0]
# append loss
self.loss_curve_.append(self.loss_)
# append training acore
self.train_scores_.append(self.score(X, np.argmax(y, axis = 1)))
if self.evaluation == False:
if self.verbose:
print("Iteration %d, loss = %.8f Training score: %f" % (self.n_iter_,self.loss_, self.train_scores_[-1]))
else:
# compute validation score, use that for stopping
self.validation_scores_.append(self.score(X_val, y_val))
if self.verbose:
print("Iteration %d, loss = %.8f Training score: %f Validation score: %f" % (self.n_iter_,self.loss_, self.train_scores_[-1],
self.validation_scores_[-1]))
# update no_improvement_count based on training loss or
# validation score according to early_stopping
self._update_no_improvement_count(early_stopping, X_val, y_val)
if self._no_improvement_count > self.n_iter_no_change:
# not better than last `n_iter_no_change` iterations by tol
# stop or decrease learning rate
if early_stopping:
msg = ("Early stop. Validation score did not improve more than "
"tol=%f for %d consecutive epochs." % (
self.tol, self.n_iter_no_change))
print(msg)
break
self._no_improvement_count = 0
except KeyboardInterrupt:
warnings.warn("Training interrupted by user.")
# restore best weights
self.coefs_ = self._best_coefs
self.intercepts_ = self._best_intercepts
def fit(self, trX, vlX=None, restore_previous_model=False):
if vlX is not None:
self.validation_size = vlX.shape[0]
# Reset tensorflow's default graph
ops.reset_default_graph()
self._create_graph()
merged = tf.merge_all_summaries()
init_op = tf.initialize_all_variables()
self.saver = tf.train.Saver()
with tf.Session() as self.sess:
self.sess.run(init_op)
if restore_previous_model:
# Restore previous model
self.saver.restore(self.sess,
self.models_dir + self.model_name)
# Change model name
self.model_name += '-restored{}'.format(self.n_iter)
# Write tensorflow summaries to summary dir
writer = tf.train.SummaryWriter(self.summary_dir,
self.sess.graph_def)
for i in range(self.n_iter):
# Randomly shuffle the input
np.random.shuffle(trX)
batches = [_ for _ in utils.gen_batches(trX, self.batch_size)]
for batch in batches:
self.sess.run(
[self.w_upd8, self.bh_upd8, self.bv_upd8],
feed_dict={
self.x: batch,
self.hrand: np.random.rand(batch.shape[0],
self.nhid),
self.vrand: np.random.rand(batch.shape[0],
self.nvis)
})
if i % 5 == 0:
# Record summary data
if vlX is not None:
feed = {
self.x:
vlX,
self.hrand:
np.random.rand(self.validation_size, self.nhid),
self.vrand:
np.random.rand(self.validation_size, self.nvis)
}
result = self.sess.run([merged, self.cost],
feed_dict=feed)
summary_str = result[0]
err = result[1]
writer.add_summary(summary_str, 1)
if self.verbose == 1:
print("Validation cost at step %s: %s" % (i, err))
# Save trained model
self.saver.save(self.sess, self.models_dir + self.model_name)
def fit(self, trX, vlX=None, restore_previous_model=False):
""" Fit the model to the data.
:type trX: array_like, shape (n_samples, n_features).
:param trX: Training data.
:type vlX: array_like, shape (n_validation_samples, n_features).
:param vlX: optional, default None. Validation data.
:return: self
"""
n_features = trX.shape[1]
self._create_graph(n_features)
# Merge all the summaries
merged = tf.merge_all_summaries()
# Initialize variables
init_op = tf.initialize_all_variables()
# Add ops to save and restore all the variables
self.saver = tf.train.Saver()
with tf.Session() as self.sess:
self.sess.run(init_op)
if restore_previous_model:
# Restore previous model
self.saver.restore(self.sess,
self.models_dir + self.model_name)
# Change model name
self.model_name += '-restored{}'.format(self.n_iter)
# ################## #
# Training phase #
# ################## #
v = np.round(self.corr_frac * n_features).astype(np.int)
# Write the summaries to summary_dir
writer = tf.train.SummaryWriter(self.summary_dir,
self.sess.graph_def)
for i in range(self.n_iter):
# #################### #
# Input Corruption #
# #################### #
if self.corr_type == 'masking':
x_corrupted = utils.masking_noise(trX, v)
elif self.corr_type == 'salt_and_pepper':
x_corrupted = utils.salt_and_pepper_noise(trX, v)
else: # none, normal autoencoder
x_corrupted = trX
# Randomly shuffle the input
shuff = zip(trX, x_corrupted)
np.random.shuffle(shuff)
# # Divide dataset into mini-batches
batches = [
_ for _ in utils.gen_batches(shuff, self.batch_size)
]
# All the batches for each epoch
for batch in batches:
x_batch, x_corr_batch = zip(*batch)
tr_feed = {
self.x: x_batch,
self.x_corr: x_corr_batch,
self.keep_prob: self.dropout
}
self.sess.run(self.train_step, feed_dict=tr_feed)
# Record summary data
if vlX is not None:
vl_feed = {
self.x: vlX,
self.x_corr: vlX,
self.keep_prob: 1.
}
result = self.sess.run([merged, self.cost],
feed_dict=vl_feed)
summary_str = result[0]
err = result[1]
writer.add_summary(summary_str, i)
if self.verbose == 1:
print("Validation cost at step %s: %s" % (i, err))
# Save trained model
self.saver.save(self.sess, self.models_dir + self.model_name)
EPOCHES = 200
SAMPLE_LEN = 64
modi_adj = loadADJ()
DAD = cal_DAD(modi_adj).to(device)
load_mat_samples = LoadMatSamples()
feature, label, SCADA = load_mat_samples.load('pf')
ava_idx = load_mat_samples.ava_idx
if SAVE:
feature_train = feature[:-TEST_SIZE]
feature_test = feature[-TEST_SIZE:]
label_train = label[:-TEST_SIZE]
label_test = label[-TEST_SIZE:]
features_batches_train, labels_batches_train = gen_batches(
feature_train, label_train, BATCH_SIZE)
gcn = PF_GCN(DAD).to(device) # get object of the GCN with matrix DAD.
criterion = nn.MSELoss()
# print(features_batches_train.shape)
loss = 0.1
pf_loss = []
for iepoch in range(EPOCHES):
for ibatch in range(len(features_batches_train)):
step = float(loss / 500)
optimizer = optim.Adam(gcn.parameters(), lr=step)
optimizer.zero_grad()
output = gcn(features_batches_train[ibatch].to(device))
loss = criterion(output, labels_batches_train[ibatch].to(device))
loss.backward()
optimizer.step()
def main(_):
##
# Load Data
##
with open(FLAGS.data_path, 'r') as f:
reader = csv.reader(f)
# data is a list of tuples (img path, steering angle)
data = np.array([row for row in reader])
# Split train and validation data
np.random.shuffle(data)
split_i = int(len(data) * 0.9)
X_train, y_train = list(zip(*data[:split_i]))
X_val, y_val = list(zip(*data[split_i:]))
X_train, y_train = np.array(X_train), np.array(y_train)
X_val, y_val = np.array(X_val), np.array(y_val)
##
# Define Model
##
model = Sequential([
Conv2D(32, (3, 3),
input_shape=(64, 32, 1),
padding='same',
activation='relu'),
Conv2D(64, (3, 3), padding='same', activation='relu'),
MaxPooling2D(),
Dropout(0.5),
Conv2D(128, (3, 3), padding='same', activation='relu'),
Conv2D(256, (3, 3), padding='same', activation='relu'),
MaxPooling2D(),
Dropout(0.5),
Flatten(),
Dense(1024, activation='relu'),
# Dense(512, activation='relu'),
# Dense(128, activation='relu'),
Dense(1, name='output', activation='tanh'),
])
model.compile(optimizer=Adam(lr=FLAGS.lrate), loss='mse')
##
# Train
##
history = model.fit_generator(
gen_batches(X_train, y_train, FLAGS.batch_size),
len(X_train),
FLAGS.num_epochs,
validation_data=gen_batches(X_val, y_val, FLAGS.batch_size),
validation_steps=(len(X_val) / FLAGS.batch_size))
##
# Save model
##
json = model.to_json()
model.save_weights('save/sdc-mjc+mrg2/model.h5')
with open('save/sdc-mjc+mrg2/model.json', 'w') as f:
f.write(json)
# Some information about the data after splitting
print("Splitting with split rate: ", SPLIT)
print("Training Data Size: ", X_train.shape, y_train.shape)
print("Validation Data Size: ", X_valid.shape, y_valid.shape)
# Freeing the memory block cause you know, it needs to be free.
data = None
X = None
y = None
ans = str(input("Continue? ([Y]/N) --- "))
if (ans == 'N' or ans == 'n'):
exit()
# Generate training and validation data for the model compilation
train = utils.gen_batches(X_train, y_train, True, BATCH_SIZE)
valid = utils.gen_batches(X_valid, y_valid, False, BATCH_SIZE)
######################### Model Training ###########################
def nvidia(LR=1e-4, inputshape=(64, 64, 1), comp=False, summary=False):
model = Sequential()
model.add(Lambda(lambda x: x/127.5-1.0, input_shape=inputshape))
model.add(Conv2D(24, 5, 5, subsample=(2, 2), border_mode='valid', W_regularizer=l2(0.001)))
model.add(ELU())
model.add(Conv2D(36, 5, 5, subsample=(2, 2), border_mode='valid', W_regularizer=l2(0.001)))
model.add(ELU())
model.add(Conv2D(48, 5, 5, subsample=(2, 2), border_mode='valid', W_regularizer=l2(0.001)))
model.add(ELU())
model.add(Conv2D(64, 3, 3, border_mode='valid', W_regularizer=l2(0.001)))
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument("--train_file", type=str, default="", help="Input train file.")
parser.add_argument("--vocab_file", type=str, default="", help="Input vocab file.")
parser.add_argument("--model_save_dir", type=str, default="",
help="Specified the directory in which the model should stored.")
parser.add_argument("--lstm_dim", type=int, default=100, help="Dimension of LSTM cell.")
parser.add_argument("--embedding_dim", type=int, default=100, help="Dimension of word embedding.")
parser.add_argument("--layer_num", type=int, default=2, help="LSTM layer num.")
parser.add_argument("--batch_size", type=int, default=32, help="Batch size.")
parser.add_argument("--train_step", type=int, default=10000, help="Number of training steps.")
parser.add_argument("--warmup_step", type=int, default=1000, help="Number of warmup steps.")
parser.add_argument("--learning_rate", type=float, default=0.001, help="The initial learning rate")
parser.add_argument("--dropout_rate", type=float, default=0.5, help="Dropout rate")
parser.add_argument("--seed", type=int, default=0, help="Random seed value.")
parser.add_argument("--print_step", type=int, default=1000, help="Print log every x step.")
parser.add_argument("--max_predictions_per_seq", type=int, default=10,
help="For each sequence, predict x words at most.")
parser.add_argument("--weight_decay", type=float, default=0, help='Weight decay rate')
parser.add_argument("--clip_norm", type=float, default=1, help='Clip normalization rate.')
parser.add_argument("--max_seq_len", type=int, default=100, help="Max seqence length.")
parser.add_argument("--use_queue", type=int, default=0, help="Whether or not using a queue for input.")
parser.add_argument("--init_checkpoint", type=str, default="", help="Initial checkpoint")
parser.add_argument("--enqueue_thread_num", type=int, default=5, help="Enqueue thread count.")
args = parser.parse_args()
np.random.seed(args.seed)
tf.set_random_seed(args.seed)
tf.logging.info(args)
if not os.path.exists(args.model_save_dir):
os.mkdir(args.model_save_dir)
# load data
word2id, id2word = utils.load_vocab_file(args.vocab_file)
training_sens = utils.load_pretraining_data(args.train_file, args.max_seq_len)
if not args.use_queue:
utils.to_ids(training_sens, word2id, args, id2word)
other_arg_dict = {}
other_arg_dict['token_num'] = len(word2id)
# load model
model = models.create_bidirectional_lm_training_op(args, other_arg_dict)
gc.collect()
saver = tf.train.Saver(max_to_keep=2)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
if args.init_checkpoint:
tf.logging.info('restore the checkpoint : ' + str(args.init_checkpoint))
saver.restore(sess, args.init_checkpoint)
total_loss = 0
num = 0
global_step = 0
while global_step < args.train_step:
if not args.use_queue:
iterator = utils.gen_batches(training_sens, args.batch_size)
else:
iterator = utils.queue_gen_batches(training_sens, args, word2id, id2word)
for batch_data in iterator:
feed_dict = {model.ph_tokens: batch_data[0],
model.ph_length: batch_data[1],
model.ph_labels: batch_data[2],
model.ph_positions: batch_data[3],
model.ph_weights: batch_data[4],
model.ph_dropout_rate: args.dropout_rate}
_, global_step, loss, learning_rate = sess.run([model.train_op, \
model.global_step, model.loss_op,
model.learning_rate_op], feed_dict=feed_dict)
total_loss += loss
num += 1
if global_step % args.print_step == 0:
tf.logging.info("\nglobal step : " + str(global_step) +
", avg loss so far : " + str(total_loss / num) +
", instant loss : " + str(loss) +
", learning_rate : " + str(learning_rate) +
", time :" + str(time.strftime('%Y-%m-%d %H:%M:%S')))
tf.logging.info("save model ...")
saver.save(sess, args.model_save_dir + '/lm_pretrain.ckpt', global_step=global_step)
gc.collect()
if not args.use_queue:
utils.to_ids(training_sens, word2id, args, id2word) # MUST run this for randomization for each sentence
gc.collect()
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import numpy as np
import tensorflow as tf
from tensorflow import keras
import threading
from network import AGAN
from utils import to_array, gen_batches
BATCH_SIZE = 25
paths = os.listdir('assets/')
training_data = gen_batches(paths, BATCH_SIZE)
print('Done preparing training data')
aGAN = AGAN(noise_size=200, batch_size=BATCH_SIZE)
aGAN.restore()
try:
aGAN.train(dataset=training_data,
epochs=12000,
example_interval=5,
save_interval=100)
except KeyboardInterrupt:
try:
print('Saving...')
aGAN.save()
print('Clearing session...')
def _fit(self, X, y):
hidden_layer_sizes = self.hidden_layer_sizes
if not hasattr(hidden_layer_sizes, '__iter__'):
hidden_layer_sizes = [hidden_layer_sizes]
hidden_layer_sizes = list(hidden_layer_sizes)
self._validate_hyperparams(X, y)
X, y = self._validate_input(X, y)
# make y 2D
if len(y.shape) == 1:
y.reshape(y.shape[0], 1)
# tell the size of every layers
layer_sizes = [X.shape[1]]
layer_sizes.extend(hidden_layer_sizes)
layer_sizes.append(y.shape[1])
self._initilize(layer_sizes)
packed_params = self._pack_W_b(self.W_, self.b_)
# trainning params assignment
batch_size = self.batch_size
learning_rate = self.learning_rate
momentum = self.momentum
tol = self.tol
verbose = self.verbose
velocities = np.zeros_like(packed_params)
self.loss_ = []
for i in range(self.max_iter):
epoch_loss = 0
for batch_slice in gen_batches(X.shape[0], batch_size):
_X, _y = X[batch_slice], y[batch_slice]
activations = self._forward_pass(_X, self.W_, self.b_)
loss, grad_W, grad_b = self._backprop(_X, _y, self.W_, self.b_,
activations)
packed_grad = self._pack_W_b(grad_W, grad_b)
packed_params = self._pack_W_b(self.W_, self.b_)
# apply gradients
updates = momentum * velocities - learning_rate * packed_grad
velocities = updates
packed_params = packed_params + updates
self.W_, self.b_ = self._unpack_W_b(packed_params, self.W_,
self.b_)
epoch_loss += loss * (batch_slice.stop - batch_slice.start)
epoch_loss /= X.shape[0]
self.loss_.append(epoch_loss)
# two continuous epoch get loss changes lower than tol
if i > 2 and \
(self.loss_[i] - self.loss_[i-1] > -tol) and \
(self.loss_[i-1] - self.loss_[i-2] > -tol):
break
if verbose:
print('Iteration {:d}, loss = {:f}'.format(i + 1, epoch_loss))
return self
def slice_into_batches(self, Input_list):
produced_batches = []
length = len(Input_list)
for each in gen_batches(length, 100):
produced_batches.append(Input_list[each])
return produced_batches
def _train_model(self, train_X, train_y, validation_X, validation_y,
test_X, test_y):
"""
This function perform the training process
"""
feed_train = {
self.input_data: train_X,
self.input_labels: train_y,
self.keep_prob: self.finetune_dropout
}
feed_validation = {
self.input_data: validation_X,
self.input_labels: validation_y,
self.keep_prob: 1
}
feed_test = {
self.input_data: test_X,
self.input_labels: test_y,
self.keep_prob: 1
}
shuff = list(zip(train_X, train_y))
old_iter, old_accuracy = 0, 0
for iteration in range(1, self.finetune_num_epochs + 1):
np.random.shuffle(shuff)
batches = [_ for _ in gen_batches(shuff, self.finetune_batch_size)]
for batch in batches:
X_batch, y_batch = zip(*batch)
feed_batch = {
self.input_data: X_batch,
self.input_labels: y_batch,
self.keep_prob: self.finetune_dropout
}
self.sess.run(self.train_step, feed_dict=feed_batch)
updates = [self.loss, self.accuracy]
loss_train, accuracy_train = self.sess.run(updates,
feed_dict=feed_train)
print('Iter %d: ' % iteration, end='')
if validation_X is not None:
loss_val, accuracy_val = self.sess.run(
updates, feed_dict=feed_validation)
loss_test, accuracy_test = self.sess.run(updates,
feed_dict=feed_test)
print('Training: current loss %f' % loss_train,
'current accuracy %f' % accuracy_train,
sep=' | ',
end=' || ')
print('Validation: current loss %f' % loss_val,
'current accuracy %f' % accuracy_val,
sep=' | ',
end=' || ')
print('Test: accuracy %f' % accuracy_test)
self.accuracy_summary.append(accuracy_test)
self.loss_summary.append(loss_test)
if old_accuracy > accuracy_test and (iteration - old_iter) > 5:
return
if accuracy_test > old_accuracy:
old_accuracy = accuracy_test
old_iter = iteration
else:
print('Training: current loss %f' % loss_train,
'current accuracy %f' % accuracy_train,
sep=' | ')
