def __init__(self, data_path, epochs=50, max_sample_records=500):
self.data_path = data_path
self.epochs = int(epochs)
self.max_sample_records = max_sample_records
tfboard_basedir = mkdir(data_path + '/tf_visual_data/runs/')
tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir)
model_checkpoint_path = mkdir(tfboard_run_dir + '/trained_model')
def __init__(self,
data_path,
model_file,
s3_bucket,
epochs=50,
max_sample_records=500,
start_epoch=0,
restored_model=False,
restored_model_dir=None,
tf_timeline=False):
self.data_path = data_path
self.s3_bucket = format_s3_bucket(s3_bucket)
self.s3_data_dir = format_s3_data_dir(self.s3_bucket)
self.model_file = model_file
self.n_epochs = int(epochs)
self.max_sample_records = max_sample_records
self.tf_timeline = tf_timeline
# Always sync before training in case I ever train multiple models in parallel
sync_from_aws(s3_path=self.s3_data_dir, local_path=self.data_path)
if restored_model:
self.model_dir = restored_model_dir
else:
self.tfboard_basedir = os.path.join(self.data_path,
'tf_visual_data', 'runs')
self.model_dir = mkdir_tfboard_run_dir(self.tfboard_basedir)
self.results_file = os.path.join(self.model_dir, 'results.txt')
self.model_checkpoint_dir = os.path.join(self.model_dir, 'checkpoints')
self.saver = tf.train.Saver()
self.start_epoch = start_epoch
self.restored_model = restored_model
mkdir(self.model_checkpoint_dir)
def __init__(self, data_path, model_file, epochs=50, max_sample_records=500):
self.model_file = model_file
self.data_path = data_path
self.epochs = int(epochs)
self.max_sample_records = max_sample_records
self.tfboard_basedir = os.path.join(self.data_path, 'tf_visual_data', 'runs')
self.tfboard_run_dir = mkdir_tfboard_run_dir(self.tfboard_basedir)
self.results_file = os.path.join(self.tfboard_run_dir, 'results.txt')
model_checkpoint_path = os.path.join(self.tfboard_run_dir,'trained_model')
def __init__(self,
data_path,
model_file,
s3_bucket,
epochs=50,
max_sample_records=500,
start_epoch=0,
restored_model=False,
restored_model_dir=None,
tf_timeline=False,
show_speed=False,
s3_sync=True):
self.data_path = data_path
self.s3_bucket = format_s3_bucket(s3_bucket)
self.s3_data_dir = format_s3_data_dir(self.s3_bucket)
self.model_file = model_file
self.n_epochs = int(epochs)
self.max_sample_records = max_sample_records
self.tf_timeline = tf_timeline
self.s3_sync = s3_sync
# Always sync before training in case I ever train multiple models in parallel
if self.s3_sync is True: # You have the option to turn off the sync during development to save disk space
sync_from_aws(s3_path=self.s3_data_dir, local_path=self.data_path)
if restored_model:
self.model_dir = restored_model_dir
else:
self.tfboard_basedir = os.path.join(self.data_path,
'tf_visual_data', 'runs')
self.model_dir = mkdir_tfboard_run_dir(self.tfboard_basedir)
self.results_file = os.path.join(self.model_dir, 'results.txt')
self.speed_file = os.path.join(self.model_dir, 'speed.txt')
self.model_checkpoint_dir = os.path.join(self.model_dir, 'checkpoints')
self.saver = tf.train.Saver()
self.start_epoch = start_epoch
self.restored_model = restored_model
mkdir(self.model_checkpoint_dir)
# Prints batch processing speed, among other things
self.show_speed = show_speed
def __init__(self,
data_path,
model_file,
epochs=50,
max_sample_records=500,
start_epoch=0):
self.model_file = model_file
self.data_path = data_path
self.n_epochs = int(epochs)
self.max_sample_records = max_sample_records
self.tfboard_basedir = os.path.join(self.data_path, 'tf_visual_data',
'runs')
self.tfboard_run_dir = mkdir_tfboard_run_dir(self.tfboard_basedir)
self.results_file = os.path.join(self.tfboard_run_dir, 'results.txt')
self.model_checkpoint_dir = os.path.join(self.tfboard_run_dir,
'checkpoints')
self.saver = tf.train.Saver()
self.start_epoch = start_epoch
mkdir(self.model_checkpoint_dir)
# python3 /root/Self-Driving-Car/train_1x1_conv_net.py -d /root -b 1400
ap = argparse.ArgumentParser()
ap.add_argument("-d", "--datapath", required = False,
help = "path to all of the data",
default='/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car')
ap.add_argument("-b", "--batches", required = False,
help = "quantity of batch iterations to run",
default='1000')
args = vars(ap.parse_args())
data_path = args["datapath"]
batch_iterations = int(args["batches"])
input_file_path = data_path+'/data_115.npz'
tfboard_basedir = mkdir(data_path+'/tf_visual_data/runs/')
tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir)
model_checkpoint_path = mkdir(tfboard_run_dir+'/trained_model')
npzfile = np.load(input_file_path)
# training data
train_predictors = npzfile['train_predictors']
train_targets = npzfile['train_targets']
train_predictors, train_targets = shuffle_dataset(train_predictors, train_targets)
# validation/test data
validation_predictors = npzfile['validation_predictors']
validation_targets = npzfile['validation_targets']
validation_predictors, validation_targets = shuffle_dataset(validation_predictors, validation_targets)
def train(self, sess, x, y_, accuracy, train_step, train_feed_dict,
test_feed_dict):
# To view graph: tensorboard --logdir=/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car/tf_visual_data/runs
tf.scalar_summary('accuracy', accuracy)
merged = tf.merge_all_summaries()
tfboard_basedir = '/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car/tf_visual_data/runs/'
tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir)
# Archive this script to document model design in event of good results that need to be replicated
model_file_path = os.path.dirname(
os.path.realpath(__file__)) + '/' + os.path.basename(__file__)
cmd = 'cp {model_file} {archive_path}'
shell_command(
cmd.format(model_file=model_file_path,
archive_path=tfboard_run_dir + '/'))
sess.run(tf.initialize_all_variables())
input_file_path = os.path.join(self.data_path, 'data')
dataset = Dataset(input_file_path=input_file_path,
max_sample_records=self.max_sample_records)
# Not sure what these two lines do
run_opts = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_opts_metadata = tf.RunMetadata()
train_images, train_labels = process_data(
dataset.get_sample(train=True))
train_feed_dict[x] = train_images
train_feed_dict[y_] = train_labels
train_summary, train_accuracy = sess.run(
[merged, accuracy],
feed_dict=train_feed_dict,
options=run_opts,
run_metadata=run_opts_metadata)
test_images, test_labels = process_data(
dataset.get_sample(train=False))
test_feed_dict[x] = test_images
test_feed_dict[y_] = test_labels
test_summary, test_accuracy = sess.run([merged, accuracy],
feed_dict=test_feed_dict,
options=run_opts,
run_metadata=run_opts_metadata)
message = "epoch: {0}, training accuracy: {1}, validation accuracy: {2}"
print(message.format(-1, train_accuracy, test_accuracy))
for epoch in range(self.epochs):
train_batches = dataset.get_batches(train=True)
for batch in train_batches:
images, labels = process_data(batch)
train_feed_dict[x] = images
train_feed_dict[y_] = labels
train_step.run(feed_dict=train_feed_dict)
# TODO: remove all this hideous boilerplate
run_opts = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_opts_metadata = tf.RunMetadata()
train_images, train_labels = process_data(
dataset.get_sample(train=True))
train_feed_dict[x] = train_images
train_feed_dict[y_] = train_labels
train_summary, train_accuracy = sess.run(
[merged, accuracy],
feed_dict=train_feed_dict,
options=run_opts,
run_metadata=run_opts_metadata)
test_images, test_labels = process_data(
dataset.get_sample(train=False))
test_feed_dict[x] = test_images
test_feed_dict[y_] = test_labels
test_summary, test_accuracy = sess.run(
[merged, accuracy],
feed_dict=test_feed_dict,
options=run_opts,
run_metadata=run_opts_metadata)
print(message.format(epoch, train_accuracy, test_accuracy))
# Save the trained model to a file
saver = tf.train.Saver()
save_path = saver.save(sess, tfboard_run_dir + "/model.ckpt")
# Marks unambiguous successful completion to prevent deletion by cleanup script
shell_command('touch ' + tfboard_run_dir + '/SUCCESS')
help="Increase training size by flipping left and right images",
default='y')
ap.add_argument("-d",
"--debug",
required=False,
help="Print more details for easier debugging",
default='n')
args = vars(ap.parse_args())
data_path = args["datapath"]
batch_iterations = int(args["batches"])
perform_image_flipping = True if args["flip"].lower() == 'y' else False
debug_mode = True if args["debug"].lower() == 'y' else False
input_file_path = data_path + '/final_processed_data_3_channels.npz'
tfboard_basedir = mkdir(data_path + '/tf_visual_data/runs/')
tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir)
model_checkpoint_path = mkdir(tfboard_run_dir + '/trained_model')
npzfile = np.load(input_file_path)
# training data
train_predictors = npzfile['train_predictors']
train_targets = npzfile['train_targets']
# validation/test data
validation_predictors = npzfile['validation_predictors']
validation_targets = npzfile['validation_targets']
validation_predictors, validation_targets = shuffle_dataset(
validation_predictors, validation_targets)
sess = tf.InteractiveSession(config=tf.ConfigProto())
