def __init__(self, data_path, epochs=50, max_sample_records=500): self.data_path = data_path self.epochs = int(epochs) self.max_sample_records = max_sample_records tfboard_basedir = mkdir(data_path + '/tf_visual_data/runs/') tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir) model_checkpoint_path = mkdir(tfboard_run_dir + '/trained_model')
def __init__(self, data_path, model_file, s3_bucket, epochs=50, max_sample_records=500, start_epoch=0, restored_model=False, restored_model_dir=None, tf_timeline=False): self.data_path = data_path self.s3_bucket = format_s3_bucket(s3_bucket) self.s3_data_dir = format_s3_data_dir(self.s3_bucket) self.model_file = model_file self.n_epochs = int(epochs) self.max_sample_records = max_sample_records self.tf_timeline = tf_timeline # Always sync before training in case I ever train multiple models in parallel sync_from_aws(s3_path=self.s3_data_dir, local_path=self.data_path) if restored_model: self.model_dir = restored_model_dir else: self.tfboard_basedir = os.path.join(self.data_path, 'tf_visual_data', 'runs') self.model_dir = mkdir_tfboard_run_dir(self.tfboard_basedir) self.results_file = os.path.join(self.model_dir, 'results.txt') self.model_checkpoint_dir = os.path.join(self.model_dir, 'checkpoints') self.saver = tf.train.Saver() self.start_epoch = start_epoch self.restored_model = restored_model mkdir(self.model_checkpoint_dir)
def __init__(self, data_path, model_file, epochs=50, max_sample_records=500): self.model_file = model_file self.data_path = data_path self.epochs = int(epochs) self.max_sample_records = max_sample_records self.tfboard_basedir = os.path.join(self.data_path, 'tf_visual_data', 'runs') self.tfboard_run_dir = mkdir_tfboard_run_dir(self.tfboard_basedir) self.results_file = os.path.join(self.tfboard_run_dir, 'results.txt') model_checkpoint_path = os.path.join(self.tfboard_run_dir,'trained_model')
def __init__(self, data_path, model_file, s3_bucket, epochs=50, max_sample_records=500, start_epoch=0, restored_model=False, restored_model_dir=None, tf_timeline=False, show_speed=False, s3_sync=True): self.data_path = data_path self.s3_bucket = format_s3_bucket(s3_bucket) self.s3_data_dir = format_s3_data_dir(self.s3_bucket) self.model_file = model_file self.n_epochs = int(epochs) self.max_sample_records = max_sample_records self.tf_timeline = tf_timeline self.s3_sync = s3_sync # Always sync before training in case I ever train multiple models in parallel if self.s3_sync is True: # You have the option to turn off the sync during development to save disk space sync_from_aws(s3_path=self.s3_data_dir, local_path=self.data_path) if restored_model: self.model_dir = restored_model_dir else: self.tfboard_basedir = os.path.join(self.data_path, 'tf_visual_data', 'runs') self.model_dir = mkdir_tfboard_run_dir(self.tfboard_basedir) self.results_file = os.path.join(self.model_dir, 'results.txt') self.speed_file = os.path.join(self.model_dir, 'speed.txt') self.model_checkpoint_dir = os.path.join(self.model_dir, 'checkpoints') self.saver = tf.train.Saver() self.start_epoch = start_epoch self.restored_model = restored_model mkdir(self.model_checkpoint_dir) # Prints batch processing speed, among other things self.show_speed = show_speed
def __init__(self, data_path, model_file, epochs=50, max_sample_records=500, start_epoch=0): self.model_file = model_file self.data_path = data_path self.n_epochs = int(epochs) self.max_sample_records = max_sample_records self.tfboard_basedir = os.path.join(self.data_path, 'tf_visual_data', 'runs') self.tfboard_run_dir = mkdir_tfboard_run_dir(self.tfboard_basedir) self.results_file = os.path.join(self.tfboard_run_dir, 'results.txt') self.model_checkpoint_dir = os.path.join(self.tfboard_run_dir, 'checkpoints') self.saver = tf.train.Saver() self.start_epoch = start_epoch mkdir(self.model_checkpoint_dir)
# python3 /root/Self-Driving-Car/train_1x1_conv_net.py -d /root -b 1400 ap = argparse.ArgumentParser() ap.add_argument("-d", "--datapath", required = False, help = "path to all of the data", default='/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car') ap.add_argument("-b", "--batches", required = False, help = "quantity of batch iterations to run", default='1000') args = vars(ap.parse_args()) data_path = args["datapath"] batch_iterations = int(args["batches"]) input_file_path = data_path+'/data_115.npz' tfboard_basedir = mkdir(data_path+'/tf_visual_data/runs/') tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir) model_checkpoint_path = mkdir(tfboard_run_dir+'/trained_model') npzfile = np.load(input_file_path) # training data train_predictors = npzfile['train_predictors'] train_targets = npzfile['train_targets'] train_predictors, train_targets = shuffle_dataset(train_predictors, train_targets) # validation/test data validation_predictors = npzfile['validation_predictors'] validation_targets = npzfile['validation_targets'] validation_predictors, validation_targets = shuffle_dataset(validation_predictors, validation_targets)
def train(self, sess, x, y_, accuracy, train_step, train_feed_dict, test_feed_dict): # To view graph: tensorboard --logdir=/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car/tf_visual_data/runs tf.scalar_summary('accuracy', accuracy) merged = tf.merge_all_summaries() tfboard_basedir = '/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car/tf_visual_data/runs/' tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir) # Archive this script to document model design in event of good results that need to be replicated model_file_path = os.path.dirname( os.path.realpath(__file__)) + '/' + os.path.basename(__file__) cmd = 'cp {model_file} {archive_path}' shell_command( cmd.format(model_file=model_file_path, archive_path=tfboard_run_dir + '/')) sess.run(tf.initialize_all_variables()) input_file_path = os.path.join(self.data_path, 'data') dataset = Dataset(input_file_path=input_file_path, max_sample_records=self.max_sample_records) # Not sure what these two lines do run_opts = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_opts_metadata = tf.RunMetadata() train_images, train_labels = process_data( dataset.get_sample(train=True)) train_feed_dict[x] = train_images train_feed_dict[y_] = train_labels train_summary, train_accuracy = sess.run( [merged, accuracy], feed_dict=train_feed_dict, options=run_opts, run_metadata=run_opts_metadata) test_images, test_labels = process_data( dataset.get_sample(train=False)) test_feed_dict[x] = test_images test_feed_dict[y_] = test_labels test_summary, test_accuracy = sess.run([merged, accuracy], feed_dict=test_feed_dict, options=run_opts, run_metadata=run_opts_metadata) message = "epoch: {0}, training accuracy: {1}, validation accuracy: {2}" print(message.format(-1, train_accuracy, test_accuracy)) for epoch in range(self.epochs): train_batches = dataset.get_batches(train=True) for batch in train_batches: images, labels = process_data(batch) train_feed_dict[x] = images train_feed_dict[y_] = labels train_step.run(feed_dict=train_feed_dict) # TODO: remove all this hideous boilerplate run_opts = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_opts_metadata = tf.RunMetadata() train_images, train_labels = process_data( dataset.get_sample(train=True)) train_feed_dict[x] = train_images train_feed_dict[y_] = train_labels train_summary, train_accuracy = sess.run( [merged, accuracy], feed_dict=train_feed_dict, options=run_opts, run_metadata=run_opts_metadata) test_images, test_labels = process_data( dataset.get_sample(train=False)) test_feed_dict[x] = test_images test_feed_dict[y_] = test_labels test_summary, test_accuracy = sess.run( [merged, accuracy], feed_dict=test_feed_dict, options=run_opts, run_metadata=run_opts_metadata) print(message.format(epoch, train_accuracy, test_accuracy)) # Save the trained model to a file saver = tf.train.Saver() save_path = saver.save(sess, tfboard_run_dir + "/model.ckpt") # Marks unambiguous successful completion to prevent deletion by cleanup script shell_command('touch ' + tfboard_run_dir + '/SUCCESS')
help="Increase training size by flipping left and right images", default='y') ap.add_argument("-d", "--debug", required=False, help="Print more details for easier debugging", default='n') args = vars(ap.parse_args()) data_path = args["datapath"] batch_iterations = int(args["batches"]) perform_image_flipping = True if args["flip"].lower() == 'y' else False debug_mode = True if args["debug"].lower() == 'y' else False input_file_path = data_path + '/final_processed_data_3_channels.npz' tfboard_basedir = mkdir(data_path + '/tf_visual_data/runs/') tfboard_run_dir = mkdir_tfboard_run_dir(tfboard_basedir) model_checkpoint_path = mkdir(tfboard_run_dir + '/trained_model') npzfile = np.load(input_file_path) # training data train_predictors = npzfile['train_predictors'] train_targets = npzfile['train_targets'] # validation/test data validation_predictors = npzfile['validation_predictors'] validation_targets = npzfile['validation_targets'] validation_predictors, validation_targets = shuffle_dataset( validation_predictors, validation_targets) sess = tf.InteractiveSession(config=tf.ConfigProto())