# return sorted(os.listdir('train_history/'), key=lambda x: os.path.getmtime(os.path.join('train_history', x)), reverse=True)[0] if options.config_file: config_file = options.config_file else: # config_file = get_most_recent_file_from_dir('train_history/') config_file = choose_file_with_stdin('train_history/') print("Config file: {}".format(config_file)) cm = ConfigManager(config_file, folder='train_history') cfg = cm.get_json() model = get_model_and_load_weights(cfg) compile_model(model, cfg['training']) data = Data(options.test_path, cfg['training'], train_test='test') print("Testing on {} samples".format(data.test_len)) data_gen = data.test_generator() displayer = Display() for frame_batch in data_gen: print('Frame batch') pred = model.predict_on_batch(frame_batch) # print pred H = np.reshape(pred, (3,3)) print('Prediction:\n{}'.format(H)) frame = np.array(frame_batch[0]*255).astype('uint8') warped = displayer.get_warp_overlay(frame, H, dsize=(300,300)) displayer.display_vstacked(frame, warped, dsize=(600,600)) # displayer.display_warp_overlay(frame, H, dsize=(600,600)) print 'done'
def evaluate(csv_file, eval_path="", s3_bucket="bsivisiondata", train_config=None, write_output_images=False): ''' eval_path: top directory, look for frames/ and annotations/ underneath here. If csv specified, then paths will be relative to this eval_path csv_file: a csv of paths_to_images,labels. Paths are relative to eval_path s3_bucket: name of an s3 bucket where image and label data is read from. If csv is specified, labels will be taken from the csv train_config: a .json config file from train_history that gives information on the training job that we're evaluating write_output_images: bool, whether or not to write output images to data/eval_images/training_job_name_steps loop_eval_period: int, the period in seconds to loop evaluation runs. If not set, only runs once ''' if train_config: config_file = train_config else: # config_file = get_most_recent_file_from_dir('train_history/') config_file = choose_file_with_stdin('train_history/') print("Config file: {}".format(config_file)) cm = ConfigManager(config_file, folder='train_history') cfg = cm.get_json() data = DataExtractor(eval_path, s3_bucket, csv_file=csv_file) # batch_size = cfg['training']['hyperparameters']['batch_size'] batch_size = 1 target_shape = cfg['training']['model']['target_shape'] data_gen = data.get_processed_data_generator(True, batch_size, target_shape, True) print("Testing on {} samples".format(data.get_data_length())) displayer = Display() model, path_and_prefix = get_model_and_load_weights(cfg) compile_model(model, cfg['training']) ckpt_prefix = os.path.basename(path_and_prefix) total_loss = 0 num_testing_samples = 0 for X, y, filepaths, originals in data_gen: pred = model.predict_on_batch(X) mse_out = mean_squared_error(y, pred) loss = mse_out.eval(session=K.get_session()) total_loss += loss[0] num_testing_samples += 1 print("Loss: {}".format(loss[0])) H = np.reshape(pred, (3,3)) # print('Prediction:\n{}'.format(H)) if write_output_images: frame = originals[0] filepath = filepaths[0] image_text = "Pred: {}\nTruth: {}\nLoss: {}".format(np.array_str(pred), np.array_str(y[0]), str(loss[0])) warped = displayer.get_warp_overlay(frame, H, dsize=(1000,425)) vstacked = displayer.get_vstacked(frame, warped, dsize=(1000,850)) displayer.put_rectangle(vstacked, (10, 10), (800, 80)) displayer.put_text(vstacked, "Pred: {}".format(np.array_str(pred, precision=2)), text_loc=(10,30)) displayer.put_text(vstacked, "Truth: {}".format(np.array_str(y[0], precision=2)), text_loc=(10,50)) displayer.put_text(vstacked, "loss: {}".format(loss[0]), text_loc=(10,70)) step_number = path_and_prefix.split('-')[-1] job_name_and_step_num = cfg['sagemaker_job_info']['job_name'] + '-step' + str(step_number) job_name_dir = os.path.join('data/eval_images', job_name_and_step_num) if not os.path.exists(job_name_dir): os.makedirs(job_name_dir) filename = "{:.2E}-{}".format(loss[0], os.path.basename(filepath)) displayer.save_image(os.path.join(job_name_dir, filename), vstacked) # displayer.display_vstacked(frame, warped, dsize=(1000,850)) # displayer.display_warp_overlay(frame, H, dsize=(600,600)) average_loss = total_loss / num_testing_samples print("Average loss: {}".format(average_loss)) return average_loss, path_and_prefix