def plot_scatter(exp_batch, list_of_experiments, data_params, processing_params, plot_params, out_folder=None): """ Creates a scatter plot for the pairs of validation and driving evaluation. Computes the average of several offline metric and correlates each of these metrics to driving quality metric. The pairs of offline metrics and driving quality metrics that are going to be plotted are seton the plotting params files. Args: exp_batch: list_of_experiments: data_params: processing_params: plot_params: out_folder: Returns: """ # create a folder, in case it is none if out_folder is None: out_folder = time.strftime("plots_%Y_%m_%d_%H_%M_%S", time.gmtime()) print("out path") print(data_params['root_path'], exp_batch, 'plots', out_folder) out_path = os.path.join(data_params['root_path'], exp_batch, 'plots', out_folder) if not os.path.exists(out_path): os.makedirs(out_path) else: shutil.rmtree(out_path) os.makedirs(out_path) # save the parameters print("Out path", out_path) with open(os.path.join(out_path, 'params.txt'), 'w') as f: f.write('list_of_experiments:\n' + pprint.pformat(list_of_experiments, indent=4)) f.write('\n\ndata_params:\n' + pprint.pformat(data_params, indent=4)) f.write('\n\nprocessing_params:\n' + pprint.pformat(processing_params, indent=4)) f.write('\n\nplot_params:\n' + pprint.pformat(plot_params, indent=4)) list_of_exps_names = get_names(exp_batch) list_of_experiments = [ experiment.split('.')[-2] for experiment in list_of_experiments ] # The all metrics is a dictionary containing all the computed metrics for each pair # validation/drive to be computed. all_metrics = {} # Lets cache the data to improve read speed # TODO cache # TODO: add some loadbar here. for experiment in list_of_experiments: # The only thing that matters for validation, drive in data_params['validation_driving_pairs'].items( ): print( '\n === Experiment %s _ %s %s ===\n' % (list_of_exps_names[experiment + '.yaml'], validation, drive)) print('\n ** Reading the data **\n') # this reads the data and infers the masks (or offsets) for different cameras data = read_data(exp_batch, experiment, validation, drive, data_params) if data is None: # This folder did not work out, probably is missing important data print('\n ** Missing Data on Folder **\n') continue # Print data logging.debug(" %s_%s " % (validation, drive)) for step, data_item in data['values'].items(): logging.debug(" %d" % step) for k, v in data_item.items(): logging.debug("%s %d" % (k, len(v))) print('\n ** Processing the data **\n') computed_metrics = process_data(data, processing_params, validation) # Compute metrics from the data. # Can be multiple metrics, given by the processing_params list. # Should be vectorized as much as possible. # The output is a list of the same size as processing_params. metrics_dict = { experiment + ' : ' + list_of_exps_names[experiment + '.yaml']: computed_metrics } if validation + '_' + drive in all_metrics: all_metrics[validation + '_' + drive].update(metrics_dict) else: all_metrics.update({validation + '_' + drive: metrics_dict}) print(all_metrics) # Plot the results for validation, drive in data_params['validation_driving_pairs'].items(): with open( os.path.join( out_path, 'all_metrics' + validation + '_' + drive + '.json'), 'w') as fo: fo.write(json.dumps(all_metrics[validation + '_' + drive])) print('\n === Plotting the results ===\n') for plot_label, plot_param in plot_params.items(): print(plot_param) print('\n ** Plotting %s **' % plot_label) if 'analysis' in plot_param: scatter_plotter.plot_analysis( all_metrics[validation + '_' + drive], plot_param, out_file=os.path.join( out_path, plot_label + validation + '_' + drive + '.pdf')) else: scatter_plotter.plot(all_metrics[validation + '_' + drive], plot_param, out_file=os.path.join( out_path, plot_label + validation + '_' + drive + '.pdf'))
def plot_scatter(exp_batch, list_of_experiments, data_params, processing_params, plot_params, out_folder=None): # create a folder if out_folder is None: out_folder = time.strftime("plots_%Y_%m_%d_%H_%M_%S", time.gmtime()) out_path = os.path.join(data_params['root_path'], exp_batch, 'plots', out_folder) if not os.path.exists(out_path): os.makedirs(out_path) else: shutil.rmtree(out_path) os.makedirs(out_path) # save the parameters print("Out path", out_path) with open(os.path.join(out_path, 'params.txt'), 'w') as f: f.write('list_of_experiments:\n' + pprint.pformat(list_of_experiments, indent=4)) f.write('\n\ndata_params:\n' + pprint.pformat(data_params, indent=4)) f.write('\n\nprocessing_params:\n' + pprint.pformat(processing_params, indent=4)) f.write('\n\nplot_params:\n' + pprint.pformat(plot_params, indent=4)) all_metrics = {} list_of_exps_names = get_names(exp_batch) print('list, expnames', list_of_exps_names) list_of_experiments = [ experiment.split('.')[-2] for experiment in list_of_experiments ] for experiment in list_of_experiments: #if '25_nor_no' in experiment or '5_small_ndrop_single_wp' in experiment: # continue for town in data_params['towns']: print('\n === Experiment %s _ %s %s ===\n' % (list_of_exps_names[list_of_experiments.index(experiment)], town, data_params['noise'])) print('\n ** Reading the data **\n') data = read_data( exp_batch, experiment, town, data_params, data_params['noise'] ) # this reads the data and infers the masks (or offsets) for different cameras if data is None: # This folder didnt work out, probably is missing important data print('\n ** Missing Data on Folder **\n') continue # Print data print(data['town']) for step, data_item in data['values'].items(): print(step) for k, v in data_item.items(): print(k, len(v)) print('\n ** Processing the data **\n') metrics = process_data( data, processing_params, data_params['noise'] ) # Compute metrics from the data. Can be multiple metrics, given by the processing_params list. Should be vectorized as much as possible. The output is a list of the same size as processing_params. all_metrics[ experiment + ' : ' + list_of_exps_names[list_of_experiments.index(experiment)] + '_' + town] = metrics # append to the computed list of metrics to the dictionary of results. with open(os.path.join(out_path, 'all_metrics.txt'), 'w') as f: f.write('all_metrics:\n' + pprint.pformat(all_metrics, indent=4)) # Plot the results print('\n === Plotting the results ===\n') for plot_label, plot_param in plot_params.items(): print(plot_param) print('\n ** Plotting %s **' % plot_label) if 'analysis' in plot_param: make_scatter_plot_analysis(all_metrics, plot_param, out_file=os.path.join( out_path, plot_label + '.pdf')) else: make_scatter_plot(all_metrics, plot_param, out_file=os.path.join(out_path, plot_label + '.pdf'))
def plot_folder_summaries(exp_batch, train, validation_datasets, drive_environments, verbose=False): """ Main plotting function for the folder mode. Args: exp_batch: The exp batch (folder) being plotted on the screen. train: If train process is being printed validation_datasets: The validation datasets being computed drive_environments: The driving environments/ Benchmarks verbose: Returns: None """ os.system('clear') process_names = [] if train: process_names.append('train') for val in validation_datasets: process_names.append('validation' + '_' + val) # We save the drive files to be used later drive_files = {} for drive in drive_environments: drive_files.update({ ('drive' + '_' + drive.split('/')[-1].split('.')[0]): drive }) process_names.append('drive' + '_' + drive.split('/')[-1].split('.')[0]) experiments_list = os.listdir(os.path.join('configs', exp_batch)) experiments_list = [ experiment.split('.')[-2] for experiment in experiments_list ] names_list = get_names(exp_batch) sorted_keys = sorted( range(len(names_list)), key=lambda k: names_list[experiments_list[k] + '.yaml']) for key in sorted_keys: experiment = experiments_list[key] generated_name = names_list[experiment + '.yaml'] if experiment == '': raise ValueError("Empty Experiment on List") g_conf.immutable(False) merge_with_yaml( os.path.join('configs', exp_batch, experiment + '.yaml')) print(BOLD + experiment + ' : ' + generated_name + END) for process in process_names: try: output = get_status(exp_batch, experiment, process) except: # TODO: bad design. But the printing should not stop for any error on reading import traceback traceback.print_exc() status = output[0] summary = output[1] print(' ', process) if status == 'Not Started': print(' STATUS: ', BOLD + status + END) elif status == 'Loading': print(' STATUS: ', YELLOW + status + END, ' - ', YELLOW + summary + END) elif status == 'Iterating': print(' STATUS: ', YELLOW + status + END) elif status == 'Finished': print(' STATUS: ', GREEN + status + END) elif status == 'Error': print(' STATUS: ', RED + status + END, ' - ', RED + summary + END) if status == 'Iterating': if 'train' == process: print_train_summary(summary[status]) if 'validation' in process: if summary[1] != '': # If it has no summary we don't plot print_validation_summary(summary[0][status], summary[1][status]['Summary'], verbose) else: print_validation_summary(summary[0][status], '', verbose) if 'drive' in process: if 'Agent' not in summary[status]: continue checkpoint = summary[status]['Checkpoint'] # Get the sta # This contain the results from completed iterations # we read the json file directly agent_checkpoint_name = str(exp_batch) + '_' + str( experiment) + '_' + str(checkpoint) print_drive_summary(drive_files[process], agent_checkpoint_name, checkpoint)
def export_status(exp_batch, validation_datasets, driving_environments): root_path = '_logs' experiments = os.listdir(os.path.join(root_path, exp_batch)) # Make the header of the exported csv csv_outfile = os.path.join(root_path, exp_batch, 'status.csv') with open(csv_outfile, 'w') as f: f.write("experiment_alias,experiment_name") for validation in validation_datasets: f.write("," + validation) for driving in driving_environments: f.write("," + driving) f.write('\n') names_list = get_names(exp_batch) count = 0 print(names_list) for exp in experiments: if os.path.isdir(os.path.join(root_path, exp_batch, exp)): f.write("%s,%s" % (exp, names_list[exp + '.yaml'])) count += 1 print(exp) merge_with_yaml( os.path.join('configs', exp_batch, exp + '.yaml')) experiments_logs = os.listdir( os.path.join(root_path, exp_batch, exp)) for validation in validation_datasets: log = 'validation_' + validation + '_csv' if log in os.listdir( os.path.join(root_path, exp_batch, exp)): if (str(g_conf.TEST_SCHEDULE[-1]) + '.csv') in os.listdir( os.path.join(root_path, exp_batch, exp, log)): f.write(",Done") else: f.write(", ") else: f.write(", ") for driving in driving_environments: log = 'drive_' + driving + '_csv' if log in os.listdir( os.path.join(root_path, exp_batch, exp)): if g_conf.USE_ORACLE: output_name = 'control_output_auto.csv' else: output_name = 'control_output.csv' print(log) if output_name in os.listdir( os.path.join(root_path, exp_batch, exp, log)): csv_file_path = os.path.join( root_path, exp_batch, exp, log, output_name) control_csv = read_summary_csv(csv_file_path) if control_csv is not None and ( g_conf.TEST_SCHEDULE[-1]) == int( control_csv['step'][-1]): f.write(",Done") else: f.write(", ") else: f.write(", ") else: f.write(", ") f.write("\n")