def plot_scatter(exp_batch,
list_of_experiments,
data_params,
processing_params,
plot_params,
out_folder=None):
"""
Creates a scatter plot for the pairs of validation and driving evaluation.
Computes the average of several offline metric and correlates each
of these metrics to driving quality metric. The pairs of offline metrics and driving
quality metrics that are going to be plotted are seton the plotting params files.
Args:
exp_batch:
list_of_experiments:
data_params:
processing_params:
plot_params:
out_folder:
Returns:
"""
# create a folder, in case it is none
if out_folder is None:
out_folder = time.strftime("plots_%Y_%m_%d_%H_%M_%S", time.gmtime())
print("out path")
print(data_params['root_path'], exp_batch, 'plots', out_folder)
out_path = os.path.join(data_params['root_path'], exp_batch, 'plots',
out_folder)
if not os.path.exists(out_path):
os.makedirs(out_path)
else:
shutil.rmtree(out_path)
os.makedirs(out_path)
# save the parameters
print("Out path", out_path)
with open(os.path.join(out_path, 'params.txt'), 'w') as f:
f.write('list_of_experiments:\n' +
pprint.pformat(list_of_experiments, indent=4))
f.write('\n\ndata_params:\n' + pprint.pformat(data_params, indent=4))
f.write('\n\nprocessing_params:\n' +
pprint.pformat(processing_params, indent=4))
f.write('\n\nplot_params:\n' + pprint.pformat(plot_params, indent=4))
list_of_exps_names = get_names(exp_batch)
list_of_experiments = [
experiment.split('.')[-2] for experiment in list_of_experiments
]
# The all metrics is a dictionary containing all the computed metrics for each pair
# validation/drive to be computed.
all_metrics = {}
# Lets cache the data to improve read speed # TODO cache
# TODO: add some loadbar here.
for experiment in list_of_experiments:
# The only thing that matters
for validation, drive in data_params['validation_driving_pairs'].items(
):
print(
'\n === Experiment %s _ %s %s ===\n' %
(list_of_exps_names[experiment + '.yaml'], validation, drive))
print('\n ** Reading the data **\n')
# this reads the data and infers the masks (or offsets) for different cameras
data = read_data(exp_batch, experiment, validation, drive,
data_params)
if data is None: # This folder did not work out, probably is missing important data
print('\n ** Missing Data on Folder **\n')
continue
# Print data
logging.debug(" %s_%s " % (validation, drive))
for step, data_item in data['values'].items():
logging.debug(" %d" % step)
for k, v in data_item.items():
logging.debug("%s %d" % (k, len(v)))
print('\n ** Processing the data **\n')
computed_metrics = process_data(data, processing_params,
validation)
# Compute metrics from the data.
# Can be multiple metrics, given by the processing_params list.
# Should be vectorized as much as possible.
# The output is a list of the same size as processing_params.
metrics_dict = {
experiment + ' : ' + list_of_exps_names[experiment + '.yaml']:
computed_metrics
}
if validation + '_' + drive in all_metrics:
all_metrics[validation + '_' + drive].update(metrics_dict)
else:
all_metrics.update({validation + '_' + drive: metrics_dict})
print(all_metrics)
# Plot the results
for validation, drive in data_params['validation_driving_pairs'].items():
with open(
os.path.join(
out_path,
'all_metrics' + validation + '_' + drive + '.json'),
'w') as fo:
fo.write(json.dumps(all_metrics[validation + '_' + drive]))
print('\n === Plotting the results ===\n')
for plot_label, plot_param in plot_params.items():
print(plot_param)
print('\n ** Plotting %s **' % plot_label)
if 'analysis' in plot_param:
scatter_plotter.plot_analysis(
all_metrics[validation + '_' + drive],
plot_param,
out_file=os.path.join(
out_path,
plot_label + validation + '_' + drive + '.pdf'))
else:
scatter_plotter.plot(all_metrics[validation + '_' + drive],
plot_param,
out_file=os.path.join(
out_path, plot_label + validation +
'_' + drive + '.pdf'))
def plot_scatter(exp_batch,
list_of_experiments,
data_params,
processing_params,
plot_params,
out_folder=None):
# create a folder
if out_folder is None:
out_folder = time.strftime("plots_%Y_%m_%d_%H_%M_%S", time.gmtime())
out_path = os.path.join(data_params['root_path'], exp_batch, 'plots',
out_folder)
if not os.path.exists(out_path):
os.makedirs(out_path)
else:
shutil.rmtree(out_path)
os.makedirs(out_path)
# save the parameters
print("Out path", out_path)
with open(os.path.join(out_path, 'params.txt'), 'w') as f:
f.write('list_of_experiments:\n' +
pprint.pformat(list_of_experiments, indent=4))
f.write('\n\ndata_params:\n' + pprint.pformat(data_params, indent=4))
f.write('\n\nprocessing_params:\n' +
pprint.pformat(processing_params, indent=4))
f.write('\n\nplot_params:\n' + pprint.pformat(plot_params, indent=4))
all_metrics = {}
list_of_exps_names = get_names(exp_batch)
print('list, expnames', list_of_exps_names)
list_of_experiments = [
experiment.split('.')[-2] for experiment in list_of_experiments
]
for experiment in list_of_experiments:
#if '25_nor_no' in experiment or '5_small_ndrop_single_wp' in experiment:
# continue
for town in data_params['towns']:
print('\n === Experiment %s _ %s %s ===\n' %
(list_of_exps_names[list_of_experiments.index(experiment)],
town, data_params['noise']))
print('\n ** Reading the data **\n')
data = read_data(
exp_batch, experiment, town, data_params, data_params['noise']
) # this reads the data and infers the masks (or offsets) for different cameras
if data is None: # This folder didnt work out, probably is missing important data
print('\n ** Missing Data on Folder **\n')
continue
# Print data
print(data['town'])
for step, data_item in data['values'].items():
print(step)
for k, v in data_item.items():
print(k, len(v))
print('\n ** Processing the data **\n')
metrics = process_data(
data, processing_params, data_params['noise']
) # Compute metrics from the data. Can be multiple metrics, given by the processing_params list. Should be vectorized as much as possible. The output is a list of the same size as processing_params.
all_metrics[
experiment + ' : ' +
list_of_exps_names[list_of_experiments.index(experiment)] +
'_' +
town] = metrics # append to the computed list of metrics to the dictionary of results.
with open(os.path.join(out_path, 'all_metrics.txt'), 'w') as f:
f.write('all_metrics:\n' + pprint.pformat(all_metrics, indent=4))
# Plot the results
print('\n === Plotting the results ===\n')
for plot_label, plot_param in plot_params.items():
print(plot_param)
print('\n ** Plotting %s **' % plot_label)
if 'analysis' in plot_param:
make_scatter_plot_analysis(all_metrics,
plot_param,
out_file=os.path.join(
out_path, plot_label + '.pdf'))
else:
make_scatter_plot(all_metrics,
plot_param,
out_file=os.path.join(out_path,
plot_label + '.pdf'))
def plot_folder_summaries(exp_batch,
train,
validation_datasets,
drive_environments,
verbose=False):
"""
Main plotting function for the folder mode.
Args:
exp_batch: The exp batch (folder) being plotted on the screen.
train: If train process is being printed
validation_datasets: The validation datasets being computed
drive_environments: The driving environments/ Benchmarks
verbose:
Returns:
None
"""
os.system('clear')
process_names = []
if train:
process_names.append('train')
for val in validation_datasets:
process_names.append('validation' + '_' + val)
# We save the drive files to be used later
drive_files = {}
for drive in drive_environments:
drive_files.update({
('drive' + '_' + drive.split('/')[-1].split('.')[0]):
drive
})
process_names.append('drive' + '_' +
drive.split('/')[-1].split('.')[0])
experiments_list = os.listdir(os.path.join('configs', exp_batch))
experiments_list = [
experiment.split('.')[-2] for experiment in experiments_list
]
names_list = get_names(exp_batch)
sorted_keys = sorted(
range(len(names_list)),
key=lambda k: names_list[experiments_list[k] + '.yaml'])
for key in sorted_keys:
experiment = experiments_list[key]
generated_name = names_list[experiment + '.yaml']
if experiment == '':
raise ValueError("Empty Experiment on List")
g_conf.immutable(False)
merge_with_yaml(
os.path.join('configs', exp_batch, experiment + '.yaml'))
print(BOLD + experiment + ' : ' + generated_name + END)
for process in process_names:
try:
output = get_status(exp_batch, experiment, process)
except: # TODO: bad design. But the printing should not stop for any error on reading
import traceback
traceback.print_exc()
status = output[0]
summary = output[1]
print(' ', process)
if status == 'Not Started':
print(' STATUS: ', BOLD + status + END)
elif status == 'Loading':
print(' STATUS: ', YELLOW + status + END, ' - ',
YELLOW + summary + END)
elif status == 'Iterating':
print(' STATUS: ', YELLOW + status + END)
elif status == 'Finished':
print(' STATUS: ', GREEN + status + END)
elif status == 'Error':
print(' STATUS: ', RED + status + END, ' - ',
RED + summary + END)
if status == 'Iterating':
if 'train' == process:
print_train_summary(summary[status])
if 'validation' in process:
if summary[1] != '': # If it has no summary we don't plot
print_validation_summary(summary[0][status],
summary[1][status]['Summary'],
verbose)
else:
print_validation_summary(summary[0][status], '',
verbose)
if 'drive' in process:
if 'Agent' not in summary[status]:
continue
checkpoint = summary[status]['Checkpoint'] # Get the sta
# This contain the results from completed iterations
# we read the json file directly
agent_checkpoint_name = str(exp_batch) + '_' + str(
experiment) + '_' + str(checkpoint)
print_drive_summary(drive_files[process],
agent_checkpoint_name, checkpoint)
def export_status(exp_batch, validation_datasets, driving_environments):
root_path = '_logs'
experiments = os.listdir(os.path.join(root_path, exp_batch))
# Make the header of the exported csv
csv_outfile = os.path.join(root_path, exp_batch, 'status.csv')
with open(csv_outfile, 'w') as f:
f.write("experiment_alias,experiment_name")
for validation in validation_datasets:
f.write("," + validation)
for driving in driving_environments:
f.write("," + driving)
f.write('\n')
names_list = get_names(exp_batch)
count = 0
print(names_list)
for exp in experiments:
if os.path.isdir(os.path.join(root_path, exp_batch, exp)):
f.write("%s,%s" % (exp, names_list[exp + '.yaml']))
count += 1
print(exp)
merge_with_yaml(
os.path.join('configs', exp_batch, exp + '.yaml'))
experiments_logs = os.listdir(
os.path.join(root_path, exp_batch, exp))
for validation in validation_datasets:
log = 'validation_' + validation + '_csv'
if log in os.listdir(
os.path.join(root_path, exp_batch, exp)):
if (str(g_conf.TEST_SCHEDULE[-1]) +
'.csv') in os.listdir(
os.path.join(root_path, exp_batch, exp,
log)):
f.write(",Done")
else:
f.write(", ")
else:
f.write(", ")
for driving in driving_environments:
log = 'drive_' + driving + '_csv'
if log in os.listdir(
os.path.join(root_path, exp_batch, exp)):
if g_conf.USE_ORACLE:
output_name = 'control_output_auto.csv'
else:
output_name = 'control_output.csv'
print(log)
if output_name in os.listdir(
os.path.join(root_path, exp_batch, exp, log)):
csv_file_path = os.path.join(
root_path, exp_batch, exp, log, output_name)
control_csv = read_summary_csv(csv_file_path)
if control_csv is not None and (
g_conf.TEST_SCHEDULE[-1]) == int(
control_csv['step'][-1]):
f.write(",Done")
else:
f.write(", ")
else:
f.write(", ")
else:
f.write(", ")
f.write("\n")