def plot_gat_duration_error(db, algorithms: list, domain: str, instance: str):
""" Plot goal achievement time per action duration errorbar graph.
:param db: the db handler
:param algorithms: the algorithms to graph
:param domain: the domain
:param instance: the domain instance
"""
# Gather required A* data
astar_gat_per_duration = get_gat_per_duration_data(db, "A_STAR", domain, instance)
algorithm_data = {}
# Plot for each provided algorithm
for algorithm in algorithms:
algorithm_data[algorithm] = get_gat_per_duration_data(db, algorithm, domain, instance)
plotutils.plot_gat_duration_error(algorithm_data, astar_gat_per_duration, all_action_durations_ms)
def plot_gat_duration_error(db, algorithms, domain, instance):
# Gather required A* data
astar_gat_per_duration = get_gat_per_duration_data(db, "A_STAR", domain,
instance)
algorithm_data = {}
# Plot for each provided algorithm
for algorithm in algorithms:
algorithm_data[algorithm] = get_gat_per_duration_data(
db, algorithm, domain, instance)
plotutils.plot_gat_duration_error(
algorithm_data,
astar_gat_per_duration,
all_action_durations_ms,
title=plotutils.translate_domain_name(domain) + " - " + instance)
def plot_gat_duration_error(db, algorithms: list, domain: str, instance: str):
""" Plot goal achievement time per action duration errorbar graph.
:param db: the db handler
:param algorithms: the algorithms to graph
:param domain: the domain
:param instance: the domain instance
"""
# Gather required A* data
astar_gat_per_duration = get_gat_per_duration_data(db, "A_STAR", domain,
instance)
algorithm_data = {}
# Plot for each provided algorithm
for algorithm in algorithms:
algorithm_data[algorithm] = get_gat_per_duration_data(
db, algorithm, domain, instance)
plotutils.plot_gat_duration_error(algorithm_data, astar_gat_per_duration,
all_action_durations_ms)
def plot_domain_instance(instance: str, domain_configuration: dict = None):
""" Performs the plotting of all plots for the given domain instance and configuration
:param instance: the domain instance
:param domain_configuration: the domain configuration
:return: the markdown for each of the generated plots; empty string if nothing was plotted
"""
instance_file_name = instance.replace("/", "_")
plot_title = ""
# plot_title = plotutils.translate_domain_instance_name(domain, instance)
plots_markdown = ""
instance_level = "#"
if domain_configuration:
instance_level += "#"
instance_file_name += "_" + concatenate_configuration(
domain_configuration)
if not quiet:
print("Processing {} - {}".format(domain, instance))
# Gather error data for each algorithm
astar_gat_per_duration = get_gat_per_duration_data(
db,
"A_STAR",
domain,
instance,
domain_configuration,
execution=True)
algorithm_gat_per_duration = {}
for algorithm, configurations in all_algorithms.items():
if not configurations:
algorithm_gat_per_duration[algorithm] = \
get_gat_per_duration_data(db, algorithm, domain, instance, domain_configuration)
else:
for configuration in configurations:
algorithm_gat_per_duration[get_configuration_plot_name(algorithm, configuration)] = \
get_gat_per_duration_data(db, algorithm, domain, instance, configuration, domain_configuration)
remove_empty_data(algorithm_gat_per_duration)
# Store data to compute average later
for algorithm, values in algorithm_gat_per_duration.items():
count = 0
for val in values:
if val:
all_error_data[algorithm][count].append(val[0])
count += 1
count = 0
for val in astar_gat_per_duration:
if val:
all_astar_error_data[count].append(val[0])
else:
all_astar_error_data[count].append([])
count += 1
# Do individual plots if average_only not specified
if not average_only:
# Errorbar plot
if algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {}".format(
domain, instance, domain_configuration))
file_header = "{}_{}".format(domain, instance_file_name)
plots_markdown += do_plot(
file_header, "error", lambda: plotutils.
plot_gat_duration_error(algorithm_gat_per_duration,
all_action_durations_ms,
normalize=True,
astar_data=astar_gat_per_duration,
title=plot_title,
log10=log10))
# Also plot errorbar without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = algorithm_gat_per_duration.pop(
algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
# Plot it if there was anything to pop and there is still data left over
if removed and algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {} without {}".
format(domain, instance, domain_configuration,
removed))
file_header = "{}_{}_NO_{}".format(domain,
instance_file_name,
removed)
plots_markdown += do_plot(
file_header, "error",
lambda: plotutils.plot_gat_duration_error(
algorithm_gat_per_duration,
all_action_durations_ms,
normalize=True,
astar_data=astar_gat_per_duration,
title=plot_title,
log10=log10))
if not error_only:
for action_duration in all_action_durations:
action_plot_title = ""
# plot_title + " - " + str(int(plotutils.cnv_ns_to_ms(action_duration))) + " ms"
file_header = "{}_{}_{}".format(domain, instance_file_name,
action_duration)
# Gather GAT data
gat_data, gat_labels, gat_indices = get_gat_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration, old_idle_time)
y_gat = gat_data['goalAchievementTime']
y_idle = gat_data['idlePlanningTime']
# Gather node data
node_data, node_labels, node_indices = get_node_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration)
y_generated = node_data['generatedNodes']
y_expanded = node_data['expandedNodes']
if y_gat or y_generated:
action_duration_level = instance_level + "#"
plots_markdown += "{} Action Duration: {} ns\n\n".format(
action_duration_level, action_duration)
def plot_stacked(data,
labels,
file_header=file_header,
print_suffix=""):
stacked_markdown = ""
if not quiet:
msg = "Plotting stacked bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
stacked_markdown += do_plot(
file_header, "stacked", lambda: plotutils.
plot_gat_stacked_bars(data,
labels,
title=action_plot_title,
log10=log10))
return stacked_markdown
def plot_node_bars(data,
labels,
file_header=file_header,
print_suffix=""):
nodes_markdown = ""
if not quiet:
msg = "Plotting node bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
nodes_markdown += do_plot(
file_header, "nodes", lambda: plotutils.
plot_node_count_bars(data,
labels,
title=action_plot_title,
log10=log10))
return nodes_markdown
if y_gat and y_idle:
plots_markdown += plot_stacked(gat_data, gat_labels)
removed = remove_algorithms(gat_data, gat_labels,
gat_indices, plot_without)
# Plot it if there was anything to pop and there is still data left over
if removed and gat_data[
'goalAchievementTime'] and gat_data[
"idlePlanningTime"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_stacked(
gat_data,
gat_labels,
file_header=new_file_header,
print_suffix=suffix)
if y_generated and y_expanded:
plots_markdown += plot_node_bars(
node_data, node_labels)
removed = remove_algorithms(node_data, node_labels,
node_indices, plot_without)
# Plot it if there was anything to pop and there is still data left over
if removed and node_data[
'generatedNodes'] and node_data[
"expandedNodes"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_node_bars(
node_data,
node_labels,
file_header=new_file_header,
print_suffix=suffix)
if not plots_markdown:
return ""
else:
instance_markdown_document = "{} Instance: {}\n\n".format(
instance_level, instance)
return instance_markdown_document + plots_markdown
def plot_all_for_domain(
db,
domain: str,
instances: list,
plot_average: bool = False,
average_only: bool = False,
markdown_summary: bool = True,
error_only: bool = False,
plot_without: tuple = ("RTA_STAR", "LSS_LRTA_STAR_STATIC_MULTIPLE",
"DYNAMIC_F_HAT_STATIC_MULTIPLE"),
old_idle_time: bool = False,
log10: bool = True):
""" Creates all plots for the given domain. Also optionally produces a summary markdown file containing all plots
which can be converted to a pdf if pandoc is present on the system. Can also make an additional errorbar plot of
the averaged data across all domain instances gathered. This function is optimized to reused data from queries to
reduce the number of needed queries to the database.
:param db: the db handler
:param domain: the domain
:param instances: the domain instances
:param plot_average: whether to produce an additional errorbar plot of the average data across all domain instances
:param average_only: whether to only produce the average errorbar plot and no other plots; implies plot_average
:param markdown_summary: whether to produce a summary markdown file containing all individual plots which is then
converted to a pdf file if pandoc is present on the system
:param error_only: whether to only produce errorbar plots and no other plot types
:param plot_without: a tuple of algorithm names. If the tuple is not empty, then each of the provided algorithms is
removed from the dataset after plotting and an additional plot is generated without the provided algorithms' data
:param old_idle_time: whether to use the old idle planning time parameters before a designated field was added
:param log10: whether the plots should have a log10 scale on the y-axis
"""
if average_only:
markdown_summary = False
plot_average = True
def do_plot(file_header: str, file_suffix: str, plot):
""" Helper to perform the plotting operation, save the plot, clear the figure, and return the applicable
markdown for the plot.
:param file_header: header of the file to save the plot to
:param file_suffix: suffix to append to the file header
:param plot: plotter function to create the actual plot
:return: the markdown for this plot entry
"""
# file_header = "{}_{}".format(domain, instance_file_name)
filename = "plots/{}_{}.{}".format(file_header, file_suffix,
file_format)
legend = plot()
plotutils.save_plot(plt, filename, legend)
plt.close('all')
return "\n\n\\clearpage\n\n".format(file_header, filename)
# translated_domain_name = plotutils.translate_domain_name(domain)
domain_configurations = all_domains[domain]
all_error_data = {}
all_astar_error_data = []
markdown_document = "# Domain: {}\n\n".format(domain)
for index, plot_algorithm_name in enumerate(plot_algorithm_names.keys()):
all_error_data[plot_algorithm_name] = []
for _ in all_action_durations:
all_error_data[plot_algorithm_name].append([])
for _ in all_action_durations:
all_astar_error_data.append([])
def plot_domain_instance(instance: str, domain_configuration: dict = None):
""" Performs the plotting of all plots for the given domain instance and configuration
:param instance: the domain instance
:param domain_configuration: the domain configuration
:return: the markdown for each of the generated plots; empty string if nothing was plotted
"""
instance_file_name = instance.replace("/", "_")
plot_title = ""
# plot_title = plotutils.translate_domain_instance_name(domain, instance)
plots_markdown = ""
instance_level = "#"
if domain_configuration:
instance_level += "#"
instance_file_name += "_" + concatenate_configuration(
domain_configuration)
if not quiet:
print("Processing {} - {}".format(domain, instance))
# Gather error data for each algorithm
astar_gat_per_duration = get_gat_per_duration_data(
db,
"A_STAR",
domain,
instance,
domain_configuration,
execution=True)
algorithm_gat_per_duration = {}
for algorithm, configurations in all_algorithms.items():
if not configurations:
algorithm_gat_per_duration[algorithm] = \
get_gat_per_duration_data(db, algorithm, domain, instance, domain_configuration)
else:
for configuration in configurations:
algorithm_gat_per_duration[get_configuration_plot_name(algorithm, configuration)] = \
get_gat_per_duration_data(db, algorithm, domain, instance, configuration, domain_configuration)
remove_empty_data(algorithm_gat_per_duration)
# Store data to compute average later
for algorithm, values in algorithm_gat_per_duration.items():
count = 0
for val in values:
if val:
all_error_data[algorithm][count].append(val[0])
count += 1
count = 0
for val in astar_gat_per_duration:
if val:
all_astar_error_data[count].append(val[0])
else:
all_astar_error_data[count].append([])
count += 1
# Do individual plots if average_only not specified
if not average_only:
# Errorbar plot
if algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {}".format(
domain, instance, domain_configuration))
file_header = "{}_{}".format(domain, instance_file_name)
plots_markdown += do_plot(
file_header, "error", lambda: plotutils.
plot_gat_duration_error(algorithm_gat_per_duration,
all_action_durations_ms,
normalize=True,
astar_data=astar_gat_per_duration,
title=plot_title,
log10=log10))
# Also plot errorbar without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = algorithm_gat_per_duration.pop(
algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
# Plot it if there was anything to pop and there is still data left over
if removed and algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {} without {}".
format(domain, instance, domain_configuration,
removed))
file_header = "{}_{}_NO_{}".format(domain,
instance_file_name,
removed)
plots_markdown += do_plot(
file_header, "error",
lambda: plotutils.plot_gat_duration_error(
algorithm_gat_per_duration,
all_action_durations_ms,
normalize=True,
astar_data=astar_gat_per_duration,
title=plot_title,
log10=log10))
if not error_only:
for action_duration in all_action_durations:
action_plot_title = ""
# plot_title + " - " + str(int(plotutils.cnv_ns_to_ms(action_duration))) + " ms"
file_header = "{}_{}_{}".format(domain, instance_file_name,
action_duration)
# Gather GAT data
gat_data, gat_labels, gat_indices = get_gat_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration, old_idle_time)
y_gat = gat_data['goalAchievementTime']
y_idle = gat_data['idlePlanningTime']
# Gather node data
node_data, node_labels, node_indices = get_node_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration)
y_generated = node_data['generatedNodes']
y_expanded = node_data['expandedNodes']
if y_gat or y_generated:
action_duration_level = instance_level + "#"
plots_markdown += "{} Action Duration: {} ns\n\n".format(
action_duration_level, action_duration)
def plot_stacked(data,
labels,
file_header=file_header,
print_suffix=""):
stacked_markdown = ""
if not quiet:
msg = "Plotting stacked bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
stacked_markdown += do_plot(
file_header, "stacked", lambda: plotutils.
plot_gat_stacked_bars(data,
labels,
title=action_plot_title,
log10=log10))
return stacked_markdown
def plot_node_bars(data,
labels,
file_header=file_header,
print_suffix=""):
nodes_markdown = ""
if not quiet:
msg = "Plotting node bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
nodes_markdown += do_plot(
file_header, "nodes", lambda: plotutils.
plot_node_count_bars(data,
labels,
title=action_plot_title,
log10=log10))
return nodes_markdown
if y_gat and y_idle:
plots_markdown += plot_stacked(gat_data, gat_labels)
removed = remove_algorithms(gat_data, gat_labels,
gat_indices, plot_without)
# Plot it if there was anything to pop and there is still data left over
if removed and gat_data[
'goalAchievementTime'] and gat_data[
"idlePlanningTime"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_stacked(
gat_data,
gat_labels,
file_header=new_file_header,
print_suffix=suffix)
if y_generated and y_expanded:
plots_markdown += plot_node_bars(
node_data, node_labels)
removed = remove_algorithms(node_data, node_labels,
node_indices, plot_without)
# Plot it if there was anything to pop and there is still data left over
if removed and node_data[
'generatedNodes'] and node_data[
"expandedNodes"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_node_bars(
node_data,
node_labels,
file_header=new_file_header,
print_suffix=suffix)
if not plots_markdown:
return ""
else:
instance_markdown_document = "{} Instance: {}\n\n".format(
instance_level, instance)
return instance_markdown_document + plots_markdown
if domain_configurations:
for domain_configuration in domain_configurations:
instance_markdown = ""
for instance in instances:
instance_markdown += plot_domain_instance(
instance, domain_configuration)
if instance_markdown:
markdown_document += "# Configuration {}\n\n".format(
concatenate_configuration(domain_configuration,
separator=" ",
include_names=True))
markdown_document += instance_markdown
else:
for instance in instances:
markdown_document += plot_domain_instance(instance, None)
# Produce markdown file and convert to pdf if pandoc present
if markdown_summary:
if not quiet:
print("Saving markdown file")
file_header = "plots/{}_plots".format(domain)
markdown_file = "{}.md".format(file_header)
pdf_file = "{}.pdf".format(file_header)
save_to_file(markdown_file, markdown_document)
pandoc = which("pandoc")
if pandoc:
call([pandoc, "-o", pdf_file, markdown_file])
# Plot average data
remove_empty_data(all_error_data)
if plot_average:
if not quiet:
print("Plotting {} averages".format(domain))
lgd = plotutils.plot_gat_duration_error(
all_error_data,
all_action_durations_ms,
normalize=True,
astar_data=all_astar_error_data,
title="{} data over all instances".format(
plotutils.translate_domain_name(domain)))
plotutils.save_plot(plt, "plots/{}_average.pdf".format(domain), lgd)
plt.close('all')
# Plot averages without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = all_error_data.pop(algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
if removed:
lgd = plotutils.plot_gat_duration_error(
all_error_data,
all_action_durations_ms,
normalize=True,
astar_data=all_astar_error_data,
title="{} data over all instances no {}".format(
plotutils.translate_domain_name(domain), removed))
plotutils.save_plot(
plt, "plots/{}_NO_{}_average.pdf".format(domain, removed), lgd)
plt.close('all')
def plot_domain_instance(instance: str, domain_configuration: dict = None):
""" Performs the plotting of all plots for the given domain instance and configuration
:param instance: the domain instance
:param domain_configuration: the domain configuration
:return: the markdown for each of the generated plots; empty string if nothing was plotted
"""
instance_file_name = instance.replace("/", "_")
plot_title = ""
# plot_title = plotutils.translate_domain_instance_name(domain, instance)
plots_markdown = ""
instance_level = "#"
if domain_configuration:
instance_level += "#"
instance_file_name += "_" + concatenate_configuration(domain_configuration)
if not quiet:
print("Processing {} - {}".format(domain, instance))
# Gather error data for each algorithm
astar_gat_per_duration = get_gat_per_duration_data(db, "A_STAR", domain, instance, domain_configuration,
execution=True)
algorithm_gat_per_duration = {}
for algorithm, configurations in all_algorithms.items():
if not configurations:
algorithm_gat_per_duration[algorithm] = \
get_gat_per_duration_data(db, algorithm, domain, instance, domain_configuration)
else:
for configuration in configurations:
algorithm_gat_per_duration[get_configuration_plot_name(algorithm, configuration)] = \
get_gat_per_duration_data(db, algorithm, domain, instance, configuration, domain_configuration)
remove_empty_data(algorithm_gat_per_duration)
# Store data to compute average later
for algorithm, values in algorithm_gat_per_duration.items():
count = 0
for val in values:
if val:
all_error_data[algorithm][count].append(val[0])
count += 1
count = 0
for val in astar_gat_per_duration:
if val:
all_astar_error_data[count].append(val[0])
else:
all_astar_error_data[count].append([])
count += 1
# Do individual plots if average_only not specified
if not average_only:
# Errorbar plot
if algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {}".format(domain, instance, domain_configuration))
file_header = "{}_{}".format(domain, instance_file_name)
plots_markdown += do_plot(file_header, "error", lambda:
plotutils.plot_gat_duration_error(algorithm_gat_per_duration, all_action_durations_ms, normalize=True,
astar_data=astar_gat_per_duration, title=plot_title, log10=log10))
# Also plot errorbar without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = algorithm_gat_per_duration.pop(algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
# Plot it if there was anything to remove and there is still data left over
if removed and algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {} without {}".format(domain, instance,
domain_configuration, removed))
file_header = "{}_{}_NO_{}".format(domain, instance_file_name, removed)
plots_markdown += do_plot(file_header, "error", lambda:
plotutils.plot_gat_duration_error(algorithm_gat_per_duration, all_action_durations_ms,
normalize=True, astar_data=astar_gat_per_duration,
title=plot_title, log10=log10))
if not error_only:
for action_duration in all_action_durations:
action_plot_title = ""
# plot_title + " - " + str(int(plotutils.cnv_ns_to_ms(action_duration))) + " ms"
file_header = "{}_{}_{}".format(domain, instance_file_name, action_duration)
# Gather GAT data
gat_data, gat_labels, gat_indices = get_gat_data(db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration,
old_idle_time)
y_gat = gat_data['goalAchievementTime']
y_idle = gat_data['idlePlanningTime']
# Gather node data
node_data, node_labels, node_indices = get_node_data(db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration)
y_generated = node_data['generatedNodes']
y_expanded = node_data['expandedNodes']
if y_gat or y_generated:
action_duration_level = instance_level + "#"
plots_markdown += "{} Action Duration: {} ns\n\n".format(action_duration_level, action_duration)
def plot_stacked(data, labels, file_header=file_header, print_suffix=""):
stacked_markdown = ""
if not quiet:
msg = "Plotting stacked bars: {} - {} - {} - {}".format(domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
stacked_markdown += do_plot(file_header, "stacked", lambda:
plotutils.plot_gat_stacked_bars(data, labels, title=action_plot_title, log10=log10))
return stacked_markdown
def plot_node_bars(data, labels, file_header=file_header, print_suffix=""):
nodes_markdown = ""
if not quiet:
msg = "Plotting node bars: {} - {} - {} - {}".format(domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
nodes_markdown += do_plot(file_header, "nodes", lambda:
plotutils.plot_node_count_bars(data, labels, title=action_plot_title, log10=log10))
return nodes_markdown
if y_gat and y_idle:
plots_markdown += plot_stacked(gat_data, gat_labels)
removed = remove_algorithms(gat_data, gat_labels, gat_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and gat_data['goalAchievementTime'] and gat_data["idlePlanningTime"]:
new_file_header = "{}_{}_{}_NO_{}".format(domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_stacked(gat_data, gat_labels, file_header=new_file_header,
print_suffix=suffix)
if y_generated and y_expanded:
plots_markdown += plot_node_bars(node_data, node_labels)
removed = remove_algorithms(node_data, node_labels, node_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and node_data['generatedNodes'] and node_data["expandedNodes"]:
new_file_header = "{}_{}_{}_NO_{}".format(domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_node_bars(node_data, node_labels, file_header=new_file_header,
print_suffix=suffix)
if not plots_markdown:
return ""
else:
instance_markdown_document = "{} Instance: {}\n\n".format(instance_level, instance)
return instance_markdown_document + plots_markdown
def plot_all_for_domain(db, domain: str, instances: list, plot_average: bool = False, average_only: bool = False,
markdown_summary: bool = True, error_only: bool = False, plot_without: tuple =
("RTA_STAR", "LSS_LRTA_STAR_STATIC_MULTIPLE", "DYNAMIC_F_HAT_STATIC_MULTIPLE"),
old_idle_time: bool = False, log10: bool = True):
""" Creates all plots for the given domain. Also optionally produces a summary markdown file containing all plots
which can be converted to a pdf if pandoc is present on the system. Can also make an additional errorbar plot of
the averaged data across all domain instances gathered. This function is optimized to reused data from queries to
reduce the number of needed queries to the database.
:param db: the db handler
:param domain: the domain
:param instances: the domain instances
:param plot_average: whether to produce an additional errorbar plot of the average data across all domain instances
:param average_only: whether to only produce the average errorbar plot and no other plots; implies plot_average
:param markdown_summary: whether to produce a summary markdown file containing all individual plots which is then
converted to a pdf file if pandoc is present on the system
:param error_only: whether to only produce errorbar plots and no other plot types
:param plot_without: a tuple of algorithm names. If the tuple is not empty, then each of the provided algorithms is
removed from the dataset after plotting and an additional plot is generated without the provided algorithms' data
:param old_idle_time: whether to use the old idle planning time parameters before a designated field was added
:param log10: whether the plots should have a log10 scale on the y-axis
"""
if average_only:
markdown_summary = False
plot_average = True
def do_plot(file_header: str, file_suffix: str, plot):
""" Helper to perform the plotting operation, save the plot, clear the figure, and return the applicable
markdown for the plot.
:param file_header: header of the file to save the plot to
:param file_suffix: suffix to append to the file header
:param plot: plotter function to create the actual plot
:return: the markdown for this plot entry
"""
# file_header = "{}_{}".format(domain, instance_file_name)
filename = "plots/{}_{}.{}".format(file_header, file_suffix, file_format)
legend = plot()
plotutils.save_plot(plt, filename, legend)
plt.close('all')
return "\n\n\\clearpage\n\n".format(file_header, filename)
# translated_domain_name = plotutils.translate_domain_name(domain)
domain_configurations = all_domains[domain]
all_error_data = {}
all_astar_error_data = []
markdown_document = "# Domain: {}\n\n".format(domain)
for index, plot_algorithm_name in enumerate(plot_algorithm_names.keys()):
all_error_data[plot_algorithm_name] = []
for _ in all_action_durations:
all_error_data[plot_algorithm_name].append([])
for _ in all_action_durations:
all_astar_error_data.append([])
def plot_domain_instance(instance: str, domain_configuration: dict = None):
""" Performs the plotting of all plots for the given domain instance and configuration
:param instance: the domain instance
:param domain_configuration: the domain configuration
:return: the markdown for each of the generated plots; empty string if nothing was plotted
"""
instance_file_name = instance.replace("/", "_")
plot_title = ""
# plot_title = plotutils.translate_domain_instance_name(domain, instance)
plots_markdown = ""
instance_level = "#"
if domain_configuration:
instance_level += "#"
instance_file_name += "_" + concatenate_configuration(domain_configuration)
if not quiet:
print("Processing {} - {}".format(domain, instance))
# Gather error data for each algorithm
astar_gat_per_duration = get_gat_per_duration_data(db, "A_STAR", domain, instance, domain_configuration,
execution=True)
algorithm_gat_per_duration = {}
for algorithm, configurations in all_algorithms.items():
if not configurations:
algorithm_gat_per_duration[algorithm] = \
get_gat_per_duration_data(db, algorithm, domain, instance, domain_configuration)
else:
for configuration in configurations:
algorithm_gat_per_duration[get_configuration_plot_name(algorithm, configuration)] = \
get_gat_per_duration_data(db, algorithm, domain, instance, configuration, domain_configuration)
remove_empty_data(algorithm_gat_per_duration)
# Store data to compute average later
for algorithm, values in algorithm_gat_per_duration.items():
count = 0
for val in values:
if val:
all_error_data[algorithm][count].append(val[0])
count += 1
count = 0
for val in astar_gat_per_duration:
if val:
all_astar_error_data[count].append(val[0])
else:
all_astar_error_data[count].append([])
count += 1
# Do individual plots if average_only not specified
if not average_only:
# Errorbar plot
if algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {}".format(domain, instance, domain_configuration))
file_header = "{}_{}".format(domain, instance_file_name)
plots_markdown += do_plot(file_header, "error", lambda:
plotutils.plot_gat_duration_error(algorithm_gat_per_duration, all_action_durations_ms, normalize=True,
astar_data=astar_gat_per_duration, title=plot_title, log10=log10))
# Also plot errorbar without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = algorithm_gat_per_duration.pop(algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
# Plot it if there was anything to remove and there is still data left over
if removed and algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {} without {}".format(domain, instance,
domain_configuration, removed))
file_header = "{}_{}_NO_{}".format(domain, instance_file_name, removed)
plots_markdown += do_plot(file_header, "error", lambda:
plotutils.plot_gat_duration_error(algorithm_gat_per_duration, all_action_durations_ms,
normalize=True, astar_data=astar_gat_per_duration,
title=plot_title, log10=log10))
if not error_only:
for action_duration in all_action_durations:
action_plot_title = ""
# plot_title + " - " + str(int(plotutils.cnv_ns_to_ms(action_duration))) + " ms"
file_header = "{}_{}_{}".format(domain, instance_file_name, action_duration)
# Gather GAT data
gat_data, gat_labels, gat_indices = get_gat_data(db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration,
old_idle_time)
y_gat = gat_data['goalAchievementTime']
y_idle = gat_data['idlePlanningTime']
# Gather node data
node_data, node_labels, node_indices = get_node_data(db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration)
y_generated = node_data['generatedNodes']
y_expanded = node_data['expandedNodes']
if y_gat or y_generated:
action_duration_level = instance_level + "#"
plots_markdown += "{} Action Duration: {} ns\n\n".format(action_duration_level, action_duration)
def plot_stacked(data, labels, file_header=file_header, print_suffix=""):
stacked_markdown = ""
if not quiet:
msg = "Plotting stacked bars: {} - {} - {} - {}".format(domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
stacked_markdown += do_plot(file_header, "stacked", lambda:
plotutils.plot_gat_stacked_bars(data, labels, title=action_plot_title, log10=log10))
return stacked_markdown
def plot_node_bars(data, labels, file_header=file_header, print_suffix=""):
nodes_markdown = ""
if not quiet:
msg = "Plotting node bars: {} - {} - {} - {}".format(domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
nodes_markdown += do_plot(file_header, "nodes", lambda:
plotutils.plot_node_count_bars(data, labels, title=action_plot_title, log10=log10))
return nodes_markdown
if y_gat and y_idle:
plots_markdown += plot_stacked(gat_data, gat_labels)
removed = remove_algorithms(gat_data, gat_labels, gat_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and gat_data['goalAchievementTime'] and gat_data["idlePlanningTime"]:
new_file_header = "{}_{}_{}_NO_{}".format(domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_stacked(gat_data, gat_labels, file_header=new_file_header,
print_suffix=suffix)
if y_generated and y_expanded:
plots_markdown += plot_node_bars(node_data, node_labels)
removed = remove_algorithms(node_data, node_labels, node_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and node_data['generatedNodes'] and node_data["expandedNodes"]:
new_file_header = "{}_{}_{}_NO_{}".format(domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_node_bars(node_data, node_labels, file_header=new_file_header,
print_suffix=suffix)
if not plots_markdown:
return ""
else:
instance_markdown_document = "{} Instance: {}\n\n".format(instance_level, instance)
return instance_markdown_document + plots_markdown
if domain_configurations:
for domain_configuration in domain_configurations:
instance_markdown = ""
for instance in instances:
instance_markdown += plot_domain_instance(instance, domain_configuration)
if instance_markdown:
markdown_document += "# Configuration {}\n\n".format(
concatenate_configuration(domain_configuration, separator=" ", include_names=True))
markdown_document += instance_markdown
else:
for instance in instances:
markdown_document += plot_domain_instance(instance, None)
# Produce markdown file and convert to pdf if pandoc present
if markdown_summary:
if not quiet:
print("Saving markdown file")
file_header = "plots/{}_plots".format(domain)
markdown_file = "{}.md".format(file_header)
pdf_file = "{}.pdf".format(file_header)
save_to_file(markdown_file, markdown_document)
pandoc = which("pandoc")
if pandoc:
call([pandoc, "-o", pdf_file, markdown_file])
# Plot average data
remove_empty_data(all_error_data)
if plot_average:
if not quiet:
print("Plotting {} averages".format(domain))
lgd = plotutils.plot_gat_duration_error(all_error_data, all_action_durations_ms, normalize=True,
astar_data=all_astar_error_data,
title="{} data over all instances".format(
plotutils.translate_domain_name(domain)))
plotutils.save_plot(plt, "plots/{}_average.pdf".format(domain), lgd)
plt.close('all')
# Plot averages without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = all_error_data.pop(algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
if removed:
lgd = plotutils.plot_gat_duration_error(all_error_data, all_action_durations_ms, normalize=True,
astar_data=all_astar_error_data,
title="{} data over all instances no {}".format(
plotutils.translate_domain_name(domain), removed))
plotutils.save_plot(plt, "plots/{}_NO_{}_average.pdf".format(domain, removed), lgd)
plt.close('all')
def plot_all_for_domain(
db,
domain: str,
instances: list,
plot_average: bool = False,
average_only: bool = False,
markdown_summary: bool = True,
error_only: bool = False,
plot_without: tuple = ("RTA_STAR", "LSS_LRTA_STAR_STATIC_MULTIPLE",
"DYNAMIC_F_HAT_STATIC_MULTIPLE"),
old_idle_time: bool = False,
log10: bool = True):
if average_only:
markdown_summary = False
plot_average = True
# translated_domain_name = plotutils.translate_domain_name(domain)
domain_configurations = all_domains[domain]
all_error_data = {}
all_astar_error_data = []
markdown_document = "# Domain: {}\n\n".format(domain)
for index, plot_algorithm_name in enumerate(plot_algorithm_names.keys()):
all_error_data[plot_algorithm_name] = []
for _ in all_action_durations:
all_error_data[plot_algorithm_name].append([])
for _ in all_action_durations:
all_astar_error_data.append([])
def plot_domain_instance(instance, domain_configuration):
instance_file_name = instance.replace("/", "_")
plot_title = plotutils.translate_domain_instance_name(
domain, instance) # translated_domain_name + " - " + instance
plots_markdown = ""
instance_level = "#"
if domain_configuration:
instance_level += "#"
instance_file_name += "_" + concatenate_configuration(
domain_configuration)
if not quiet:
print("Processing {} - {}".format(domain, instance))
# Gather error data for each algorithm
astar_gat_per_duration = get_gat_per_duration_data(
db, "A_STAR", domain, instance, domain_configuration)
algorithm_gat_per_duration = {}
for algorithm, configurations in all_algorithms.items():
if not configurations:
algorithm_gat_per_duration[algorithm] = \
get_gat_per_duration_data(db, algorithm, domain, instance, domain_configuration)
else:
for configuration in configurations:
algorithm_gat_per_duration[get_configuration_plot_name(algorithm, configuration)] = \
get_gat_per_duration_data(db, algorithm, domain, instance, configuration, domain_configuration)
remove_empty_data(algorithm_gat_per_duration)
# Store data to compute average later
for algorithm, values in algorithm_gat_per_duration.items():
count = 0
for val in values:
if val:
all_error_data[algorithm][count].append(val[0])
count += 1
count = 0
for val in astar_gat_per_duration:
if val:
all_astar_error_data[count].append(val[0])
else:
all_astar_error_data[count].append([])
count += 1
# Do individual plots if average_only not specified
if not average_only:
# Errorbar plot
if algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {}".format(
domain, instance, domain_configuration))
file_header = "{}_{}".format(domain, instance_file_name)
plots_markdown += do_plot(
file_header, "error", lambda: plotutils.
plot_gat_duration_error(algorithm_gat_per_duration,
astar_gat_per_duration,
all_action_durations_ms,
title=plot_title,
log10=log10))
# Also plot errorbar without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = algorithm_gat_per_duration.pop(
algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
# Plot it if there was anything to remove and there is still data left over
if removed and algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {} without {}".
format(domain, instance, domain_configuration,
removed))
file_header = "{}_{}_NO_{}".format(domain,
instance_file_name,
removed)
plots_markdown += do_plot(
file_header, "error", lambda: plotutils.
plot_gat_duration_error(algorithm_gat_per_duration,
astar_gat_per_duration,
all_action_durations_ms,
title=plot_title))
if not error_only:
for action_duration in all_action_durations:
action_plot_title = plot_title + " - " + str(
int(plotutils.cnv_ns_to_ms(action_duration))) + " ms"
file_header = "{}_{}_{}".format(domain, instance_file_name,
action_duration)
# Gather GAT data
gat_data, gat_labels, gat_indices = get_gat_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration, old_idle_time)
y_gat = gat_data['goalAchievementTime']
y_idle = gat_data['idlePlanningTime']
# Gather node data
node_data, node_labels, node_indices = get_node_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration)
y_generated = node_data['generatedNodes']
y_expanded = node_data['expandedNodes']
if y_gat or y_generated:
action_duration_level = instance_level + "#"
plots_markdown += "{} Action Duration: {} ns\n\n".format(
action_duration_level, action_duration)
def plot_stacked(data,
labels,
file_header=file_header,
print_suffix=""):
stacked_markdown = ""
if not quiet:
msg = "Plotting stacked bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
stacked_markdown += do_plot(
file_header, "stacked", lambda: plotutils.
plot_gat_stacked_bars(data,
labels,
title=action_plot_title,
log10=log10))
return stacked_markdown
def plot_node_bars(data,
labels,
file_header=file_header,
print_suffix=""):
nodes_markdown = ""
if not quiet:
msg = "Plotting node bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
nodes_markdown += do_plot(
file_header, "nodes", lambda: plotutils.
plot_node_count_bars(data,
labels,
title=action_plot_title,
log10=log10))
return nodes_markdown
if y_gat and y_idle:
plots_markdown += plot_stacked(gat_data, gat_labels)
removed = remove_algorithms(gat_data, gat_labels,
gat_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and gat_data[
'goalAchievementTime'] and gat_data[
"idlePlanningTime"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_stacked(
gat_data,
gat_labels,
file_header=new_file_header,
print_suffix=suffix)
if y_generated and y_expanded:
plots_markdown += plot_node_bars(
node_data, node_labels)
removed = remove_algorithms(node_data, node_labels,
node_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and node_data[
'generatedNodes'] and node_data[
"expandedNodes"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_node_bars(
node_data,
node_labels,
file_header=new_file_header,
print_suffix=suffix)
if not plots_markdown:
return ""
else:
instance_markdown_document = "{} Instance: {}\n\n".format(
instance_level, instance)
return instance_markdown_document + plots_markdown
if domain_configurations:
for domain_configuration in domain_configurations:
instance_markdown = ""
for instance in instances:
instance_markdown += plot_domain_instance(
instance, domain_configuration)
if instance_markdown:
markdown_document += "# Configuration {}\n\n".format(
concatenate_configuration(domain_configuration,
separator=" ",
include_names=True))
markdown_document += instance_markdown
else:
for instance in instances:
markdown_document += plot_domain_instance(instance, None)
# Produce markdown file and convert to pdf if pandoc present
if markdown_summary:
if not quiet:
print("Saving markdown file")
file_header = "plots/{}_plots".format(domain)
markdown_file = "{}.md".format(file_header)
pdf_file = "{}.pdf".format(file_header)
save_to_file(markdown_file, markdown_document)
pandoc = which("pandoc")
if pandoc:
call([pandoc, "-o", pdf_file, markdown_file])
# Plot average data
remove_empty_data(all_error_data)
if plot_average:
if not quiet:
print("Plotting {} averages".format(domain))
lgd = plotutils.plot_gat_duration_error(
all_error_data,
all_astar_error_data,
all_action_durations_ms,
title="{} data over all instances".format(
plotutils.translate_domain_name(domain)))
plotutils.save_plot(plt, "plots/{}_average.pdf".format(domain), lgd)
plt.close('all')
# Plot averages without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = all_error_data.pop(algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
if removed:
lgd = plotutils.plot_gat_duration_error(
all_error_data,
all_astar_error_data,
all_action_durations_ms,
title="{} data over all instances no {}".format(
plotutils.translate_domain_name(domain), removed))
plotutils.save_plot(
plt, "plots/{}_NO_{}_average.pdf".format(domain, removed), lgd)
plt.close('all')
