def gather_gat_data(db, query: dict, old_idle_time: bool = False):
""" Queries the results collection and parses all returned data for goal achievement time (GAT) and idle planning
time.
:param db: the db handler
:param query: the query
:param old_idle_time: whether to use the old idle planning time parameters before a designated field was added
:return: the GAT and idle planning times as separate lists
"""
data_tiles = db.experimentResult.find(query)
data = []
idle_data = []
for result in data_tiles:
data.append(plotutils.cnv_ns_to_ms(result['result']['goalAchievementTime']))
if old_idle_time:
algorithm = query["result.experimentConfiguration.algorithmName"]
if algorithm is "A_STAR" or algorithm is "ARA_STAR" or algorithm is "WEIGHTED_A_STAR":
idle_data.append(plotutils.cnv_ns_to_ms(result['result']['planningTime']))
elif algorithm is "LSS_LRTA_STAR" or algorithm is "DYNAMIC_F_HAT" or algorithm is "RTA_STAR":
idle_data.append(plotutils.cnv_ns_to_ms(result['result']['experimentConfiguration']['actionDuration']))
else:
print("Don't know how to get idle planning time for {}!!!!".format(algorithm))
else:
idle_data.append(plotutils.cnv_ns_to_ms(result['result']['idlePlanningTime']))
return data, idle_data
def gather_gat_data(db, query: dict, old_idle_time: bool = False):
""" Queries the results collection and parses all returned data for goal achievement time (GAT) and idle planning
time.
:param db: the db handler
:param query: the query
:param old_idle_time: whether to use the old idle planning time parameters before a designated field was added
:return: the GAT and idle planning times as separate lists
"""
data_tiles = db.experimentResult.find(query)
data = []
idle_data = []
for result in data_tiles:
data.append(
plotutils.cnv_ns_to_ms(result['result']['goalAchievementTime']))
if old_idle_time:
algorithm = query["result.experimentConfiguration.algorithmName"]
if algorithm is "A_STAR" or algorithm is "ARA_STAR" or algorithm is "WEIGHTED_A_STAR":
idle_data.append(
plotutils.cnv_ns_to_ms(result['result']['planningTime']))
elif algorithm is "LSS_LRTA_STAR" or algorithm is "DYNAMIC_F_HAT" or algorithm is "RTA_STAR":
idle_data.append(
plotutils.cnv_ns_to_ms(
result['result']['experimentConfiguration']
['actionDuration']))
else:
print("Don't know how to get idle planning time for {}!!!!".
format(algorithm))
else:
idle_data.append(
plotutils.cnv_ns_to_ms(result['result']['idlePlanningTime']))
return data, idle_data
def gather_gat_data(db, query, old_idle_time: bool = False):
data_tiles = db.experimentResultV5_inertia.find(query)
data = []
idle_data = []
for result in data_tiles:
data.append(
plotutils.cnv_ns_to_ms(result['result']['goalAchievementTime']))
if old_idle_time:
algorithm = query["result.experimentConfiguration.algorithmName"]
if algorithm is "A_STAR" or algorithm is "ARA_STAR" or algorithm is "WEIGHTED_A_STAR":
idle_data.append(
plotutils.cnv_ns_to_ms(result['result']['planningTime']))
elif algorithm is "LSS_LRTA_STAR" or algorithm is "DYNAMIC_F_HAT" or algorithm is "RTA_STAR":
idle_data.append(
plotutils.cnv_ns_to_ms(
result['result']['experimentConfiguration']
['actionDuration']))
else:
print("Don't know how to get idle planning time for {}!!!!".
format(algorithm))
else:
idle_data.append(
plotutils.cnv_ns_to_ms(result['result']['idlePlanningTime']))
return data, idle_data
def get_gat_per_duration_data(db,
algorithm,
domain,
instance,
planner_configuration: dict = None,
domain_configuration: dict = None,
execution=False):
""" Retrieves goal achievement time (GAT) data for each action duration in the results collection for the given
algorithm/domain/instance combination.
:param db: the db handler
:param algorithm: the algorithm to gather data for
:param domain: the domain to gather data for
:param instance: the domain instance to gather data for
:param planner_configuration: the configuration of the planner algorithm
:param domain_configuration: the configuration of the domain
:param execution: whether to retrieve action execution time instead of goal achivement time
:return: the data as a list of lists where each inner list is designated to a different action duration
"""
data_action_durations = []
query = {
"result.experimentConfiguration.domainName": domain,
"result.experimentConfiguration.algorithmName": algorithm,
"result.experimentConfiguration.domainInstanceName": instance,
"result.success": True
}
if planner_configuration:
for name, value in planner_configuration.items():
query[get_configuration_query(name)] = value
if domain_configuration:
for name, value in domain_configuration.items():
query[get_configuration_query(name)] = value
for action_duration in all_action_durations:
query[get_configuration_query("actionDuration")] = action_duration
data_tiles = db.experimentResult.find(query)
times_for_durations = []
for result in data_tiles:
if execution:
times_for_durations.append(
plotutils.cnv_ns_to_ms(
result['result']['actionExecutionTime']))
else:
times_for_durations.append(
plotutils.cnv_ns_to_ms(
result['result']['goalAchievementTime']))
# if times_for_durations: # not empty
data_action_durations.append(times_for_durations)
return data_action_durations
def get_gat_per_duration_data(db, algorithm, domain, instance, planner_configuration: dict = None,
domain_configuration: dict = None, execution=False):
""" Retrieves goal achievement time (GAT) data for each action duration in the results collection for the given
algorithm/domain/instance combination.
:param db: the db handler
:param algorithm: the algorithm to gather data for
:param domain: the domain to gather data for
:param instance: the domain instance to gather data for
:param planner_configuration: the configuration of the planner algorithm
:param domain_configuration: the configuration of the domain
:param execution: whether to retrieve action execution time instead of goal achivement time
:return: the data as a list of lists where each inner list is designated to a different action duration
"""
data_action_durations = []
query = {
"result.experimentConfiguration.domainName": domain,
"result.experimentConfiguration.algorithmName": algorithm,
"result.experimentConfiguration.domainInstanceName": instance,
"result.success": True
}
if planner_configuration:
for name, value in planner_configuration.items():
query[get_configuration_query(name)] = value
if domain_configuration:
for name, value in domain_configuration.items():
query[get_configuration_query(name)] = value
for action_duration in all_action_durations:
query[get_configuration_query("actionDuration")] = action_duration
data_tiles = db.experimentResult.find(query)
times_for_durations = []
for result in data_tiles:
if execution:
times_for_durations.append(plotutils.cnv_ns_to_ms(result['result']['actionExecutionTime']))
else:
times_for_durations.append(plotutils.cnv_ns_to_ms(result['result']['goalAchievementTime']))
# if times_for_durations: # not empty
data_action_durations.append(times_for_durations)
return data_action_durations
def get_gat_per_duration_data(db,
algorithm,
domain,
instance,
planner_configuration: dict = None,
domain_configuration: dict = None):
data_action_durations = []
query = {
"experimentConfiguration.domainName": domain,
"experimentConfiguration.algorithmName": algorithm,
"experimentConfiguration.domainPath": instance,
"success": True
}
if planner_configuration:
for name, value in planner_configuration.items():
query[get_configuration_query(name)] = value
if domain_configuration:
for name, value in domain_configuration.items():
query[get_configuration_query(name)] = value
for action_duration in all_action_durations:
query[get_configuration_query("actionDuration")] = action_duration
data_tiles = db.experimentResultV5_inertia.find(query)
times_for_durations = []
for result in data_tiles:
times_for_durations.append(
plotutils.cnv_ns_to_ms(
result['result']['goalAchievementTime']))
# if times_for_durations: # not empty
data_action_durations.append(times_for_durations)
return data_action_durations
configurations.append({
"numActions": num_actions,
"actionFraction": action_fraction,
"stateFraction": state_fraction
})
return configurations
# All valid action durations
all_action_durations = (6000000, 10000000, 20000000, 40000000, 80000000,
160000000, 320000000, 640000000)
# all_action_durations = (850000000, 960000000, 1070000000, 1280000000)
# All valid action durations converted to milliseconds
all_action_durations_ms = [
plotutils.cnv_ns_to_ms(duration) for duration in all_action_durations
]
# All planning algorithms and their configurations
# all_algorithms = ["A_STAR", "ARA_STAR", "RTA_STAR", "LSS_LRTA_STAR", "DYNAMIC_F_HAT"]
ss_configuration = {"timeBoundType": "STATIC", "commitmentStrategy": "SINGLE"}
sm_configuration = {
"timeBoundType": "STATIC",
"commitmentStrategy": "MULTIPLE"
}
dm_configuration = {
"timeBoundType": "DYNAMIC",
"commitmentStrategy": "MULTIPLE"
}
all_algorithms = {
"A_STAR": [],
configurations = []
for num_actions in pri_num_actions:
for action_fraction in pri_action_fractions:
for state_fraction in pri_state_fractions:
configurations.append({"numActions": num_actions,
"actionFraction": action_fraction,
"stateFraction": state_fraction})
return configurations
# All valid action durations
all_action_durations = (6000000, 10000000, 20000000, 40000000, 80000000, 160000000, 320000000, 640000000)
# all_action_durations = (850000000, 960000000, 1070000000, 1280000000)
# All valid action durations converted to milliseconds
all_action_durations_ms = [plotutils.cnv_ns_to_ms(duration) for duration in all_action_durations]
# All planning algorithms and their configurations
# all_algorithms = ["A_STAR", "ARA_STAR", "RTA_STAR", "LSS_LRTA_STAR", "DYNAMIC_F_HAT"]
ss_configuration = {"timeBoundType": "STATIC", "commitmentStrategy": "SINGLE"}
sm_configuration = {"timeBoundType": "STATIC", "commitmentStrategy": "MULTIPLE"}
dm_configuration = {"timeBoundType": "DYNAMIC", "commitmentStrategy": "MULTIPLE"}
all_algorithms = {"A_STAR": [], "ARA_STAR": [], "RTA_STAR": [],
"LSS_LRTA_STAR": [ss_configuration, sm_configuration, dm_configuration],
"DYNAMIC_F_HAT": [ss_configuration, sm_configuration, dm_configuration]}
plot_algorithm_names = get_plot_algorithm_names()
# All domains and their configurations
all_domains = {"GRID_WORLD": [],
"SLIDING_TILE_PUZZLE_4": [],
"ACROBOT": [],
def plot_domain_instance(instance, domain_configuration):
instance_file_name = instance.replace("/", "_")
plot_title = plotutils.translate_domain_instance_name(
domain, instance) # translated_domain_name + " - " + instance
plots_markdown = ""
instance_level = "#"
if domain_configuration:
instance_level += "#"
instance_file_name += "_" + concatenate_configuration(
domain_configuration)
if not quiet:
print("Processing {} - {}".format(domain, instance))
# Gather error data for each algorithm
astar_gat_per_duration = get_gat_per_duration_data(
db, "A_STAR", domain, instance, domain_configuration)
algorithm_gat_per_duration = {}
for algorithm, configurations in all_algorithms.items():
if not configurations:
algorithm_gat_per_duration[algorithm] = \
get_gat_per_duration_data(db, algorithm, domain, instance, domain_configuration)
else:
for configuration in configurations:
algorithm_gat_per_duration[get_configuration_plot_name(algorithm, configuration)] = \
get_gat_per_duration_data(db, algorithm, domain, instance, configuration, domain_configuration)
remove_empty_data(algorithm_gat_per_duration)
# Store data to compute average later
for algorithm, values in algorithm_gat_per_duration.items():
count = 0
for val in values:
if val:
all_error_data[algorithm][count].append(val[0])
count += 1
count = 0
for val in astar_gat_per_duration:
if val:
all_astar_error_data[count].append(val[0])
else:
all_astar_error_data[count].append([])
count += 1
# Do individual plots if average_only not specified
if not average_only:
# Errorbar plot
if algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {}".format(
domain, instance, domain_configuration))
file_header = "{}_{}".format(domain, instance_file_name)
plots_markdown += do_plot(
file_header, "error", lambda: plotutils.
plot_gat_duration_error(algorithm_gat_per_duration,
astar_gat_per_duration,
all_action_durations_ms,
title=plot_title,
log10=log10))
# Also plot errorbar without exception algorithms
removed = ""
for algorithm in plot_without:
algorithm_data = algorithm_gat_per_duration.pop(
algorithm, None)
if algorithm_data is not None:
if not removed: # empty
removed += algorithm
else:
removed += "_" + algorithm
# Plot it if there was anything to remove and there is still data left over
if removed and algorithm_gat_per_duration:
if not quiet:
print("Plotting error plot: {} - {} - {} without {}".
format(domain, instance, domain_configuration,
removed))
file_header = "{}_{}_NO_{}".format(domain,
instance_file_name,
removed)
plots_markdown += do_plot(
file_header, "error", lambda: plotutils.
plot_gat_duration_error(algorithm_gat_per_duration,
astar_gat_per_duration,
all_action_durations_ms,
title=plot_title))
if not error_only:
for action_duration in all_action_durations:
action_plot_title = plot_title + " - " + str(
int(plotutils.cnv_ns_to_ms(action_duration))) + " ms"
file_header = "{}_{}_{}".format(domain, instance_file_name,
action_duration)
# Gather GAT data
gat_data, gat_labels, gat_indices = get_gat_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration, old_idle_time)
y_gat = gat_data['goalAchievementTime']
y_idle = gat_data['idlePlanningTime']
# Gather node data
node_data, node_labels, node_indices = get_node_data(
db, all_algorithms.keys(), domain, instance,
action_duration, domain_configuration)
y_generated = node_data['generatedNodes']
y_expanded = node_data['expandedNodes']
if y_gat or y_generated:
action_duration_level = instance_level + "#"
plots_markdown += "{} Action Duration: {} ns\n\n".format(
action_duration_level, action_duration)
def plot_stacked(data,
labels,
file_header=file_header,
print_suffix=""):
stacked_markdown = ""
if not quiet:
msg = "Plotting stacked bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
stacked_markdown += do_plot(
file_header, "stacked", lambda: plotutils.
plot_gat_stacked_bars(data,
labels,
title=action_plot_title,
log10=log10))
return stacked_markdown
def plot_node_bars(data,
labels,
file_header=file_header,
print_suffix=""):
nodes_markdown = ""
if not quiet:
msg = "Plotting node bars: {} - {} - {} - {}".format(
domain, instance, action_duration,
domain_configuration)
if print_suffix:
msg += " - {}".format(print_suffix)
print(msg)
nodes_markdown += do_plot(
file_header, "nodes", lambda: plotutils.
plot_node_count_bars(data,
labels,
title=action_plot_title,
log10=log10))
return nodes_markdown
if y_gat and y_idle:
plots_markdown += plot_stacked(gat_data, gat_labels)
removed = remove_algorithms(gat_data, gat_labels,
gat_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and gat_data[
'goalAchievementTime'] and gat_data[
"idlePlanningTime"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_stacked(
gat_data,
gat_labels,
file_header=new_file_header,
print_suffix=suffix)
if y_generated and y_expanded:
plots_markdown += plot_node_bars(
node_data, node_labels)
removed = remove_algorithms(node_data, node_labels,
node_indices, plot_without)
# Plot it if there was anything to remove and there is still data left over
if removed and node_data[
'generatedNodes'] and node_data[
"expandedNodes"]:
new_file_header = "{}_{}_{}_NO_{}".format(
domain, instance_file_name, action_duration,
removed)
suffix = "without {}".format(removed)
plots_markdown += plot_node_bars(
node_data,
node_labels,
file_header=new_file_header,
print_suffix=suffix)
if not plots_markdown:
return ""
else:
instance_markdown_document = "{} Instance: {}\n\n".format(
instance_level, instance)
return instance_markdown_document + plots_markdown