def plotly_single_run_set(run_id, run_info=None):
# Establish a database connection
pgdb = DBUtils()
# Fetch this run's information, if not provided.
if run_info is None:
run_info = pgdb.fetchRunInfo(run_id)[run_id]
# Determine the maximum amount of food and moves possible.
trail_data = pgdb.getTrailData(run_info["trails_id"])[0]
max_food = np.bincount(np.squeeze(np.asarray(trail_data.flatten())))[1]
max_moves = np.array(run_info["moves_limit"])
# Fetch the data on the run and determine number of generations.
gens_data = pgdb.fetchRunGenerations([run_id])[run_id]
num_gens = len(gens_data)
x = np.linspace(0, num_gens - 1, num=num_gens)
# Settings used for plotting.
chart_set_config = {
"food": {
"db_key": "food",
"stats": ["max", "min", "avg", "std"],
"title": "Food vs. Generations for Run ID {0}",
"type": Scatter,
"plot-mode": "lines",
"xaxis": "Generations",
"yaxis": "Food Consumed",
"max-line": max_food,
"max-title": "Available"
},
"moves-taken": {
"db_key": "moves",
"stats": ["max", "min", "avg", "std"],
"title": "Moves Taken vs. Generations for Run ID {0}",
"type": Scatter,
"plot-mode": "lines",
"xaxis": "Generations",
"yaxis": "Moves Taken",
"max-line": max_moves,
"max-title": "Limit"
},
"moves-dir": {
"db_key": "moves",
"stats": ["left", "right", "forward", "none"],
"title": "Move Types vs. Generations for Run ID {0}",
"type": Scatter,
"plot-mode": "lines",
"xaxis": "Generations",
"yaxis": "Move Type",
"max-line": None,
}
}
plot_urls = {}
# TODO: Could multithread here to speed things up.
for chart_type, settings in chart_set_config.items():
traces_list = []
# Go through each of the stats and build the traces.
for stat in settings["stats"]:
data_set = np.zeros((num_gens))
for curr_gen in range(0, num_gens):
data_set[curr_gen] = (
gens_data[curr_gen][settings["db_key"]][stat])
this_trace = settings["type"](x=x,
y=data_set,
mode=settings["plot-mode"],
name=stat.title())
traces_list.append(this_trace)
# If desired, add the maximum line.
if settings["max-line"] is not None:
y_val = np.empty(len(x))
y_val.fill(settings["max-line"])
traces_list.append(settings["type"](
x=x,
y=y_val,
mode="lines",
line={
"dash": "dash"
},
name=settings["max-title"].title()))
layout = Layout(
title=settings["title"].format(run_id),
xaxis=XAxis(title=settings["xaxis"].format(run_id)),
yaxis=YAxis(title=settings["yaxis"].format(run_id)),
)
fig = Figure(data=Data(traces_list), layout=layout)
# Generate the URL.
plot_urls[chart_type] = chart.__generate_plotly_url(
fig,
filename="apigen/{0}_{1}".format(chart_type, run_id),
fileopt='overwrite',
)
return plot_urls
class chart:
def __init__(self):
self.__pgdb = DBUtils()
# Caches used for some functions.
self.__gen_data_cache = {}
@staticmethod
def __generate_plotly_url(fig, **kwargs):
""" Returns a ready-to-embed URL to a provided fig.
"""
# Sign in, if necessary.
if py.get_credentials()["username"] == "":
py.sign_in("jmoles", os.environ.get('PLOTLY_API_KEY'))
return py.plot(fig, auto_open=False, **kwargs)
@staticmethod
def plotly_single_run_set(run_id, run_info=None):
# Establish a database connection
pgdb = DBUtils()
# Fetch this run's information, if not provided.
if run_info is None:
run_info = pgdb.fetchRunInfo(run_id)[run_id]
# Determine the maximum amount of food and moves possible.
trail_data = pgdb.getTrailData(run_info["trails_id"])[0]
max_food = np.bincount(np.squeeze(np.asarray(trail_data.flatten())))[1]
max_moves = np.array(run_info["moves_limit"])
# Fetch the data on the run and determine number of generations.
gens_data = pgdb.fetchRunGenerations([run_id])[run_id]
num_gens = len(gens_data)
x = np.linspace(0, num_gens - 1, num=num_gens)
# Settings used for plotting.
chart_set_config = {
"food": {
"db_key": "food",
"stats": ["max", "min", "avg", "std"],
"title": "Food vs. Generations for Run ID {0}",
"type": Scatter,
"plot-mode": "lines",
"xaxis": "Generations",
"yaxis": "Food Consumed",
"max-line": max_food,
"max-title": "Available"
},
"moves-taken": {
"db_key": "moves",
"stats": ["max", "min", "avg", "std"],
"title": "Moves Taken vs. Generations for Run ID {0}",
"type": Scatter,
"plot-mode": "lines",
"xaxis": "Generations",
"yaxis": "Moves Taken",
"max-line": max_moves,
"max-title": "Limit"
},
"moves-dir": {
"db_key": "moves",
"stats": ["left", "right", "forward", "none"],
"title": "Move Types vs. Generations for Run ID {0}",
"type": Scatter,
"plot-mode": "lines",
"xaxis": "Generations",
"yaxis": "Move Type",
"max-line": None,
}
}
plot_urls = {}
# TODO: Could multithread here to speed things up.
for chart_type, settings in chart_set_config.items():
traces_list = []
# Go through each of the stats and build the traces.
for stat in settings["stats"]:
data_set = np.zeros((num_gens))
for curr_gen in range(0, num_gens):
data_set[curr_gen] = (
gens_data[curr_gen][settings["db_key"]][stat])
this_trace = settings["type"](x=x,
y=data_set,
mode=settings["plot-mode"],
name=stat.title())
traces_list.append(this_trace)
# If desired, add the maximum line.
if settings["max-line"] is not None:
y_val = np.empty(len(x))
y_val.fill(settings["max-line"])
traces_list.append(settings["type"](
x=x,
y=y_val,
mode="lines",
line={
"dash": "dash"
},
name=settings["max-title"].title()))
layout = Layout(
title=settings["title"].format(run_id),
xaxis=XAxis(title=settings["xaxis"].format(run_id)),
yaxis=YAxis(title=settings["yaxis"].format(run_id)),
)
fig = Figure(data=Data(traces_list), layout=layout)
# Generate the URL.
plot_urls[chart_type] = chart.__generate_plotly_url(
fig,
filename="apigen/{0}_{1}".format(chart_type, run_id),
fileopt='overwrite',
)
return plot_urls
@staticmethod
def sweep_charts(db_data,
config_id,
config_info,
sweep_type,
x_label,
y_label=None):
""" Given a set of db_data from
DBUtils.fetch_run_config_sweep_by_network along with the config_id,
and maximum amount of food, generates a food and moves taken sweep
plot.
Returns ready to embed URLs.
"""
plot_urls = {}
is_3d = False # Determines if plot is 3d
# Determine how to label the axes.
if sweep_type == "selection":
# Grab the x-axis labels for this plot.
x_label_vals = [y[3] for y in [db_data[x][0] for x in db_data]]
else:
x_label_vals = sorted(db_data.keys())
chart_set_config = {
"food": {
"title": "Food vs. {0} Sweep".format(x_label),
"db-idx": 0,
"val-func": [max, np.average],
"plot-mode": "lines",
"xaxis": x_label.title(),
"yaxis": "Food Consumed",
"max-line": config_info["max_food"],
"max-title": "Available",
"label": ["max", "mean", "std"]
},
"moves-taken": {
"title": "Moves Taken vs. {0} Sweep".format(x_label),
"db-idx": 1,
"val-func": [min, np.average],
"plot-mode": "lines",
"xaxis": x_label.title(),
"yaxis": "Moves Taken",
"label": ["min", "mean", "std"]
},
"num-runs": {
"title": "Number of runs",
"db-idx": 1,
"val-func": [len],
"plot-mode": "lines",
"xaxis": x_label.title(),
"yaxis": "Moves Taken",
"label": ["min", "mean", "std"]
},
}
# Add the max line for moves if not "moves_limit" type.
if sweep_type != "moves_limit":
chart_set_config["moves-taken"]["max-line"] = (
config_info["moves_limit"])
chart_set_config["moves-taken"]["max-title"] = "Limit"
if (sweep_type == "p_mutate_crossover"
or sweep_type == "dl_length_hidden"):
for curr_key in chart_set_config.keys():
chart_set_config[curr_key]["xaxis"] = x_label
chart_set_config[curr_key]["yaxis"] = y_label
chart_set_config[curr_key]["type"] = Heatmap
if curr_key == "food":
chart_set_config[curr_key]["zaxis"] = "Food Consumed"
chart_set_config[curr_key]["title"] = "Food 3D Sweep"
chart_set_config[curr_key]["val-func"] = [max]
elif curr_key == "moves-taken":
chart_set_config[curr_key]["zaxis"] = "Food Consumed"
chart_set_config[curr_key][
"title"] = "Moves Taken 3D Sweep"
chart_set_config[curr_key]["val-func"] = [min]
elif curr_key == "num-runs":
chart_set_config[curr_key]["zaxis"] = "Number of Runs"
if sweep_type == "p_mutate_crossover":
step_size = 0.1
else:
step_size = 1.0
chart_set_config[curr_key]["step-size"] = step_size
is_3d = True
# TODO: Could multithread here to speed things up.
for chart_type, settings in chart_set_config.items():
traces_list = []
for idx, this_func in enumerate(settings["val-func"]):
x_vals = []
y_vals = []
z_vals = []
y_std_dev = []
if is_3d:
y_vals = sorted(db_data.keys())
# Need to find the length of x and min/max x to
# figure out the labels and empty spots on heat chart.
len_y = len(y_vals)
x_vals = []
for cy in y_vals:
curr_x = sorted(db_data[cy].keys())
x_vals.extend(curr_x)
x_vals = list(set(x_vals))
x_vals.sort()
y_vals = list(
np.around(np.arange(start=min(y_vals),
stop=max(y_vals) +
settings["step-size"],
step=settings["step-size"]),
decimals=4))
x_vals = list(
np.around(np.arange(start=min(x_vals),
stop=max(x_vals) +
settings["step-size"],
step=settings["step-size"]),
decimals=4))
# Go through all of the y/x values and fill in z.
for cy in y_vals:
this_z = dict.fromkeys(x_vals)
if cy in db_data:
for cx in sorted(db_data[cy].keys()):
this_z[cx] = this_func([
x[settings["db-idx"]]
for x in db_data[cy][cx]
])
this_z = [myz[1] for myz in sorted(this_z.items())]
z_vals.append(this_z)
this_trace = settings["type"](
x=x_vals,
y=y_vals,
z=z_vals,
name=settings["label"][idx].title())
else:
for curr_x in sorted(db_data.keys()):
y_vals.append(
this_func([
x[settings["db-idx"]] for x in db_data[curr_x]
]))
if this_func == np.average:
y_std_dev.append(
np.std([
x[settings["db-idx"]]
for x in db_data[curr_x]
]))
if this_func == np.average:
this_trace = Scatter(
x=x_label_vals,
y=y_vals,
mode=settings["plot-mode"],
name=settings["label"][idx].title(),
error_y=ErrorY(
type='data',
array=y_std_dev,
visible=True,
))
else:
this_trace = Scatter(
x=x_label_vals,
y=y_vals,
mode=settings["plot-mode"],
name=settings["label"][idx].title())
traces_list.append(this_trace)
# If desired, add the maximum line.
if "max-line" in settings and not is_3d:
y_val = np.empty(len(x_label_vals))
y_val.fill(settings["max-line"])
traces_list.append(
Scatter(x=x_label_vals,
y=y_val,
mode="lines",
line={"dash": "dash"},
name=settings["max-title"].title()))
layout = Layout(
title=settings["title"],
xaxis=XAxis(title=settings["xaxis"]),
yaxis=YAxis(title=settings["yaxis"]),
)
fig = Figure(data=Data(traces_list), layout=layout)
# Generate the URL.
plot_urls[chart_type] = chart.__generate_plotly_url(
fig,
filename="apigen/sweep_{0}_{1}_{2}".format(
''.join(e for e in x_label if e.isalnum()), config_id,
chart_type),
fileopt="overwrite")
return plot_urls
def line_by_config_id(self,
config_id,
ext="png",
stat_group="food",
stat=None,
show_title=True):
if stat_group == "moves_stats" and stat == None:
stat = ["left", "right", "forward", "none"]
elif stat == None:
stat = ["min", "max", "avg"]
# Get the list of run_ids with this configuration.
run_ids_l = self.__pgdb.getRunsWithConfigID(config_id)
# Generate the figure and axes common to all of these.
fig = pyplot.Figure()
axis = fig.add_subplot(1, 1, 1)
# Get information on the run
run_info = self.__pgdb.fetchConfigInfo(config_id)
max_food = run_info["max_food"]
# Find the network name, trail name, and number of generations.
net_name = run_info["network_name"]
trail_name = run_info["trail_name"]
num_gens = run_info["generations"]
max_moves = np.array(run_info["moves_limit"])
# Take each run and now fetch data for each.
ids_search_l = []
for curr_id in run_ids_l:
if not self.__gen_data_cache.has_key(curr_id):
ids_search_l.append(curr_id)
if len(ids_search_l) > 0:
self.__gen_data_cache = dict(
self.__gen_data_cache.items() +
self.__pgdb.fetchRunGenerations(ids_search_l).items())
gens_data = self.__gen_data_cache
x = np.linspace(0, num_gens - 1, num=num_gens)
for curr_stat in stat:
data_set = np.zeros((num_gens))
for curr_gen in range(0, num_gens):
if stat_group == "moves_stats":
curr_stat_group = "moves"
else:
curr_stat_group = stat_group
this_gen = []
for curr_run in run_ids_l:
if curr_gen in gens_data[curr_run]:
this_gen.append(gens_data[curr_run][curr_gen]
[curr_stat_group][curr_stat])
else:
this_gen.append(None)
data_set[curr_gen] = np.mean(
filter(lambda a: a is not None, this_gen))
axis.plot(x, data_set, '-', label=curr_stat.title())
if show_title:
plot_title = ("Mean - {0} - {1} g{2}/p{3}".format(
net_name, trail_name, num_gens, run_info["population"]))
axis.set_title(plot_title)
# Determine the maximum type to show.
if stat_group == "food":
axis.plot(x, np.repeat(np.array(max_food), num_gens), 'r--')
axis.axis((0, num_gens, 0, max_food + 5))
axis.set_ylabel("Food Consumed")
axis.set_xlabel("Generations")
axis.legend(loc="best")
elif stat_group == "moves":
axis.plot(x, np.repeat(np.array(max_moves), num_gens), 'r--')
axis.axis((0, num_gens, 0, max_moves + 5))
axis.set_ylabel("Moves Taken")
axis.set_xlabel("Generations")
axis.legend(loc="lower left")
elif stat_group == "moves_stats":
axis.axis((0, num_gens, 0, max_moves + 5))
axis.set_ylabel("Moves Taken")
axis.set_xlabel("Generations")
axis.legend(loc="upper left", ncol=2)
fig.set_facecolor('w')
return (self.__createImage(fig, ext), len(run_ids_l))
def __createImage(self, fig, ext="jpg"):
""" Takes a matplotlib fig and generates given ext type.
Returns
"""
canvas = pltagg.FigureCanvasAgg(fig)
output = StringIO.StringIO()
if ext == "tif" or ext == "tiff":
canvas.print_tif(output)
elif ext == "bmp":
canvas.print_bmp(output)
elif ext == "eps":
canvas.print_eps(output)
elif ext == "png":
canvas.print_png(output)
elif ext == "pdf":
canvas.print_pdf(output)
elif ext == "svg":
canvas.print_svg(output)
else:
canvas.print_jpg(output)
return output
