def _get_stage_data_helper(path, stage_idx):
"""
Return a dataframe, each row of which corresponds to the `stage_idx`-th stage
of a different run.
"""
job = HyperSearch(path)
stage_data = job.extract_stage_data()
dist_keys = job.dist_keys()
records = []
for i, (key, value) in enumerate(sorted(stage_data.items())):
for (repeat, seed), (df, sc, md) in value.items():
record = dict(df.iloc[stage_idx])
for dk in dist_keys:
record[dk] = md[dk]
record['idx'] = key.idx
record['repeat'] = repeat
record['seed'] = seed
records.append(record)
return pd.DataFrame.from_records(records)
def get_mnist_data(path, y_keys, spread_measure):
if isinstance(y_keys, str):
y_keys = y_keys.split()
if not isinstance(y_keys, dict):
y_keys = {yk: lambda y: y for yk in y_keys}
job = HyperSearch(path)
stage_data = job.extract_stage_data()
assert len(stage_data) == 1 # Should only be one parameter setting
stage_data = next(iter(stage_data.values()))
data = defaultdict(list)
for (repeat, seed), (df, sc, md) in stage_data.items():
for yk, _ in y_keys.items():
data[yk].append(df['_test_' + yk])
x = list(range(1, 13))
data_stats = {}
for yk, func in y_keys.items():
data_yk = np.array(data[yk]).T
data_yk = func(data_yk)
y = data_yk.mean(axis=1)
yu, yl = spread_measures[spread_measure](data_yk)
data_stats[yk] = (x, y, yu, yl)
return data_stats
def get_arithmetic_data(paths,
x_key,
y_key,
stage_idx,
spread_measure,
y_func=None):
y_func = y_func or (lambda y: y)
data = {}
for path in paths:
job = HyperSearch(path)
stage_data = job.extract_stage_data()
for i, (key, value) in enumerate(sorted(stage_data.items())):
_data = []
for (repeat, seed), (df, sc, md) in value.items():
_data.append(y_func(df[y_key][stage_idx]))
data[getattr(key, x_key)] = _data
x = sorted(data)
_data = np.array([data[key] for key in x])
y = _data.mean(axis=1)
yu, yl = spread_measures[spread_measure](_data)
return x, y, yu, yl
def _checkpoint(self, i):
print("Fetching results of step {} at: ".format(i))
print(datetime.datetime.now())
for i, host in enumerate(self.hosts):
if host == ':':
command = "mv {local_scratch}/experiments/* ./experiments"
self.execute_command(command, robust=True)
command = "rm -rf {local_scratch}/experiments"
self.execute_command(command, robust=True)
command = "cp -ru {local_scratch}/{archive_root} ."
self.execute_command(command, robust=True)
else:
command = (
"rsync -az {rsync_verbosity} --timeout=300 -e \"ssh {ssh_options}\" "
"{host}:{local_scratch}/experiments/ ./experiments".format(
host=host, **self.__dict__))
self.execute_command(command,
frmt=False,
robust=True,
output="loud")
command = "rm -rf {local_scratch}/experiments"
self.ssh_execute(command, host, robust=True, output="loud")
command = (
"rsync -az {rsync_verbosity} --timeout=300 -e \"ssh {ssh_options}\" "
"{host}:{local_scratch}/{archive_root} .".format(
host=host, **self.__dict__))
self.execute_command(command,
frmt=False,
robust=True,
output="loud")
self.execute_command("zip -rq results {archive_root}", robust=True)
try:
from dps.hyper import HyperSearch
search = HyperSearch('.')
with redirect_stream('stdout', 'results.txt', tee=False):
search.print_summary(print_config=False, verbose=False)
print(search.job.summary(verbose=False))
except Exception:
job_path = 'results.zip' if os.path.exists(
'results.zip') else 'orig.zip'
assert os.path.exists(job_path)
job = ReadOnlyJob(job_path)
print(job.summary(verbose=False))
def get_transfer_baseline_data(path, x_key, y_key, spread_measure, y_func=None):
y_func = y_func or (lambda y: y)
job = HyperSearch(path)
stage_data = job.extract_stage_data()
x = range(1, 21)
y = []
for i, (key, value) in enumerate(sorted(stage_data.items())):
data = []
for (repeat, seed), (df, sc, md) in value.items():
data.append(df[y_key][0])
data = y_func(np.array(data))
y.append(data.mean())
return x, np.array(y)
def get_shapes_data(paths, y_keys, spread_measure):
data = defaultdict(list)
x = np.arange(1, 36)
for path in paths:
ignore = []
# For two of the experiments, the training stalled just before starting the 4th stage, so ignore those samples.
if 'max-shapes=10_small=False_alg=shapes-silot_duration=long_2019_08_13_00_00_51_seed=0' in path:
ignore = [1]
if 'max-shapes=20_small=False_alg=shapes-silot_duration=long_2019_08_19_17_30_18_seed=0' in path:
ignore = [2]
if isinstance(y_keys, str):
y_keys = y_keys.split()
if not isinstance(y_keys, dict):
y_keys = {yk: lambda y: y for yk in y_keys}
job = HyperSearch(path)
stage_data = job.extract_stage_data()
for (idx, _), d in stage_data.items():
if idx in ignore:
continue
assert len(d) == 1
df = next(iter(d.values()))[0]
for yk, _ in y_keys.items():
data[yk].append(df['_test_' + yk][x-1])
x = list(x)
data_stats = {}
for yk, func in y_keys.items():
data_yk = np.array(data[yk]).T
data_yk = func(data_yk)
y = data_yk.mean(axis=1)
yu, yl = spread_measures[spread_measure](data_yk)
data_stats[yk] = (x, y, yu, yl)
return data_stats
def _print_config_cmd(path):
search = HyperSearch(path)
print("BASE CONFIG")
print(search.objects.load_object('metadata', 'config'))
dist = search.objects.load_object('metadata', 'dist')
dist = Config(dist)
print('\n' + '*' * 100)
print("PARAMETER DISTRIBUTION")
pprint(dist)
def get_transfer_data(path, x_key, y_key, spread_measure, y_func=None):
y_func = y_func or (lambda y: y)
job = HyperSearch(path)
stage_data = job.extract_stage_data()
all_data = []
for i, (key, value) in enumerate(sorted(stage_data.items())):
data = []
for (repeat, seed), (df, sc, md) in value.items():
data.append(df[y_key][1:]) # First stage is the learning stage.
data = np.array(data).T
data = y_func(data)
x = range(1, 21)
y = data.mean(axis=1)
yu, yl = spread_measures[spread_measure](data)
all_data.append(((x, y, yu, yl), key))
return all_data
def _resubmit_cmd(path, name=""):
""" Note the resubmitting still has a limitation: experiments are not copied over
from the previous submission. Couldn't find a clean way to do this, so just do it manually
for now. In the future we should revamp the build/run process so that the possibility of
multiple runs is taken into account, and the results of the runs can be easily combined.
"""
search = HyperSearch(path)
archive_path = search.job.path
try:
with open(os.path.join(search.path, 'run_kwargs.json'), 'r') as f:
reference = json.load(f)
except FileNotFoundError:
with open(os.path.join(search.path, 'session.pkl'), 'rb') as f:
reference = dill.load(f).__dict__
sig = inspect.signature(ParallelSession.__init__)
_run_kwargs = sig.bind_partial()
_run_kwargs.apply_defaults()
run_kwargs = {}
for k, v in _run_kwargs.arguments.items():
run_kwargs[k] = reference[k]
cl_run_kwargs = clify.command_line(run_kwargs).parse()
run_kwargs.update(cl_run_kwargs)
done = False
while not done:
print("Current values for run_kwargs: ")
pprint(run_kwargs)
inp = ""
while inp not in ["y", "n"]:
inp = input("Make changes? (y/n): ")
if inp == "y":
inp = input("Specify changes: ")
new = clify.command_line(run_kwargs, cl_args=inp).parse()
run_kwargs.update(new)
else:
done = True
submit_job(archive_path, "resubmit", **run_kwargs)
def _value_plot_cmd(path, mode, field, stage, x_axis, ylim, style):
""" Plot the trajectory of a single value, specified by field, for each parameter
setting in a hyperparameter search.
Example:
dps-hyper value_plot . rl_val COST_negative_mAP None --x-axis=stages --ylim="-1,0"
Parameters
----------
path: str
Path passed to `HyperSearch`.
mode: str
Run mode to plot from (e.g. train, val).
field: str
Name of value to plot.
stage: str
String that is eval-ed to get an object specifying the stages to plot data from.
x_axis: str, one of {steps, experiences, stages}
Specifiation of value to use as x-axis for plots. If `stages` is used, then only
the value obtained by the "chosen" hypothesis for that stage is used.
ylim: str
String that is eval-ed to get a tuple specifying y-limits for plots.
style: str
Matplotlib style to use for plot.
"""
print("Plotting {} value of field {} from experiments stored at `{}`.".format(mode, field, path))
assert x_axis in "steps experiences stages"
x_axis_key = dict(
steps="global_step",
experiences="n_global_experiences",
stages="stage_idx")[x_axis]
search = HyperSearch(path)
stage = eval(stage) if stage else ""
ylim = eval(ylim) if ylim else ""
fields = [field, x_axis_key]
if x_axis == "stages":
data = search.extract_stage_data(fields, bare=True)
else:
data = search.extract_step_data(mode, fields, stage)
n_plots = len(data) + 1
w = int(np.ceil(np.sqrt(n_plots)))
h = int(np.ceil(n_plots / w))
with plt.style.context(style):
fig, axes = plt.subplots(h, w, sharex=True, sharey=True, figsize=(15, 10))
fig.suptitle("{} vs {}".format(field, x_axis_key))
axes = np.atleast_2d(axes)
final_ax = axes[-1, -1]
label_order = []
for i, (key, value) in enumerate(sorted(data.items())):
label = ",".join("{}={}".format(*kv) for kv in zip(key._fields, key))
i = int(key.idx / w)
j = key.idx % w
ax = axes[i, j]
if ylim:
ax.set_ylim(ylim)
ax.set_title(label)
label_order.append(label)
for (repeat, seed), _data in value.items():
ax.plot(_data[x_axis_key], _data[field])
to_concat = [_data.set_index(x_axis_key) for _data in value.values()]
concat = pd.concat(to_concat, axis=1, ignore_index=True)
mean = concat.mean(axis=1)
final_ax.plot(mean, label=label)
legend_handles = {l: h for h, l in zip(*final_ax.get_legend_handles_labels())}
ordered_handles = [legend_handles[l] for l in label_order]
final_ax.legend(
ordered_handles, label_order, loc='center left',
bbox_to_anchor=(1.05, 0.5), ncol=1)
if ylim:
final_ax.set_ylim(ylim)
plt.subplots_adjust(left=0.05, bottom=0.05, right=0.9, top=0.90, wspace=0.05, hspace=0.18)
plt.savefig('value_plot_mode={}_field={}_stage={}'.format(mode, field, stage))
plt.show()
def _summarize_search_cmd(path, no_config, verbose, criteria, maximize):
search = HyperSearch(path)
search.print_summary(print_config=not no_config, verbose=verbose, criteria=criteria, maximize=maximize)
def _search_plot_cmd(
path, y_field, x_field, groupby, spread_measure,
style, do_legend=False, **axes_kwargs):
path = process_path(path)
print("Plotting searches stored at {}.".format(path))
search = HyperSearch(path)
with plt.style.context(style):
ax = plt.axes(xlabel=x_field, ylabel=y_field, **axes_kwargs)
dist = search.objects.load_object('metadata', 'dist')
dist = Config(dist)
df = search.extract_summary_data()
groups = sorted(df.groupby(groupby))
colours = plt.rcParams['axes.prop_cycle'].by_key()['color']
legend = []
for i, (k, _df) in enumerate(groups):
values = list(_df.groupby(x_field))
x = [v[0] for v in values]
ys = [v[1][y_field] for v in values]
y = [_y.mean() for _y in ys]
if spread_measure == 'std_dev':
y_upper = y_lower = [_y.std() for _y in ys]
elif spread_measure == 'conf_int':
conf_int = [confidence_interval(_y.values, 0.95) for _y in ys]
y_lower = y - np.array([ci[0] for ci in conf_int])
y_upper = np.array([ci[1] for ci in conf_int]) - y
elif spread_measure == 'std_err':
y_upper = y_lower = [standard_error(_y.values) for _y in ys]
else:
raise Exception("NotImplemented")
yerr = np.vstack((y_lower, y_upper))
c = colours[i % len(colours)]
ax.semilogx(x, y, c=c, basex=2)
handle = ax.errorbar(x, y, yerr=yerr, c=c)
label = "{} = {}".format(groupby, k)
legend.append((handle, label))
if do_legend:
handles, labels = zip(*legend)
ax.legend(
handles, labels, loc='center left',
bbox_to_anchor=(1.05, 0.5), ncol=1)
# plt.subplots_adjust(
# left=0.09, bottom=0.13, right=0.7, top=0.93, wspace=0.05, hspace=0.18)
filename = "value_plot.pdf"
print("Saving plot as {}".format(filename))
plt.savefig(filename)