def main(filter=None):
"""
Usage: python make_plots.py PATH_TO_DIRECTORY
"""
parser = argparse.ArgumentParser()
parser.add_argument('--clean', action='store_true', help='remove broken jobs')
parser.add_argument('--table', action='store_true', help='print table')
parser.add_argument('--csv', action='store_true', help='save table as csv')
parser.add_argument('--d', type=int, help='sort by distance from dth smallest result')
parser.add_argument('--name', help='experiment name', default=None)
args, unknown = parser.parse_known_args()
options, expt_dir = get_options(unknown)
# print "options:"
# print_dict(options)
# reduce the grid size
options["grid_size"] = 400
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
# print "chooser", chooser
if args.name:
experiment_name = args.name
else:
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
# sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
# testing below here
jobs = load_jobs(db, experiment_name)
print len(jobs), 'jobs found'
# print jobs
# remove_broken_jobs
if args.clean:
for job in jobs:
if job['status'] == 'pending':
sys.stderr.write('Broken job %s detected.\n' % job['id'])
job['status'] = 'broken'
db.save(job, experiment_name, 'jobs', {'id' : job['id']})
# print "resources:", resources
# print_dict(resources)
resource = resources.itervalues().next()
task_options = {task: options["tasks"][task] for task in resource.tasks}
# print "task_options:"
# print_dict(task_options) # {'main': {'likelihood': u'NOISELESS', 'type': 'OBJECTIVE'}}
task_group = load_task_group(db, options, experiment_name, resource.tasks)
hypers = load_hypers(db, experiment_name)
chooser.fit(task_group, hypers, task_options)
lp, x = chooser.best()
if args.table:
os.chdir(unknown[0])
out_file = open('results.csv', 'w') if args.csv else sys.stdout
# get the observed points
task = task_group.tasks.itervalues().next()
idata = task.valid_normalized_data_dict
inputs = idata["inputs"]
inputs = map(lambda i: [paramify(task_group, task_group.from_unit(i)).values(), i], inputs)
vals = idata["values"]
vals = [task.unstandardize_mean(task.unstandardize_variance(v)) for v in vals]
out_file.write('\n%10s' % 'result')
lengths = [10]
for name, vdict in task.variables_meta.iteritems():
name = '%10s' % name
out_file.write(',' + name)
lengths.append(len(name))
out_file.write('\n')
line_template = '%' + str(lengths[0]) + '.4f,' + ','.join(['%' + str(l) +
('.4f' if 'enum' not in inputs[0][0][i]['type'] else 's') for i, l in enumerate(lengths[1:])])
points = sorted(zip(vals, inputs), key=lambda r: r[0])
if args.d is not None:
target = x
if args.d >= 0:
target = points[args.d][1][1]
points = sorted(points, key=lambda r: np.linalg.norm(r[1][1] - target))
for i, point in enumerate(points):
subs = [point[0]] + [d['values'][0] for d in point[1][0]]
out_file.write(line_template % tuple(subs) + '\n')
out_file.close()
def main():
"""
Usage: python make_plots.py PATH_TO_DIRECTORY
TODO: Some aspects of this function are specific to the simple branin example
We should clean this up so that interpretation of plots are more clear and
so that it works in more general cases
(e.g. if objective likelihood is binomial then values should not be
unstandardized)
"""
options, expt_dir = get_options()
print "options:"
print_dict(options)
# reduce the grid size
options["grid_size"] = 400
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
print "chooser", chooser
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
# testing below here
jobs = load_jobs(db, experiment_name)
remove_broken_jobs(db, jobs, experiment_name, resources)
print "resources:", resources
print_dict(resources)
resource = resources.itervalues().next()
task_options = { task: options["tasks"][task] for task in resource.tasks }
print "task_options:"
print_dict(task_options) # {'main': {'likelihood': u'NOISELESS', 'type': 'OBJECTIVE'}}
task_group = load_task_group(db, options, resource.tasks)
print "task_group", task_group # TaskGroup
print "tasks:"
print_dict(task_group.tasks) # {'main': <spearmint.tasks.task.Task object at 0x10bf63290>}
hypers = load_hypers(db, experiment_name)
print "loaded hypers", hypers # from GP.to_dict()
hypers = chooser.fit(task_group, hypers, task_options)
print "\nfitted hypers:"
print_dict(hypers)
lp, x = chooser.best()
x = x.flatten()
print "best", lp, x
bestp = task_group.paramify(task_group.from_unit(x))
print "expected best position", bestp
# get the grid of points
grid = chooser.grid
# print "chooser objectives:",
# print_dict(chooser.objective)
print "chooser models:", chooser.models
print_dict(chooser.models)
obj_model = chooser.models[chooser.objective['name']]
obj_mean, obj_var = obj_model.function_over_hypers(obj_model.predict, grid)
# un-normalize the function values and variances
obj_task = task_group.tasks['main']
obj_mean = [obj_task.unstandardize_mean(obj_task.unstandardize_variance(v)) for v in obj_mean]
obj_std = [obj_task.unstandardize_variance(np.sqrt(v)) for v in obj_var]
# for xy, m, v in izip(grid, obj_mean, obj_var):
# print xy, m, v
grid = map(task_group.from_unit, grid)
# return
xymv = [(xy[0], xy[1], m, v) for xy, m, v in izip(grid, obj_mean, obj_std)]# if .2 < xy[0] < .25]
x = map(lambda x:x[0], xymv)
y = map(lambda x:x[1], xymv)
m = map(lambda x:x[2], xymv)
sig = map(lambda x:x[3], xymv)
# print y
fig = plt.figure(dpi=100)
ax = fig.add_subplot(111, projection='3d')
ax.plot(x, y, m, marker='.', linestyle="None")
# plot errorbars
for i in np.arange(0, len(x)):
ax.plot([x[i], x[i]], [y[i], y[i]], [m[i]+sig[i], m[i]-sig[i]], marker="_", color='k')
# get the observed points
task = task_group.tasks['main']
idata = task.valid_normalized_data_dict
xy = idata["inputs"]
xy = map(task_group.from_unit, xy)
xy = np.array(xy)
vals = idata["values"]
vals = [obj_task.unstandardize_mean(obj_task.unstandardize_variance(v)) for v in vals]
ax.plot(xy[:,0], xy[:,1], vals, marker='o', color="r", linestyle="None")
plt.show()
def main():
"""
Usage: python make_plots.py PATH_TO_DIRECTORY
TODO: Some aspects of this function are specific to the simple branin example
We should clean this up so that interpretation of plots are more clear and
so that it works in more general cases
(e.g. if objective likelihood is binomial then values should not be
unstandardized)
"""
options, expt_dir = get_options()
print("options:")
print_dict(options)
# reduce the grid size
options["grid_size"] = 400
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' +
options['chooser'])
chooser = chooser_module.init(options)
print("chooser", chooser)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
# testing below here
jobs = load_jobs(db, experiment_name)
remove_broken_jobs(db, jobs, experiment_name, resources)
print("resources:", resources)
print_dict(resources)
resource = resources.itervalues().next()
task_options = {task: options["tasks"][task] for task in resource.tasks}
print("task_options:")
print_dict(task_options
) # {'main': {'likelihood': u'NOISELESS', 'type': 'OBJECTIVE'}}
task_group = load_task_group(db, options, resource.tasks)
print("task_group", task_group) # TaskGroup
print("tasks:")
print_dict(task_group.tasks
) # {'main': <spearmint.tasks.task.Task object at 0x10bf63290>}
hypers = load_hypers(db, experiment_name)
print("loaded hypers", hypers) # from GP.to_dict()
hypers = chooser.fit(task_group, hypers, task_options)
print("\nfitted hypers:")
print_dict(hypers)
lp, x = chooser.best()
x = x.flatten()
print("best", lp, x)
bestp = task_group.paramify(task_group.from_unit(x))
print("expected best position", bestp)
# get the grid of points
grid = chooser.grid
# print("chooser objectives:", )
# print_dict(chooser.objective)
print("chooser models:", chooser.models)
print_dict(chooser.models)
obj_model = chooser.models[chooser.objective['name']]
obj_mean, obj_var = obj_model.function_over_hypers(obj_model.predict, grid)
# un-normalize the function values and variances
obj_task = task_group.tasks['main']
obj_mean = [
obj_task.unstandardize_mean(obj_task.unstandardize_variance(v))
for v in obj_mean
]
obj_std = [obj_task.unstandardize_variance(np.sqrt(v)) for v in obj_var]
# for xy, m, v in izip(grid, obj_mean, obj_var):
# print(xy, m, v)
grid = map(task_group.from_unit, grid)
# return
xymv = [(xy[0], xy[1], m, v) for xy, m, v in izip(grid, obj_mean, obj_std)
] # if .2 < xy[0] < .25]
x = map(lambda x: x[0], xymv)
y = map(lambda x: x[1], xymv)
m = map(lambda x: x[2], xymv)
sig = map(lambda x: x[3], xymv)
# print(y)
fig = plt.figure(dpi=100)
ax = fig.add_subplot(111, projection='3d')
ax.plot(x, y, m, marker='.', linestyle="None")
# plot errorbars
for i in np.arange(0, len(x)):
ax.plot([x[i], x[i]], [y[i], y[i]], [m[i] + sig[i], m[i] - sig[i]],
marker="_",
color='k')
# get the observed points
task = task_group.tasks['main']
idata = task.valid_normalized_data_dict
xy = idata["inputs"]
xy = map(task_group.from_unit, xy)
xy = np.array(xy)
vals = idata["values"]
vals = [
obj_task.unstandardize_mean(obj_task.unstandardize_variance(v))
for v in vals
]
ax.plot(xy[:, 0], xy[:, 1], vals, marker='o', color="r", linestyle="None")
plt.show()