示例#1
0
def main(filter=None):
    """
    Usage: python make_plots.py PATH_TO_DIRECTORY
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--clean', action='store_true', help='remove broken jobs')
    parser.add_argument('--table', action='store_true', help='print table')
    parser.add_argument('--csv', action='store_true', help='save table as csv')
    parser.add_argument('--d', type=int, help='sort by distance from dth smallest result')
    parser.add_argument('--name', help='experiment name', default=None)
    args, unknown = parser.parse_known_args()

    options, expt_dir = get_options(unknown)
    # print "options:"
    # print_dict(options)

    # reduce the grid size
    options["grid_size"] = 400

    resources = parse_resources_from_config(options)

    # Load up the chooser.
    chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
    chooser = chooser_module.init(options)
    # print "chooser", chooser
    if args.name:
        experiment_name = args.name
    else:
        experiment_name     = options.get("experiment-name", 'unnamed-experiment')

    # Connect to the database
    db_address = options['database']['address']
    # sys.stderr.write('Using database at %s.\n' % db_address)
    db         = MongoDB(database_address=db_address)

    # testing below here
    jobs = load_jobs(db, experiment_name)
    print len(jobs), 'jobs found'
    # print jobs

    # remove_broken_jobs
    if args.clean:
        for job in jobs:
            if job['status'] == 'pending':
                sys.stderr.write('Broken job %s detected.\n' % job['id'])
                job['status'] = 'broken'
                db.save(job, experiment_name, 'jobs', {'id' : job['id']})

    # print "resources:", resources
    # print_dict(resources)
    resource = resources.itervalues().next()

    task_options = {task: options["tasks"][task] for task in resource.tasks}
    # print "task_options:"
    # print_dict(task_options) # {'main': {'likelihood': u'NOISELESS', 'type': 'OBJECTIVE'}}

    task_group = load_task_group(db, options, experiment_name, resource.tasks)
    hypers = load_hypers(db, experiment_name)
    chooser.fit(task_group, hypers, task_options)
    lp, x = chooser.best()

    if args.table:
        os.chdir(unknown[0])
        out_file = open('results.csv', 'w') if args.csv else sys.stdout

        # get the observed points
        task = task_group.tasks.itervalues().next()
        idata = task.valid_normalized_data_dict
        inputs = idata["inputs"]
        inputs = map(lambda i: [paramify(task_group, task_group.from_unit(i)).values(), i], inputs)
        vals = idata["values"]
        vals = [task.unstandardize_mean(task.unstandardize_variance(v)) for v in vals]

        out_file.write('\n%10s' % 'result')
        lengths = [10]
        for name, vdict in task.variables_meta.iteritems():
            name = '%10s' % name
            out_file.write(',' + name)
            lengths.append(len(name))
        out_file.write('\n')

        line_template = '%' + str(lengths[0]) + '.4f,' + ','.join(['%' + str(l) +
            ('.4f' if 'enum' not in inputs[0][0][i]['type'] else 's') for i, l in enumerate(lengths[1:])])

        points = sorted(zip(vals, inputs), key=lambda r: r[0])
        if args.d is not None:
            target = x
            if args.d >= 0:
                target = points[args.d][1][1]
            points = sorted(points, key=lambda r: np.linalg.norm(r[1][1] - target))
        for i, point in enumerate(points):
            subs = [point[0]] + [d['values'][0] for d in point[1][0]]
            out_file.write(line_template % tuple(subs) + '\n')
        out_file.close()
示例#2
0
def main():
    """
    Usage: python make_plots.py PATH_TO_DIRECTORY

    TODO: Some aspects of this function are specific to the simple branin example
    We should clean this up so that interpretation of plots are more clear and
    so that it works in more general cases 
    (e.g. if objective likelihood is binomial then values should not be
    unstandardized)
    """
    options, expt_dir = get_options()
    print "options:"
    print_dict(options)
    
    # reduce the grid size
    options["grid_size"] = 400

    resources = parse_resources_from_config(options)

    # Load up the chooser.
    chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
    chooser = chooser_module.init(options)
    print "chooser", chooser
    experiment_name     = options.get("experiment-name", 'unnamed-experiment')

    # Connect to the database
    db_address = options['database']['address']
    sys.stderr.write('Using database at %s.\n' % db_address)        
    db         = MongoDB(database_address=db_address)
    
    # testing below here
    jobs = load_jobs(db, experiment_name)
    remove_broken_jobs(db, jobs, experiment_name, resources)

    print "resources:", resources
    print_dict(resources)
    resource = resources.itervalues().next()
    
    task_options = { task: options["tasks"][task] for task in resource.tasks }
    print "task_options:"
    print_dict(task_options) # {'main': {'likelihood': u'NOISELESS', 'type': 'OBJECTIVE'}}
    
    task_group = load_task_group(db, options, resource.tasks)
    print "task_group", task_group # TaskGroup
    print "tasks:"
    print_dict(task_group.tasks) # {'main': <spearmint.tasks.task.Task object at 0x10bf63290>}
    
    
    hypers = load_hypers(db, experiment_name)
    print "loaded hypers", hypers # from GP.to_dict()
    
    hypers = chooser.fit(task_group, hypers, task_options)
    print "\nfitted hypers:"
    print_dict(hypers)

    lp, x = chooser.best()
    x = x.flatten()
    print "best", lp, x
    bestp = task_group.paramify(task_group.from_unit(x))
    print "expected best position", bestp
    
    # get the grid of points
    grid = chooser.grid
#     print "chooser objectives:", 
#     print_dict(chooser.objective)
    print "chooser models:", chooser.models
    print_dict(chooser.models)
    obj_model = chooser.models[chooser.objective['name']]
    obj_mean, obj_var = obj_model.function_over_hypers(obj_model.predict, grid)

    # un-normalize the function values and variances
    obj_task = task_group.tasks['main']
    obj_mean = [obj_task.unstandardize_mean(obj_task.unstandardize_variance(v)) for v in obj_mean]
    obj_std = [obj_task.unstandardize_variance(np.sqrt(v)) for v in obj_var]

    
#     for xy, m, v in izip(grid, obj_mean, obj_var):
#         print xy, m, v

    grid = map(task_group.from_unit, grid)
#     return
    
    xymv = [(xy[0], xy[1], m, v) for xy, m, v in izip(grid, obj_mean, obj_std)]# if .2 < xy[0] < .25] 
    
    x = map(lambda x:x[0], xymv)
    y = map(lambda x:x[1], xymv)
    m = map(lambda x:x[2], xymv)
    sig = map(lambda x:x[3], xymv)
#     print y
    
    fig = plt.figure(dpi=100)
    ax = fig.add_subplot(111, projection='3d')

    ax.plot(x, y, m, marker='.', linestyle="None")

    # plot errorbars
    for i in np.arange(0, len(x)):
        ax.plot([x[i], x[i]], [y[i], y[i]], [m[i]+sig[i], m[i]-sig[i]], marker="_", color='k')

    # get the observed points
    task = task_group.tasks['main']
    idata = task.valid_normalized_data_dict
    xy = idata["inputs"]
    xy = map(task_group.from_unit, xy)
    xy = np.array(xy)
    vals = idata["values"]
    vals = [obj_task.unstandardize_mean(obj_task.unstandardize_variance(v)) for v in vals]

    ax.plot(xy[:,0], xy[:,1], vals, marker='o', color="r", linestyle="None")
    
    plt.show()
示例#3
0
def main():
    """
    Usage: python make_plots.py PATH_TO_DIRECTORY

    TODO: Some aspects of this function are specific to the simple branin example
    We should clean this up so that interpretation of plots are more clear and
    so that it works in more general cases 
    (e.g. if objective likelihood is binomial then values should not be
    unstandardized)
    """
    options, expt_dir = get_options()
    print("options:")
    print_dict(options)

    # reduce the grid size
    options["grid_size"] = 400

    resources = parse_resources_from_config(options)

    # Load up the chooser.
    chooser_module = importlib.import_module('spearmint.choosers.' +
                                             options['chooser'])
    chooser = chooser_module.init(options)
    print("chooser", chooser)
    experiment_name = options.get("experiment-name", 'unnamed-experiment')

    # Connect to the database
    db_address = options['database']['address']
    sys.stderr.write('Using database at %s.\n' % db_address)
    db = MongoDB(database_address=db_address)

    # testing below here
    jobs = load_jobs(db, experiment_name)
    remove_broken_jobs(db, jobs, experiment_name, resources)

    print("resources:", resources)
    print_dict(resources)
    resource = resources.itervalues().next()

    task_options = {task: options["tasks"][task] for task in resource.tasks}
    print("task_options:")
    print_dict(task_options
               )  # {'main': {'likelihood': u'NOISELESS', 'type': 'OBJECTIVE'}}

    task_group = load_task_group(db, options, resource.tasks)
    print("task_group", task_group)  # TaskGroup
    print("tasks:")
    print_dict(task_group.tasks
               )  # {'main': <spearmint.tasks.task.Task object at 0x10bf63290>}

    hypers = load_hypers(db, experiment_name)
    print("loaded hypers", hypers)  # from GP.to_dict()

    hypers = chooser.fit(task_group, hypers, task_options)
    print("\nfitted hypers:")
    print_dict(hypers)

    lp, x = chooser.best()
    x = x.flatten()
    print("best", lp, x)
    bestp = task_group.paramify(task_group.from_unit(x))
    print("expected best position", bestp)

    # get the grid of points
    grid = chooser.grid
    #     print("chooser objectives:", )
    #     print_dict(chooser.objective)
    print("chooser models:", chooser.models)
    print_dict(chooser.models)
    obj_model = chooser.models[chooser.objective['name']]
    obj_mean, obj_var = obj_model.function_over_hypers(obj_model.predict, grid)

    # un-normalize the function values and variances
    obj_task = task_group.tasks['main']
    obj_mean = [
        obj_task.unstandardize_mean(obj_task.unstandardize_variance(v))
        for v in obj_mean
    ]
    obj_std = [obj_task.unstandardize_variance(np.sqrt(v)) for v in obj_var]

    #     for xy, m, v in izip(grid, obj_mean, obj_var):
    #         print(xy, m, v)

    grid = map(task_group.from_unit, grid)
    #     return

    xymv = [(xy[0], xy[1], m, v) for xy, m, v in izip(grid, obj_mean, obj_std)
            ]  # if .2 < xy[0] < .25]

    x = map(lambda x: x[0], xymv)
    y = map(lambda x: x[1], xymv)
    m = map(lambda x: x[2], xymv)
    sig = map(lambda x: x[3], xymv)
    #     print(y)

    fig = plt.figure(dpi=100)
    ax = fig.add_subplot(111, projection='3d')

    ax.plot(x, y, m, marker='.', linestyle="None")

    # plot errorbars
    for i in np.arange(0, len(x)):
        ax.plot([x[i], x[i]], [y[i], y[i]], [m[i] + sig[i], m[i] - sig[i]],
                marker="_",
                color='k')

    # get the observed points
    task = task_group.tasks['main']
    idata = task.valid_normalized_data_dict
    xy = idata["inputs"]
    xy = map(task_group.from_unit, xy)
    xy = np.array(xy)
    vals = idata["values"]
    vals = [
        obj_task.unstandardize_mean(obj_task.unstandardize_variance(v))
        for v in vals
    ]

    ax.plot(xy[:, 0], xy[:, 1], vals, marker='o', color="r", linestyle="None")

    plt.show()