コード例 #1
0
def attempt_dispatch(expt_config, expt_dir, chooser, driver, options):
    log("\n" + "-" * 40)
    expt = load_experiment(expt_config)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    if best_job >= 0:
        log("Current best: %f (job %d)" % (best_val, best_job))
    else:
        log("Current best: No results returned yet.")

    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()

    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending = expt_grid.get_pending()
    complete = expt_grid.get_complete()

    n_candidates = candidates.shape[0]
    n_pending = pending.shape[0]
    n_complete = complete.shape[0]
    log("%d candidates   %d pending   %d complete" %
        (n_candidates, n_pending, n_complete))

    # Verify that pending jobs are actually running, and add them back to the
    # candidate set if they have crashed or gotten lost.
    for job_id in pending:
        proc_id = expt_grid.get_proc_id(job_id)
        if not driver.is_proc_alive(job_id, proc_id):
            log("Set job %d back to pending status." % (job_id))
            expt_grid.set_candidate(job_id)

    # Track the time series of optimization.
    write_trace(expt_dir, best_val, best_job, n_candidates, n_pending,
                n_complete)

    # Print out the best job results
    write_best_job(expt_dir, best_val, best_job, expt_grid)

    if n_complete >= options.max_finished_jobs:
        log("Maximum number of finished jobs (%d) reached."
            "Exiting" % options.max_finished_jobs)
        return False

    if n_candidates == 0:
        log("There are no candidates left.  Exiting.")
        return False

    if n_pending >= options.max_concurrent:
        log("Maximum number of jobs (%d) pending." % (options.max_concurrent))
        return True

    else:

        # start a bunch of candidate jobs if possible
        #to_start = min(options.max_concurrent - n_pending, n_candidates)
        #log("Trying to start %d jobs" % (to_start))
        #for i in xrange(to_start):

        # Ask the chooser to pick the next candidate
        log("Choosing next candidate... ")
        job_id = chooser.next(grid, values, durations, candidates, pending,
                              complete)

        # If the job_id is a tuple, then the chooser picked a new job.
        # We have to add this to our grid
        if isinstance(job_id, tuple):
            (job_id, candidate) = job_id
            job_id = expt_grid.add_to_grid(candidate)

        log("selected job %d from the grid." % (job_id))

        # Convert this back into an interpretable job and add metadata.
        job = Job()
        job.id = job_id
        job.expt_dir = expt_dir
        job.name = expt.name
        job.language = expt.language
        job.status = 'submitted'
        job.submit_t = int(time.time())
        job.param.extend(expt_grid.get_params(job_id))

        save_job(job)
        pid = driver.submit_job(job)
        if pid != None:
            log("submitted - pid = %d" % (pid))
            expt_grid.set_submitted(job_id, pid)
        else:
            log("Failed to submit job!")
            log("Deleting job file.")
            os.unlink(job_file_for(job))

    return True
コード例 #2
0
def attempt_dispatch(expt_config, expt_dir, chooser, driver, options):
    log("\n" + "-" * 40)
    expt = load_experiment(expt_config)
    print(options)
    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    if best_job >= 0:
        log("Current best: %f (job %d)" % (best_val, best_job))
    else:
        log("Current best: No results returned yet.")

    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()

    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending = expt_grid.get_pending()
    complete = expt_grid.get_complete()
    executed = expt_grid.get_executed()

    n_candidates = candidates.shape[0]
    n_pending = pending.shape[0]
    n_complete = complete.shape[0]
    n_executed = executed.shape[0]

    log("%d candidates   %d pending   %d complete   %d executed" %
        (n_candidates, n_pending, n_complete, n_executed))

    # Verify that pending jobs are actually running, and add them back to the
    # candidate set if they have crashed or gotten lost.
    for job_id in pending:
        proc_id = expt_grid.get_proc_id(job_id)
        if not driver.is_proc_alive(job_id, proc_id):
            log("Set job %d back to pending status." % (job_id))
            expt_grid.set_candidate(job_id)

    # Track the time series of optimization.
    write_trace(expt_dir, best_val, best_job, n_candidates, n_pending,
                n_complete, n_executed)

    # Print out the best job results
    write_best_job(expt_dir, best_val, best_job, expt_grid)

    if n_complete >= options.max_finished_jobs:
        log("Maximum number of finished jobs (%d) reached."
            "Exiting" % options.max_finished_jobs)
        return False

    if n_candidates == 0:
        log("There are no candidates left.  Exiting.")
        return False

    if n_pending >= options.max_concurrent:
        log("Maximum number of jobs (%d) pending." % (options.max_concurrent))
        return True

    else:

        # start a bunch of candidate jobs if possible
        #to_start = min(options.max_concurrent - n_pending, n_candidates)
        #log("Trying to start %d jobs" % (to_start))
        #for i in xrange(to_start):

        # Ask the chooser to pick the next candidate
        log("Choosing next candidate... ")
        job_id, ei = chooser.next(grid, values, durations, candidates, pending,
                                  complete)
        log("Expected improvement: %.6f" % ei)

        print ">>>>>>>", n_executed, ei
        if ei < config.EI and n_executed >= config.MIN_ACCEPTED_RUNS:
            config.strikes += 1
            if config.strikes > 0:
                return False
        else:
            config.strikes = 0

        # If the job_id is a tuple, then the chooser picked a new job.
        # We have to add this to our grid
        if isinstance(job_id, tuple):
            (job_id, candidate) = job_id
            job_id = expt_grid.add_to_grid(candidate)

        log("selected job %d from the grid." % (job_id))

        # Convert this back into an interpretable job and add metadata.
        job = Job()
        job.id = job_id
        job.expt_dir = expt_dir
        job.name = expt.name
        job.language = expt.language
        job.status = 'submitted'
        job.submit_t = int(time.time())
        job.param.extend(expt_grid.get_params(job_id))

        #TODO: (@omid) check if the job has been previously completed; if so
        #      mark the job as completed and use the cached value
        params = job_params(job)
        for key, val in params.items():
            if isinstance(val, np.ndarray):
                val = val.tolist()
            if isinstance(val, list):
                val = frozenset(val)
            params[key] = val
        params = frozenset(params.items())
        if params in jobs_executed:
            jid = jobs_executed[params]
            print ">>>> Bypassing job execution."
            for stat in ['status', 'values', 'durs']:
                dic = getattr(expt_grid, stat)
                dic[job_id] = dic[jid]
            expt_grid._save_jobs()
            return True
        jobs_executed[params] = job_id

        save_job(job)
        pid = driver.submit_job(job)
        if pid != None:
            log("submitted - pid = %d" % (pid))
            expt_grid.set_submitted(job_id, pid)
        else:
            log("Failed to submit job!")
            log("Deleting job file.")
            os.unlink(job_file_for(job))

    return True
コード例 #3
0
ファイル: main.py プロジェクト: aurora1625/spearmint
def attempt_dispatch(expt_config, expt_dir, chooser, driver, options):
    log("\n" + "-" * 40)
    expt = load_experiment(expt_config)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir,
                               expt.variable,
                               options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    if best_job >= 0:
        log("Current best: %f (job %d)" % (best_val, best_job))
    else:
        log("Current best: No results returned yet.")

    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()

    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending    = expt_grid.get_pending()
    complete   = expt_grid.get_complete()

    n_candidates = candidates.shape[0]
    n_pending    = pending.shape[0]
    n_complete   = complete.shape[0]
    log("%d candidates   %d pending   %d complete" %
        (n_candidates, n_pending, n_complete))

    # Verify that pending jobs are actually running, and add them back to the
    # candidate set if they have crashed or gotten lost.
    for job_id in pending:
        proc_id = expt_grid.get_proc_id(job_id)
        if not driver.is_proc_alive(job_id, proc_id):
            log("Set job %d back to pending status." % (job_id))
            expt_grid.set_candidate(job_id)

    # Track the time series of optimization.
    write_trace(expt_dir, best_val, best_job, n_candidates, n_pending, n_complete)

    # Print out the best job results
    write_best_job(expt_dir, best_val, best_job, expt_grid)

    if n_complete >= options.max_finished_jobs:
        log("Maximum number of finished jobs (%d) reached."
                         "Exiting" % options.max_finished_jobs)
        return False

    if n_candidates == 0:
        log("There are no candidates left.  Exiting.")
        return False

    if n_pending >= options.max_concurrent:
        log("Maximum number of jobs (%d) pending." % (options.max_concurrent))
        return True

    else:

        # start a bunch of candidate jobs if possible
        #to_start = min(options.max_concurrent - n_pending, n_candidates)
        #log("Trying to start %d jobs" % (to_start))
        #for i in xrange(to_start):

        # Ask the chooser to pick the next candidate
        log("Choosing next candidate... ")
        job_id = chooser.next(grid, values, durations, candidates, pending, complete)

        # If the job_id is a tuple, then the chooser picked a new job.
        # We have to add this to our grid
        if isinstance(job_id, tuple):
            (job_id, candidate) = job_id
            job_id = expt_grid.add_to_grid(candidate)

        log("selected job %d from the grid." % (job_id))

        # Convert this back into an interpretable job and add metadata.
        job = Job()
        job.id        = job_id
        job.expt_dir  = expt_dir
        job.name      = expt.name
        job.language  = expt.language
        job.status    = 'submitted'
        job.submit_t  = int(time.time())
        job.param.extend(expt_grid.get_params(job_id))

        save_job(job)
        pid = driver.submit_job(job)
        if pid != None:
            log("submitted - pid = %d" % (pid))
            expt_grid.set_submitted(job_id, pid)
        else:
            log("Failed to submit job!")
            log("Deleting job file.")
            os.unlink(job_file_for(job))

    return True
コード例 #4
0
def attempt_yaml_to_hyper(expt_config, expt_dir,  driver, options, count, hyperList, yamlsDirectoryPath, number,mode):



    log("\n" + "-" * 40)
    expt = load_experiment(expt_config)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir,
                               expt.variable,
                               options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    if best_job >= 0:
        log("Current best: %f (job %d)" % (best_val, best_job))
    else:
        log("Current best: No results returned yet.")


    # Returns lists of indices.
    pending    = expt_grid.get_pending()
    complete   = expt_grid.get_complete()


    n_pending    = pending.shape[0]
    n_complete   = complete.shape[0]

    log("  %d pending   %d complete" %
        ( n_pending, n_complete))

    if n_complete < count:
        print("  %d count   %d complete" %( count, n_complete))
        global Flag
        Flag = False
        return True

    else:
        global Flag
        Flag = True
    # Verify that pending jobs are actually running, and add them back to the
    # candidate set if they have crashed or gotten lost.
    for job_id in pending:
        proc_id = expt_grid.get_proc_id(job_id)
        if not driver.is_proc_alive(job_id, proc_id):
            log("Set job %d back to pending status." % (job_id))
            expt_grid.set_candidate(job_id)


    # Print out the best job results
    write_best_job(expt_dir, best_val, best_job, expt_grid)



    if n_complete >= number:
        log("All yaml files are read and run in full data set."
                         "Exiting" % options.max_finished_jobs)
        return False

    else:

        # choose the job-id to be the next job that read from hyper_yaml files
        job_id = count

        log("selected job %d from the grid." % (job_id))

        # Convert this back into an interpretable job and add metadata.
        job = Job()
        job.id        = job_id
        job.expt_dir  = expt_dir
        job.name      = expt.name
        job.language  = expt.language
        job.status    = 'submitted'
        job.submit_t  = int(time.time())
        # todo: read the line after the learn-rate in yamls file and reconding to the hyperopt.yaml file
        params = []

        travelFile = listdir( yamlsDirectoryPath)[ count]
        filePath = join(yamlsDirectoryPath, travelFile)

        if mode== "get_hyper_from_output":
            if isfile( filePath):
                
                print("the path is " + travelFile)
                

                with open(filePath, 'r') as fin:
                    for line in fin:
                        if 'spear_wrapper params are:' in line:
                            for i in xrange(len(hyperList)):
                                hyper_param = hyperList[i]
                                hyper_split = hyper_param.split()
                                if hyper_split[1] in line:
                                    dic = ((line.split(hyper_split[1])[1]).split(": array([")[1]).split("])")[0];
                                    param = Parameter()
                                
                                    param.name = hyper_split[1]
                                
                                    type = hyper_split[2]
                                    if type == "FLOAT":
                                        param.dbl_val.append (float(dic))
                                    elif type == "INT":
                                        param.int_val.append (int(dic))
                                    elif type == "ENUM":
                                        param.str_val.append (str(dic))
                                    else:
                                        param.str_val.append (str(dic))
                            
                                    params.append(param)
                            break


                param = Parameter()
                param.name = "experiment_dir"
                param.str_val.append (expt_dir)
                params.append(param)

                param = Parameter()
                param.name = "pre_output_file_name"
                param.str_val.append (travelFile)
                params.append(param)

                job.param.extend(params)
            else :
                job.param.extend(params)
        else:
            if isfile( filePath):

                print("the path is " + travelFile)

                i = 0
                with open(filePath, 'r') as fin:
                    for line in fin:
                        if(i< hyperList.__len__()):
                            hyper_param = hyperList[i]
                            hyper_split = hyper_param.split()
                            if hyper_split[0] in line:
                                dic = [k.strip(",") for k in line.split()][1]
                                param = Parameter()

                                param.name = hyper_split[1]

                                type = hyper_split[2]
                                if type == "FLOAT":
                                    param.dbl_val.append (float(dic))
                                elif type == "INT":
                                    param.int_val.append (int(dic))
                                elif type == "ENUM":
                                    param.str_val.append (str(dic))
                                else:
                                    param.str_val.append (str(dic))

                                params.append(param)

                            i = i+1


                param = Parameter()
                param.name = "experiment_dir"
                param.str_val.append (expt_dir)
                params.append(param)

                job.param.extend(params)
            else :
                job.param.extend(params)

        save_job(job)
        pid = driver.submit_job(job)
        if pid != None:
            log("submitted - pid = %d" % (pid))
            expt_grid.set_submitted(job_id, pid)
        else:
            log("Failed to submit job!")
            log("Deleting job file.")
            os.unlink(job_file_for(job))

    return True