コード例 #1
0
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
    sys.stderr.write("\n")
    
    expt_file = os.path.join(expt_dir, options.config_file)
    expt      = load_expt(expt_file)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir,
                               expt.variable,
                               options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()    
    if best_job >= 0:
        sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
    else:
        sys.stderr.write("Current best: No results returned yet.\n")

    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()
    
    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending    = expt_grid.get_pending()
    complete   = expt_grid.get_complete()
    sys.stderr.write("%d candidates   %d pending   %d complete\n" % 
                     (candidates.shape[0], pending.shape[0], complete.shape[0]))
      
    # Verify that pending jobs are actually running.
    for job_id in pending:
        sgeid = expt_grid.get_sgeid(job_id)
        reset_job = False
        
        try:
            # Send an alive signal to proc (note this could kill it in windows)
            os.kill(sgeid, 0)
        except OSError:
            # Job is no longer running but still in the candidate list. Assume it crashed out.
            expt_grid.set_candidate(job_id)

    # Track the time series of optimization.
    trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
    trace_fh.write("%d,%f,%d,%d,%d,%d\n"
                   % (time.time(), best_val, best_job,
                      candidates.shape[0], pending.shape[0], complete.shape[0]))
    trace_fh.close()

    # Print out the best job results
    best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'w')
    best_job_fh.write("Best result: %f\nJob-id: %d\nParameters: \n" % 
                      (best_val, best_job))    
    for best_params in expt_grid.get_params(best_job):
        best_job_fh.write(str(best_params) + '\n')
    best_job_fh.close()

    if complete.shape[0] >= options.max_finished_jobs:
        sys.stderr.write("Maximum number of finished jobs (%d) reached."
                         "Exiting\n" % options.max_finished_jobs)
        sys.exit(0)

    if candidates.shape[0] == 0:
        sys.stderr.write("There are no candidates left.  Exiting.\n")
        sys.exit(0)

    if pending.shape[0] >= options.max_concurrent:
        sys.stderr.write("Maximum number of jobs (%d) pending.\n"
                         % (options.max_concurrent))
        return

    # Ask the chooser to actually pick one.
    job_id = chooser.next(grid, values, durations, candidates, pending,
                          complete)

    # If the job_id is a tuple, then the chooser picked a new job.
    # We have to add this to our grid
    if isinstance(job_id, tuple):
        (job_id, candidate) = job_id
        job_id = expt_grid.add_to_grid(candidate)

    sys.stderr.write("Selected job %d from the grid.\n" % (job_id))

    # Convert this back into an interpretable job and add metadata.
    job = Job()
    job.id        = job_id
    job.expt_dir  = expt_dir
    job.name      = expt.name
    job.language  = expt.language
    job.status    = 'submitted'
    job.submit_t  = int(time.time())
    job.param.extend(expt_grid.get_params(job_id))

    # Make sure we have a job subdirectory.
    job_subdir = os.path.join(expt_dir, 'jobs')
    if not os.path.exists(job_subdir):
        os.mkdir(job_subdir)

    # Name this job file.
    job_file = os.path.join(job_subdir,
                            '%08d.pb' % (job_id))

    # Store the job file.
    save_job(job_file, job)

    # Make sure there is a directory for output.
    output_subdir = os.path.join(expt_dir, 'output')
    if not os.path.exists(output_subdir):
        os.mkdir(output_subdir)
    output_file = os.path.join(output_subdir,
                               '%08d.out' % (job_id))

    process = job_submit("%s-%08d" % (expt_name, job_id),
                         output_file,
                         job_file, work_dir)
    process.poll()
    if process.returncode is not None and process.returncode < 0:
        sys.stderr.write("Failed to submit job or job crashed "
                         "with return code %d !\n" % process.returncode)
        sys.stderr.write("Deleting job file.\n")
        os.unlink(job_file)
        return
    else:
        sys.stderr.write("Submitted job as process: %d\n" % process.pid)

    # Now, update the experiment status to submitted.
    expt_grid.set_submitted(job_id, process.pid)

    return
コード例 #2
0
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
    import drmaa

    sys.stderr.write("\n")
    
    expt_file = os.path.join(expt_dir, options.config_file)
    expt      = load_expt(expt_file)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir,
                               expt.variable,
                               options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
 
    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()
    
    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending    = expt_grid.get_pending()
    complete   = expt_grid.get_complete()
    sys.stderr.write("%d candidates   %d pending   %d complete\n" % 
                     (candidates.shape[0], pending.shape[0], complete.shape[0]))

    # Verify that pending jobs are actually running.
    s = drmaa.Session()
    s.initialize()
    for job_id in pending:
        sgeid = expt_grid.get_sgeid(job_id)
        reset_job = False
        
        try:
            status = s.jobStatus(str(sgeid))
        except:
            sys.stderr.write("EXC: %s\n" % (str(sys.exc_info()[0])))
            sys.stderr.write("Could not find SGE id for job %d (%d)\n" % 
                             (job_id, sgeid))
            status = -1
            reset_job = True

        if status == drmaa.JobState.UNDETERMINED:
            sys.stderr.write("Job %d (%d) in undetermined state.\n" % 
                             (job_id, sgeid))
            reset_job = True

        elif status in [drmaa.JobState.QUEUED_ACTIVE, drmaa.JobState.RUNNING]:
            pass # Good shape.

        elif status in [drmaa.JobState.SYSTEM_ON_HOLD,
                        drmaa.JobState.USER_ON_HOLD,
                        drmaa.JobState.USER_SYSTEM_ON_HOLD,
                        drmaa.JobState.SYSTEM_SUSPENDED,
                        drmaa.JobState.USER_SUSPENDED]:
            sys.stderr.write("Job %d (%d) is held or suspended.\n" % 
                             (job_id, sgeid))
            reset_job = True

        elif status == drmaa.JobState.DONE:
            sys.stderr.write("Job %d (%d) complete but not yet updated.\n" % 
                             (job_id, sgeid))

        elif status == drmaa.JobState.FAILED:
            sys.stderr.write("Job %d (%d) failed.\n" % (job_id, sgeid))
            reset_job = True

        if reset_job:

            try:
                # Kill the job.
                s.control(str(sgeid), drmaa.JobControlAction.TERMINATE)
                sys.stderr.write("Killed SGE job %d.\n" % (sgeid))
            except:
                sys.stderr.write("Failed to kill SGE job %d.\n" % (sgeid))

            # Set back to being a candidate state.
            expt_grid.set_candidate(job_id)
            sys.stderr.write("Set job %d back to pending status.\n" % (job_id))

    s.exit()
      
    # Track the time series of optimization.
    trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
    trace_fh.write("%d,%f,%d,%d,%d,%d\n"
                   % (time.time(), best_val, best_job,
                      candidates.shape[0], pending.shape[0], complete.shape[0]))
    trace_fh.close()

    # Print out the best job results
    best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'a')
    best_job_fh.write("Best result: %f\n Job-id: %d\n Parameters: %s\n"
                      % (best_val, best_job, expt_grid.get_params(best_job)))
    best_job_fh.close()

    if complete.shape[0] >= options.max_finished_jobs:
        sys.stderr.write("Maximum number of finished jobs (%d) reached. "
                         "Exiting\n" % options.max_finished_jobs)
        sys.exit(0)

    if candidates.shape[0] == 0:
        sys.stderr.write("There are no candidates left.  Exiting.\n")
        sys.exit(0)

    if pending.shape[0] >= options.max_concurrent:
        sys.stderr.write("Maximum number of jobs (%d) pending.\n"
                         % (options.max_concurrent))
        return

    # Ask the chooser to actually pick one.
    job_id = chooser.next(grid, values, durations, candidates, pending,
                          complete)

    # If the job_id is a tuple, then the chooser picked a new job.
    # We have to add this to our grid
    if isinstance(job_id, tuple):
        (job_id, candidate) = job_id
        job_id = expt_grid.add_to_grid(candidate)

    sys.stderr.write("Selected job %d from the grid.\n" % (job_id))

    # Convert this back into an interpretable job and add metadata.
    job = Job()
    job.id        = job_id
    job.expt_dir  = expt_dir
    job.name      = expt.name
    job.language  = expt.language
    job.status    = 'submitted'
    job.submit_t  = int(time.time())
    job.param.extend(expt_grid.get_params(job_id))

    # Make sure we have a job subdirectory.
    job_subdir = os.path.join(expt_dir, 'jobs')
    if not os.path.exists(job_subdir):
        os.mkdir(job_subdir)

    # Name this job file.
    job_file = os.path.join(job_subdir,
                            '%08d.pb' % (job_id))

    # Store the job file.
    save_job(job_file, job)

    # Make sure there is a directory for output.
    output_subdir = os.path.join(expt_dir, 'output')
    if not os.path.exists(output_subdir):
        os.mkdir(output_subdir)
    output_file = os.path.join(output_subdir,
                               '%08d.out' % (job_id))

    queue_id, msg = sge_submit("%s-%08d" % (expt_name, job_id),
                             output_file,
                             DEFAULT_MODULES,
                             job_file, work_dir)
    if queue_id is None:
        sys.stderr.write("Failed to submit job: %s" % (msg))
        sys.stderr.write("Deleting job file.\n")
        os.unlink(job_file)
        return
    else:
        sys.stderr.write("Submitted as job %d\n" % (queue_id))

    # Now, update the experiment status to submitted.
    expt_grid.set_submitted(job_id, queue_id)

    return
コード例 #3
0
ファイル: spearmint.py プロジェクト: andreirusu/spearmint
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
    import drmaa

    sys.stderr.write("\n")
    
    expt_file = os.path.join(expt_dir, options.config_file)
    expt      = load_expt(expt_file)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir,
                               expt.variable,
                               options.grid_size,
                               options.grid_seed)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
 
    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()
    
    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending    = expt_grid.get_pending()
    complete   = expt_grid.get_complete()
    sys.stderr.write("%d candidates   %d pending   %d complete\n" % 
                     (candidates.shape[0], pending.shape[0], complete.shape[0]))

    # Verify that pending jobs are actually running.
    s = drmaa.Session()
    s.initialize()
    for job_id in pending:
        sgeid = expt_grid.get_sgeid(job_id)
        reset_job = False
        
        try:
            status = s.jobStatus(str(sgeid))
        except:
            sys.stderr.write("EXC: %s\n" % (str(sys.exc_info()[0])))
            sys.stderr.write("Could not find SGE id for job %d (%d)\n" % 
                             (job_id, sgeid))
            status = -1
            reset_job = True

        if status == drmaa.JobState.UNDETERMINED:
            sys.stderr.write("Job %d (%d) in undetermined state.\n" % 
                             (job_id, sgeid))
            reset_job = True

        elif status in [drmaa.JobState.QUEUED_ACTIVE, drmaa.JobState.RUNNING]:
            pass # Good shape.

        elif status in [drmaa.JobState.SYSTEM_ON_HOLD,
                        drmaa.JobState.USER_ON_HOLD,
                        drmaa.JobState.USER_SYSTEM_ON_HOLD,
                        drmaa.JobState.SYSTEM_SUSPENDED,
                        drmaa.JobState.USER_SUSPENDED]:
            sys.stderr.write("Job %d (%d) is held or suspended.\n" % 
                             (job_id, sgeid))
            reset_job = True

        elif status == drmaa.JobState.DONE:
            sys.stderr.write("Job %d (%d) complete but not yet updated.\n" % 
                             (job_id, sgeid))

        elif status == drmaa.JobState.FAILED:
            sys.stderr.write("Job %d (%d) failed.\n" % (job_id, sgeid))
            reset_job = True

        if reset_job:

            try:
                # Kill the job.
                s.control(str(sgeid), drmaa.JobControlAction.TERMINATE)
                sys.stderr.write("Killed SGE job %d.\n" % (sgeid))
            except:
                sys.stderr.write("Failed to kill SGE job %d.\n" % (sgeid))

            # Set back to being a candidate state.
            expt_grid.set_candidate(job_id)
            sys.stderr.write("Set job %d back to pending status.\n" % (job_id))

    s.exit()
      
    # Track the time series of optimization.
    trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
    trace_fh.write("%d,%f,%d,%d,%d,%d\n"
                   % (time.time(), best_val, best_job,
                      candidates.shape[0], pending.shape[0], complete.shape[0]))
    trace_fh.close()

    # Print out the best job results
    best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'a')
    best_job_fh.write("Best result: %f\n Job-id: %d\n Parameters: %s\n"
                      % (best_val, best_job, expt_grid.get_params(best_job)))
    best_job_fh.close()

    if complete.shape[0] >= options.max_finished_jobs:
        sys.stderr.write("Maximum number of finished jobs (%d) reached. "
                         "Exiting\n" % options.max_finished_jobs)
        sys.exit(0)

    if candidates.shape[0] == 0:
        sys.stderr.write("There are no candidates left.  Exiting.\n")
        sys.exit(0)

    if pending.shape[0] >= options.max_concurrent:
        sys.stderr.write("Maximum number of jobs (%d) pending.\n"
                         % (options.max_concurrent))
        return

    # Ask the chooser to actually pick one.
    job_id = chooser.next(grid, values, durations, candidates, pending,
                          complete)

    # If the job_id is a tuple, then the chooser picked a new job.
    # We have to add this to our grid
    if isinstance(job_id, tuple):
        (job_id, candidate) = job_id
        job_id = expt_grid.add_to_grid(candidate)

    sys.stderr.write("Selected job %d from the grid.\n" % (job_id))

    # Convert this back into an interpretable job and add metadata.
    job = Job()
    job.id        = job_id
    job.expt_dir  = expt_dir
    job.name      = expt.name
    job.language  = expt.language
    job.status    = 'submitted'
    job.submit_t  = int(time.time())
    job.param.extend(expt_grid.get_params(job_id))

    # Make sure we have a job subdirectory.
    job_subdir = os.path.join(expt_dir, 'jobs')
    if not os.path.exists(job_subdir):
        os.mkdir(job_subdir)

    # Name this job file.
    job_file = os.path.join(job_subdir,
                            '%08d.pb' % (job_id))

    # Store the job file.
    save_job(job_file, job)

    # Make sure there is a directory for output.
    output_subdir = os.path.join(expt_dir, 'output')
    if not os.path.exists(output_subdir):
        os.mkdir(output_subdir)
    output_file = os.path.join(output_subdir,
                               '%08d.out' % (job_id))

    queue_id, msg = sge_submit("%s-%08d" % (expt_name, job_id),
                             output_file,
                             DEFAULT_MODULES,
                             job_file, work_dir)
    if queue_id is None:
        sys.stderr.write("Failed to submit job: %s" % (msg))
        sys.stderr.write("Deleting job file.\n")
        os.unlink(job_file)
        return
    else:
        sys.stderr.write("Submitted as job %d\n" % (queue_id))

    # Now, update the experiment status to submitted.
    expt_grid.set_submitted(job_id, queue_id)

    return
コード例 #4
0
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
    sys.stderr.write("\n")

    expt_file = os.path.join(expt_dir, options.config_file)
    expt = load_expt(expt_file)

    # Build the experiment grid.
    expt_grid = ExperimentGrid(expt_dir,
                               expt.variable,
                               options.grid_size,
                               options.grid_seed,
                               locking=True)

    # Print out the current best function value.
    best_val, best_job = expt_grid.get_best()
    if best_job >= 0:
        sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
    else:
        sys.stderr.write("Current best: No results returned yet.\n")

    # Gets you everything - NaN for unknown values & durations.
    grid, values, durations = expt_grid.get_grid()

    # Returns lists of indices.
    candidates = expt_grid.get_candidates()
    pending = expt_grid.get_pending()
    complete = expt_grid.get_complete()
    sys.stderr.write(
        "%d candidates   %d pending   %d complete\n" %
        (candidates.shape[0], pending.shape[0], complete.shape[0]))

    # Verify that pending jobs are actually running.
    for job_id in pending:
        sgeid = expt_grid.get_sgeid(job_id)
        if not sgeid in psutil.pids():
            # Job is no longer running but still in the candidate list. Assume it crashed out.
            expt_grid.set_candidate(job_id)

    # Track the time series of optimization.
    trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
    trace_fh.write("%d,%f,%d,%d,%d,%d\n" %
                   (time.time(), best_val, best_job, candidates.shape[0],
                    pending.shape[0], complete.shape[0]))
    trace_fh.close()

    # Print out the best job results
    best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'w')
    best_job_fh.write("Best result: %f\nJob-id: %d\nParameters: \n" %
                      (best_val, best_job))
    for best_params in expt_grid.get_params(best_job):
        best_job_fh.write(str(best_params) + '\n')
    best_job_fh.close()

    if complete.shape[0] >= options.max_finished_jobs:
        sys.stderr.write("Maximum number of finished jobs (%d) reached.\n"
                         "Exiting\n" % options.max_finished_jobs)
        sys.exit(0)

    if candidates.shape[0] == 0 and pending.shape[0] > 0:
        sys.stderr.write(
            "There are no candidates left. Waiting for job completion.\n")
        return

    if candidates.shape[0] == 0 and pending.shape[0] == 0:
        sys.stderr.write("There are no candidates left. Exiting.\n")
        sys.exit(0)

    if pending.shape[0] >= options.max_concurrent:
        sys.stderr.write("Maximum number of jobs (%d) pending.\n" %
                         (options.max_concurrent))
        return

    # Dont submit if pending + finished > max_finished_jobs.
    if pending.shape[0] + complete.shape[0] >= options.max_finished_jobs:
        sys.stderr.write("Full number of jobs (%d) submitted. Waiting for "
                         "completion.\n" % (options.max_finished_jobs))
        return

    # Ask the chooser to actually pick one.
    job_id = chooser.next(grid, values, durations, candidates, pending,
                          complete)

    # If the job_id is a tuple, then the chooser picked a new job.
    # We have to add this to our grid
    if isinstance(job_id, tuple):
        (job_id, candidate) = job_id
        job_id = expt_grid.add_to_grid(candidate)

    sys.stderr.write("Selected job %d from the grid.\n" % (job_id))

    # Convert this back into an interpretable job and add metadata.
    job = Job()
    job.id = job_id
    job.expt_dir = expt_dir
    job.name = expt.name
    job.language = expt.language
    job.status = 'submitted'
    job.submit_t = int(time.time())
    job.param.extend(expt_grid.get_params(job_id))

    # Make sure we have a job subdirectory.
    job_subdir = os.path.join(expt_dir, 'jobs')
    if not os.path.exists(job_subdir):
        os.mkdir(job_subdir)

    # Name this job file.
    job_file = os.path.join(job_subdir, '%08d.pb' % (job_id))

    # Store the job file.
    save_job(job_file, job)

    # Make sure there is a directory for output.
    output_subdir = os.path.join(expt_dir, 'output')
    if not os.path.exists(output_subdir):
        os.mkdir(output_subdir)
    output_file = os.path.join(output_subdir, '%08d.out' % (job_id))

    process = job_submit("%s-%08d" % (expt_name, job_id), output_file,
                         job_file, work_dir)
    process.poll()
    if process.returncode is not None and process.returncode < 0:
        sys.stderr.write("Failed to submit job or job crashed "
                         "with return code %d !\n" % process.returncode)
        sys.stderr.write("Deleting job file.\n")
        os.unlink(job_file)
        return
    else:
        sys.stderr.write("Submitted job as process: %d\n" % process.pid)

    # Now, update the experiment status to submitted.
    expt_grid.set_submitted(job_id, process.pid)

    return