def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
sys.stderr.write("\n")
expt_file = os.path.join(expt_dir, options.config_file)
expt = load_expt(expt_file)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir,
expt.variable,
options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
if best_job >= 0:
sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
else:
sys.stderr.write("Current best: No results returned yet.\n")
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
sys.stderr.write("%d candidates %d pending %d complete\n" %
(candidates.shape[0], pending.shape[0], complete.shape[0]))
# Verify that pending jobs are actually running.
for job_id in pending:
sgeid = expt_grid.get_sgeid(job_id)
reset_job = False
try:
# Send an alive signal to proc (note this could kill it in windows)
os.kill(sgeid, 0)
except OSError:
# Job is no longer running but still in the candidate list. Assume it crashed out.
expt_grid.set_candidate(job_id)
# Track the time series of optimization.
trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
trace_fh.write("%d,%f,%d,%d,%d,%d\n"
% (time.time(), best_val, best_job,
candidates.shape[0], pending.shape[0], complete.shape[0]))
trace_fh.close()
# Print out the best job results
best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'w')
best_job_fh.write("Best result: %f\nJob-id: %d\nParameters: \n" %
(best_val, best_job))
for best_params in expt_grid.get_params(best_job):
best_job_fh.write(str(best_params) + '\n')
best_job_fh.close()
if complete.shape[0] >= options.max_finished_jobs:
sys.stderr.write("Maximum number of finished jobs (%d) reached."
"Exiting\n" % options.max_finished_jobs)
sys.exit(0)
if candidates.shape[0] == 0:
sys.stderr.write("There are no candidates left. Exiting.\n")
sys.exit(0)
if pending.shape[0] >= options.max_concurrent:
sys.stderr.write("Maximum number of jobs (%d) pending.\n"
% (options.max_concurrent))
return
# Ask the chooser to actually pick one.
job_id = chooser.next(grid, values, durations, candidates, pending,
complete)
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
sys.stderr.write("Selected job %d from the grid.\n" % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
# Make sure we have a job subdirectory.
job_subdir = os.path.join(expt_dir, 'jobs')
if not os.path.exists(job_subdir):
os.mkdir(job_subdir)
# Name this job file.
job_file = os.path.join(job_subdir,
'%08d.pb' % (job_id))
# Store the job file.
save_job(job_file, job)
# Make sure there is a directory for output.
output_subdir = os.path.join(expt_dir, 'output')
if not os.path.exists(output_subdir):
os.mkdir(output_subdir)
output_file = os.path.join(output_subdir,
'%08d.out' % (job_id))
process = job_submit("%s-%08d" % (expt_name, job_id),
output_file,
job_file, work_dir)
process.poll()
if process.returncode is not None and process.returncode < 0:
sys.stderr.write("Failed to submit job or job crashed "
"with return code %d !\n" % process.returncode)
sys.stderr.write("Deleting job file.\n")
os.unlink(job_file)
return
else:
sys.stderr.write("Submitted job as process: %d\n" % process.pid)
# Now, update the experiment status to submitted.
expt_grid.set_submitted(job_id, process.pid)
return
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
import drmaa
sys.stderr.write("\n")
expt_file = os.path.join(expt_dir, options.config_file)
expt = load_expt(expt_file)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir,
expt.variable,
options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
sys.stderr.write("%d candidates %d pending %d complete\n" %
(candidates.shape[0], pending.shape[0], complete.shape[0]))
# Verify that pending jobs are actually running.
s = drmaa.Session()
s.initialize()
for job_id in pending:
sgeid = expt_grid.get_sgeid(job_id)
reset_job = False
try:
status = s.jobStatus(str(sgeid))
except:
sys.stderr.write("EXC: %s\n" % (str(sys.exc_info()[0])))
sys.stderr.write("Could not find SGE id for job %d (%d)\n" %
(job_id, sgeid))
status = -1
reset_job = True
if status == drmaa.JobState.UNDETERMINED:
sys.stderr.write("Job %d (%d) in undetermined state.\n" %
(job_id, sgeid))
reset_job = True
elif status in [drmaa.JobState.QUEUED_ACTIVE, drmaa.JobState.RUNNING]:
pass # Good shape.
elif status in [drmaa.JobState.SYSTEM_ON_HOLD,
drmaa.JobState.USER_ON_HOLD,
drmaa.JobState.USER_SYSTEM_ON_HOLD,
drmaa.JobState.SYSTEM_SUSPENDED,
drmaa.JobState.USER_SUSPENDED]:
sys.stderr.write("Job %d (%d) is held or suspended.\n" %
(job_id, sgeid))
reset_job = True
elif status == drmaa.JobState.DONE:
sys.stderr.write("Job %d (%d) complete but not yet updated.\n" %
(job_id, sgeid))
elif status == drmaa.JobState.FAILED:
sys.stderr.write("Job %d (%d) failed.\n" % (job_id, sgeid))
reset_job = True
if reset_job:
try:
# Kill the job.
s.control(str(sgeid), drmaa.JobControlAction.TERMINATE)
sys.stderr.write("Killed SGE job %d.\n" % (sgeid))
except:
sys.stderr.write("Failed to kill SGE job %d.\n" % (sgeid))
# Set back to being a candidate state.
expt_grid.set_candidate(job_id)
sys.stderr.write("Set job %d back to pending status.\n" % (job_id))
s.exit()
# Track the time series of optimization.
trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
trace_fh.write("%d,%f,%d,%d,%d,%d\n"
% (time.time(), best_val, best_job,
candidates.shape[0], pending.shape[0], complete.shape[0]))
trace_fh.close()
# Print out the best job results
best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'a')
best_job_fh.write("Best result: %f\n Job-id: %d\n Parameters: %s\n"
% (best_val, best_job, expt_grid.get_params(best_job)))
best_job_fh.close()
if complete.shape[0] >= options.max_finished_jobs:
sys.stderr.write("Maximum number of finished jobs (%d) reached. "
"Exiting\n" % options.max_finished_jobs)
sys.exit(0)
if candidates.shape[0] == 0:
sys.stderr.write("There are no candidates left. Exiting.\n")
sys.exit(0)
if pending.shape[0] >= options.max_concurrent:
sys.stderr.write("Maximum number of jobs (%d) pending.\n"
% (options.max_concurrent))
return
# Ask the chooser to actually pick one.
job_id = chooser.next(grid, values, durations, candidates, pending,
complete)
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
sys.stderr.write("Selected job %d from the grid.\n" % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
# Make sure we have a job subdirectory.
job_subdir = os.path.join(expt_dir, 'jobs')
if not os.path.exists(job_subdir):
os.mkdir(job_subdir)
# Name this job file.
job_file = os.path.join(job_subdir,
'%08d.pb' % (job_id))
# Store the job file.
save_job(job_file, job)
# Make sure there is a directory for output.
output_subdir = os.path.join(expt_dir, 'output')
if not os.path.exists(output_subdir):
os.mkdir(output_subdir)
output_file = os.path.join(output_subdir,
'%08d.out' % (job_id))
queue_id, msg = sge_submit("%s-%08d" % (expt_name, job_id),
output_file,
DEFAULT_MODULES,
job_file, work_dir)
if queue_id is None:
sys.stderr.write("Failed to submit job: %s" % (msg))
sys.stderr.write("Deleting job file.\n")
os.unlink(job_file)
return
else:
sys.stderr.write("Submitted as job %d\n" % (queue_id))
# Now, update the experiment status to submitted.
expt_grid.set_submitted(job_id, queue_id)
return
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
import drmaa
sys.stderr.write("\n")
expt_file = os.path.join(expt_dir, options.config_file)
expt = load_expt(expt_file)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir,
expt.variable,
options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
sys.stderr.write("%d candidates %d pending %d complete\n" %
(candidates.shape[0], pending.shape[0], complete.shape[0]))
# Verify that pending jobs are actually running.
s = drmaa.Session()
s.initialize()
for job_id in pending:
sgeid = expt_grid.get_sgeid(job_id)
reset_job = False
try:
status = s.jobStatus(str(sgeid))
except:
sys.stderr.write("EXC: %s\n" % (str(sys.exc_info()[0])))
sys.stderr.write("Could not find SGE id for job %d (%d)\n" %
(job_id, sgeid))
status = -1
reset_job = True
if status == drmaa.JobState.UNDETERMINED:
sys.stderr.write("Job %d (%d) in undetermined state.\n" %
(job_id, sgeid))
reset_job = True
elif status in [drmaa.JobState.QUEUED_ACTIVE, drmaa.JobState.RUNNING]:
pass # Good shape.
elif status in [drmaa.JobState.SYSTEM_ON_HOLD,
drmaa.JobState.USER_ON_HOLD,
drmaa.JobState.USER_SYSTEM_ON_HOLD,
drmaa.JobState.SYSTEM_SUSPENDED,
drmaa.JobState.USER_SUSPENDED]:
sys.stderr.write("Job %d (%d) is held or suspended.\n" %
(job_id, sgeid))
reset_job = True
elif status == drmaa.JobState.DONE:
sys.stderr.write("Job %d (%d) complete but not yet updated.\n" %
(job_id, sgeid))
elif status == drmaa.JobState.FAILED:
sys.stderr.write("Job %d (%d) failed.\n" % (job_id, sgeid))
reset_job = True
if reset_job:
try:
# Kill the job.
s.control(str(sgeid), drmaa.JobControlAction.TERMINATE)
sys.stderr.write("Killed SGE job %d.\n" % (sgeid))
except:
sys.stderr.write("Failed to kill SGE job %d.\n" % (sgeid))
# Set back to being a candidate state.
expt_grid.set_candidate(job_id)
sys.stderr.write("Set job %d back to pending status.\n" % (job_id))
s.exit()
# Track the time series of optimization.
trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
trace_fh.write("%d,%f,%d,%d,%d,%d\n"
% (time.time(), best_val, best_job,
candidates.shape[0], pending.shape[0], complete.shape[0]))
trace_fh.close()
# Print out the best job results
best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'a')
best_job_fh.write("Best result: %f\n Job-id: %d\n Parameters: %s\n"
% (best_val, best_job, expt_grid.get_params(best_job)))
best_job_fh.close()
if complete.shape[0] >= options.max_finished_jobs:
sys.stderr.write("Maximum number of finished jobs (%d) reached. "
"Exiting\n" % options.max_finished_jobs)
sys.exit(0)
if candidates.shape[0] == 0:
sys.stderr.write("There are no candidates left. Exiting.\n")
sys.exit(0)
if pending.shape[0] >= options.max_concurrent:
sys.stderr.write("Maximum number of jobs (%d) pending.\n"
% (options.max_concurrent))
return
# Ask the chooser to actually pick one.
job_id = chooser.next(grid, values, durations, candidates, pending,
complete)
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
sys.stderr.write("Selected job %d from the grid.\n" % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
# Make sure we have a job subdirectory.
job_subdir = os.path.join(expt_dir, 'jobs')
if not os.path.exists(job_subdir):
os.mkdir(job_subdir)
# Name this job file.
job_file = os.path.join(job_subdir,
'%08d.pb' % (job_id))
# Store the job file.
save_job(job_file, job)
# Make sure there is a directory for output.
output_subdir = os.path.join(expt_dir, 'output')
if not os.path.exists(output_subdir):
os.mkdir(output_subdir)
output_file = os.path.join(output_subdir,
'%08d.out' % (job_id))
queue_id, msg = sge_submit("%s-%08d" % (expt_name, job_id),
output_file,
DEFAULT_MODULES,
job_file, work_dir)
if queue_id is None:
sys.stderr.write("Failed to submit job: %s" % (msg))
sys.stderr.write("Deleting job file.\n")
os.unlink(job_file)
return
else:
sys.stderr.write("Submitted as job %d\n" % (queue_id))
# Now, update the experiment status to submitted.
expt_grid.set_submitted(job_id, queue_id)
return
def attempt_dispatch(expt_name, expt_dir, work_dir, chooser, options):
sys.stderr.write("\n")
expt_file = os.path.join(expt_dir, options.config_file)
expt = load_expt(expt_file)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir,
expt.variable,
options.grid_size,
options.grid_seed,
locking=True)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
if best_job >= 0:
sys.stderr.write("Current best: %f (job %d)\n" % (best_val, best_job))
else:
sys.stderr.write("Current best: No results returned yet.\n")
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
sys.stderr.write(
"%d candidates %d pending %d complete\n" %
(candidates.shape[0], pending.shape[0], complete.shape[0]))
# Verify that pending jobs are actually running.
for job_id in pending:
sgeid = expt_grid.get_sgeid(job_id)
if not sgeid in psutil.pids():
# Job is no longer running but still in the candidate list. Assume it crashed out.
expt_grid.set_candidate(job_id)
# Track the time series of optimization.
trace_fh = open(os.path.join(expt_dir, 'trace.csv'), 'a')
trace_fh.write("%d,%f,%d,%d,%d,%d\n" %
(time.time(), best_val, best_job, candidates.shape[0],
pending.shape[0], complete.shape[0]))
trace_fh.close()
# Print out the best job results
best_job_fh = open(os.path.join(expt_dir, 'best_job_and_result.txt'), 'w')
best_job_fh.write("Best result: %f\nJob-id: %d\nParameters: \n" %
(best_val, best_job))
for best_params in expt_grid.get_params(best_job):
best_job_fh.write(str(best_params) + '\n')
best_job_fh.close()
if complete.shape[0] >= options.max_finished_jobs:
sys.stderr.write("Maximum number of finished jobs (%d) reached.\n"
"Exiting\n" % options.max_finished_jobs)
sys.exit(0)
if candidates.shape[0] == 0 and pending.shape[0] > 0:
sys.stderr.write(
"There are no candidates left. Waiting for job completion.\n")
return
if candidates.shape[0] == 0 and pending.shape[0] == 0:
sys.stderr.write("There are no candidates left. Exiting.\n")
sys.exit(0)
if pending.shape[0] >= options.max_concurrent:
sys.stderr.write("Maximum number of jobs (%d) pending.\n" %
(options.max_concurrent))
return
# Dont submit if pending + finished > max_finished_jobs.
if pending.shape[0] + complete.shape[0] >= options.max_finished_jobs:
sys.stderr.write("Full number of jobs (%d) submitted. Waiting for "
"completion.\n" % (options.max_finished_jobs))
return
# Ask the chooser to actually pick one.
job_id = chooser.next(grid, values, durations, candidates, pending,
complete)
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
sys.stderr.write("Selected job %d from the grid.\n" % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
# Make sure we have a job subdirectory.
job_subdir = os.path.join(expt_dir, 'jobs')
if not os.path.exists(job_subdir):
os.mkdir(job_subdir)
# Name this job file.
job_file = os.path.join(job_subdir, '%08d.pb' % (job_id))
# Store the job file.
save_job(job_file, job)
# Make sure there is a directory for output.
output_subdir = os.path.join(expt_dir, 'output')
if not os.path.exists(output_subdir):
os.mkdir(output_subdir)
output_file = os.path.join(output_subdir, '%08d.out' % (job_id))
process = job_submit("%s-%08d" % (expt_name, job_id), output_file,
job_file, work_dir)
process.poll()
if process.returncode is not None and process.returncode < 0:
sys.stderr.write("Failed to submit job or job crashed "
"with return code %d !\n" % process.returncode)
sys.stderr.write("Deleting job file.\n")
os.unlink(job_file)
return
else:
sys.stderr.write("Submitted job as process: %d\n" % process.pid)
# Now, update the experiment status to submitted.
expt_grid.set_submitted(job_id, process.pid)
return