def attempt_dispatch(expt_config, expt_dir, chooser, driver, options):
log("\n" + "-" * 40)
expt = load_experiment(expt_config)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
if best_job >= 0:
log("Current best: %f (job %d)" % (best_val, best_job))
else:
log("Current best: No results returned yet.")
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
n_candidates = candidates.shape[0]
n_pending = pending.shape[0]
n_complete = complete.shape[0]
log("%d candidates %d pending %d complete" %
(n_candidates, n_pending, n_complete))
# Verify that pending jobs are actually running, and add them back to the
# candidate set if they have crashed or gotten lost.
for job_id in pending:
proc_id = expt_grid.get_proc_id(job_id)
if not driver.is_proc_alive(job_id, proc_id):
log("Set job %d back to pending status." % (job_id))
expt_grid.set_candidate(job_id)
# Track the time series of optimization.
write_trace(expt_dir, best_val, best_job, n_candidates, n_pending,
n_complete)
# Print out the best job results
write_best_job(expt_dir, best_val, best_job, expt_grid)
if n_complete >= options.max_finished_jobs:
log("Maximum number of finished jobs (%d) reached."
"Exiting" % options.max_finished_jobs)
return False
if n_candidates == 0:
log("There are no candidates left. Exiting.")
return False
if n_pending >= options.max_concurrent:
log("Maximum number of jobs (%d) pending." % (options.max_concurrent))
return True
else:
# start a bunch of candidate jobs if possible
#to_start = min(options.max_concurrent - n_pending, n_candidates)
#log("Trying to start %d jobs" % (to_start))
#for i in xrange(to_start):
# Ask the chooser to pick the next candidate
log("Choosing next candidate... ")
job_id = chooser.next(grid, values, durations, candidates, pending,
complete)
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
log("selected job %d from the grid." % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
save_job(job)
pid = driver.submit_job(job)
if pid != None:
log("submitted - pid = %d" % (pid))
expt_grid.set_submitted(job_id, pid)
else:
log("Failed to submit job!")
log("Deleting job file.")
os.unlink(job_file_for(job))
return True
def attempt_dispatch(expt_config, expt_dir, chooser, driver, options):
log("\n" + "-" * 40)
expt = load_experiment(expt_config)
print(options)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
if best_job >= 0:
log("Current best: %f (job %d)" % (best_val, best_job))
else:
log("Current best: No results returned yet.")
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
executed = expt_grid.get_executed()
n_candidates = candidates.shape[0]
n_pending = pending.shape[0]
n_complete = complete.shape[0]
n_executed = executed.shape[0]
log("%d candidates %d pending %d complete %d executed" %
(n_candidates, n_pending, n_complete, n_executed))
# Verify that pending jobs are actually running, and add them back to the
# candidate set if they have crashed or gotten lost.
for job_id in pending:
proc_id = expt_grid.get_proc_id(job_id)
if not driver.is_proc_alive(job_id, proc_id):
log("Set job %d back to pending status." % (job_id))
expt_grid.set_candidate(job_id)
# Track the time series of optimization.
write_trace(expt_dir, best_val, best_job, n_candidates, n_pending,
n_complete, n_executed)
# Print out the best job results
write_best_job(expt_dir, best_val, best_job, expt_grid)
if n_complete >= options.max_finished_jobs:
log("Maximum number of finished jobs (%d) reached."
"Exiting" % options.max_finished_jobs)
return False
if n_candidates == 0:
log("There are no candidates left. Exiting.")
return False
if n_pending >= options.max_concurrent:
log("Maximum number of jobs (%d) pending." % (options.max_concurrent))
return True
else:
# start a bunch of candidate jobs if possible
#to_start = min(options.max_concurrent - n_pending, n_candidates)
#log("Trying to start %d jobs" % (to_start))
#for i in xrange(to_start):
# Ask the chooser to pick the next candidate
log("Choosing next candidate... ")
job_id, ei = chooser.next(grid, values, durations, candidates, pending,
complete)
log("Expected improvement: %.6f" % ei)
print ">>>>>>>", n_executed, ei
if ei < config.EI and n_executed >= config.MIN_ACCEPTED_RUNS:
config.strikes += 1
if config.strikes > 0:
return False
else:
config.strikes = 0
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
log("selected job %d from the grid." % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
#TODO: (@omid) check if the job has been previously completed; if so
# mark the job as completed and use the cached value
params = job_params(job)
for key, val in params.items():
if isinstance(val, np.ndarray):
val = val.tolist()
if isinstance(val, list):
val = frozenset(val)
params[key] = val
params = frozenset(params.items())
if params in jobs_executed:
jid = jobs_executed[params]
print ">>>> Bypassing job execution."
for stat in ['status', 'values', 'durs']:
dic = getattr(expt_grid, stat)
dic[job_id] = dic[jid]
expt_grid._save_jobs()
return True
jobs_executed[params] = job_id
save_job(job)
pid = driver.submit_job(job)
if pid != None:
log("submitted - pid = %d" % (pid))
expt_grid.set_submitted(job_id, pid)
else:
log("Failed to submit job!")
log("Deleting job file.")
os.unlink(job_file_for(job))
return True
def attempt_dispatch(expt_config, expt_dir, chooser, driver, options):
log("\n" + "-" * 40)
expt = load_experiment(expt_config)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir,
expt.variable,
options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
if best_job >= 0:
log("Current best: %f (job %d)" % (best_val, best_job))
else:
log("Current best: No results returned yet.")
# Gets you everything - NaN for unknown values & durations.
grid, values, durations = expt_grid.get_grid()
# Returns lists of indices.
candidates = expt_grid.get_candidates()
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
n_candidates = candidates.shape[0]
n_pending = pending.shape[0]
n_complete = complete.shape[0]
log("%d candidates %d pending %d complete" %
(n_candidates, n_pending, n_complete))
# Verify that pending jobs are actually running, and add them back to the
# candidate set if they have crashed or gotten lost.
for job_id in pending:
proc_id = expt_grid.get_proc_id(job_id)
if not driver.is_proc_alive(job_id, proc_id):
log("Set job %d back to pending status." % (job_id))
expt_grid.set_candidate(job_id)
# Track the time series of optimization.
write_trace(expt_dir, best_val, best_job, n_candidates, n_pending, n_complete)
# Print out the best job results
write_best_job(expt_dir, best_val, best_job, expt_grid)
if n_complete >= options.max_finished_jobs:
log("Maximum number of finished jobs (%d) reached."
"Exiting" % options.max_finished_jobs)
return False
if n_candidates == 0:
log("There are no candidates left. Exiting.")
return False
if n_pending >= options.max_concurrent:
log("Maximum number of jobs (%d) pending." % (options.max_concurrent))
return True
else:
# start a bunch of candidate jobs if possible
#to_start = min(options.max_concurrent - n_pending, n_candidates)
#log("Trying to start %d jobs" % (to_start))
#for i in xrange(to_start):
# Ask the chooser to pick the next candidate
log("Choosing next candidate... ")
job_id = chooser.next(grid, values, durations, candidates, pending, complete)
# If the job_id is a tuple, then the chooser picked a new job.
# We have to add this to our grid
if isinstance(job_id, tuple):
(job_id, candidate) = job_id
job_id = expt_grid.add_to_grid(candidate)
log("selected job %d from the grid." % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
job.param.extend(expt_grid.get_params(job_id))
save_job(job)
pid = driver.submit_job(job)
if pid != None:
log("submitted - pid = %d" % (pid))
expt_grid.set_submitted(job_id, pid)
else:
log("Failed to submit job!")
log("Deleting job file.")
os.unlink(job_file_for(job))
return True
def attempt_yaml_to_hyper(expt_config, expt_dir, driver, options, count, hyperList, yamlsDirectoryPath, number,mode):
log("\n" + "-" * 40)
expt = load_experiment(expt_config)
# Build the experiment grid.
expt_grid = ExperimentGrid(expt_dir,
expt.variable,
options.grid_size,
options.grid_seed)
# Print out the current best function value.
best_val, best_job = expt_grid.get_best()
if best_job >= 0:
log("Current best: %f (job %d)" % (best_val, best_job))
else:
log("Current best: No results returned yet.")
# Returns lists of indices.
pending = expt_grid.get_pending()
complete = expt_grid.get_complete()
n_pending = pending.shape[0]
n_complete = complete.shape[0]
log(" %d pending %d complete" %
( n_pending, n_complete))
if n_complete < count:
print(" %d count %d complete" %( count, n_complete))
global Flag
Flag = False
return True
else:
global Flag
Flag = True
# Verify that pending jobs are actually running, and add them back to the
# candidate set if they have crashed or gotten lost.
for job_id in pending:
proc_id = expt_grid.get_proc_id(job_id)
if not driver.is_proc_alive(job_id, proc_id):
log("Set job %d back to pending status." % (job_id))
expt_grid.set_candidate(job_id)
# Print out the best job results
write_best_job(expt_dir, best_val, best_job, expt_grid)
if n_complete >= number:
log("All yaml files are read and run in full data set."
"Exiting" % options.max_finished_jobs)
return False
else:
# choose the job-id to be the next job that read from hyper_yaml files
job_id = count
log("selected job %d from the grid." % (job_id))
# Convert this back into an interpretable job and add metadata.
job = Job()
job.id = job_id
job.expt_dir = expt_dir
job.name = expt.name
job.language = expt.language
job.status = 'submitted'
job.submit_t = int(time.time())
# todo: read the line after the learn-rate in yamls file and reconding to the hyperopt.yaml file
params = []
travelFile = listdir( yamlsDirectoryPath)[ count]
filePath = join(yamlsDirectoryPath, travelFile)
if mode== "get_hyper_from_output":
if isfile( filePath):
print("the path is " + travelFile)
with open(filePath, 'r') as fin:
for line in fin:
if 'spear_wrapper params are:' in line:
for i in xrange(len(hyperList)):
hyper_param = hyperList[i]
hyper_split = hyper_param.split()
if hyper_split[1] in line:
dic = ((line.split(hyper_split[1])[1]).split(": array([")[1]).split("])")[0];
param = Parameter()
param.name = hyper_split[1]
type = hyper_split[2]
if type == "FLOAT":
param.dbl_val.append (float(dic))
elif type == "INT":
param.int_val.append (int(dic))
elif type == "ENUM":
param.str_val.append (str(dic))
else:
param.str_val.append (str(dic))
params.append(param)
break
param = Parameter()
param.name = "experiment_dir"
param.str_val.append (expt_dir)
params.append(param)
param = Parameter()
param.name = "pre_output_file_name"
param.str_val.append (travelFile)
params.append(param)
job.param.extend(params)
else :
job.param.extend(params)
else:
if isfile( filePath):
print("the path is " + travelFile)
i = 0
with open(filePath, 'r') as fin:
for line in fin:
if(i< hyperList.__len__()):
hyper_param = hyperList[i]
hyper_split = hyper_param.split()
if hyper_split[0] in line:
dic = [k.strip(",") for k in line.split()][1]
param = Parameter()
param.name = hyper_split[1]
type = hyper_split[2]
if type == "FLOAT":
param.dbl_val.append (float(dic))
elif type == "INT":
param.int_val.append (int(dic))
elif type == "ENUM":
param.str_val.append (str(dic))
else:
param.str_val.append (str(dic))
params.append(param)
i = i+1
param = Parameter()
param.name = "experiment_dir"
param.str_val.append (expt_dir)
params.append(param)
job.param.extend(params)
else :
job.param.extend(params)
save_job(job)
pid = driver.submit_job(job)
if pid != None:
log("submitted - pid = %d" % (pid))
expt_grid.set_submitted(job_id, pid)
else:
log("Failed to submit job!")
log("Deleting job file.")
os.unlink(job_file_for(job))
return True