def attempt_dispatch(expt_config, expt_dir, chooser, driver, options): log("\n" + "-" * 40) expt = load_experiment(expt_config) # Build the experiment grid. expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size, options.grid_seed) # Print out the current best function value. best_val, best_job = expt_grid.get_best() if best_job >= 0: log("Current best: %f (job %d)" % (best_val, best_job)) else: log("Current best: No results returned yet.") # Gets you everything - NaN for unknown values & durations. grid, values, durations = expt_grid.get_grid() # Returns lists of indices. candidates = expt_grid.get_candidates() pending = expt_grid.get_pending() complete = expt_grid.get_complete() n_candidates = candidates.shape[0] n_pending = pending.shape[0] n_complete = complete.shape[0] log("%d candidates %d pending %d complete" % (n_candidates, n_pending, n_complete)) # Verify that pending jobs are actually running, and add them back to the # candidate set if they have crashed or gotten lost. for job_id in pending: proc_id = expt_grid.get_proc_id(job_id) if not driver.is_proc_alive(job_id, proc_id): log("Set job %d back to pending status." % (job_id)) expt_grid.set_candidate(job_id) # Track the time series of optimization. write_trace(expt_dir, best_val, best_job, n_candidates, n_pending, n_complete) # Print out the best job results write_best_job(expt_dir, best_val, best_job, expt_grid) if n_complete >= options.max_finished_jobs: log("Maximum number of finished jobs (%d) reached." "Exiting" % options.max_finished_jobs) return False if n_candidates == 0: log("There are no candidates left. Exiting.") return False if n_pending >= options.max_concurrent: log("Maximum number of jobs (%d) pending." % (options.max_concurrent)) return True else: # start a bunch of candidate jobs if possible #to_start = min(options.max_concurrent - n_pending, n_candidates) #log("Trying to start %d jobs" % (to_start)) #for i in xrange(to_start): # Ask the chooser to pick the next candidate log("Choosing next candidate... ") job_id = chooser.next(grid, values, durations, candidates, pending, complete) # If the job_id is a tuple, then the chooser picked a new job. # We have to add this to our grid if isinstance(job_id, tuple): (job_id, candidate) = job_id job_id = expt_grid.add_to_grid(candidate) log("selected job %d from the grid." % (job_id)) # Convert this back into an interpretable job and add metadata. job = Job() job.id = job_id job.expt_dir = expt_dir job.name = expt.name job.language = expt.language job.status = 'submitted' job.submit_t = int(time.time()) job.param.extend(expt_grid.get_params(job_id)) save_job(job) pid = driver.submit_job(job) if pid != None: log("submitted - pid = %d" % (pid)) expt_grid.set_submitted(job_id, pid) else: log("Failed to submit job!") log("Deleting job file.") os.unlink(job_file_for(job)) return True
def attempt_dispatch(expt_config, expt_dir, chooser, driver, options): log("\n" + "-" * 40) expt = load_experiment(expt_config) print(options) # Build the experiment grid. expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size, options.grid_seed) # Print out the current best function value. best_val, best_job = expt_grid.get_best() if best_job >= 0: log("Current best: %f (job %d)" % (best_val, best_job)) else: log("Current best: No results returned yet.") # Gets you everything - NaN for unknown values & durations. grid, values, durations = expt_grid.get_grid() # Returns lists of indices. candidates = expt_grid.get_candidates() pending = expt_grid.get_pending() complete = expt_grid.get_complete() executed = expt_grid.get_executed() n_candidates = candidates.shape[0] n_pending = pending.shape[0] n_complete = complete.shape[0] n_executed = executed.shape[0] log("%d candidates %d pending %d complete %d executed" % (n_candidates, n_pending, n_complete, n_executed)) # Verify that pending jobs are actually running, and add them back to the # candidate set if they have crashed or gotten lost. for job_id in pending: proc_id = expt_grid.get_proc_id(job_id) if not driver.is_proc_alive(job_id, proc_id): log("Set job %d back to pending status." % (job_id)) expt_grid.set_candidate(job_id) # Track the time series of optimization. write_trace(expt_dir, best_val, best_job, n_candidates, n_pending, n_complete, n_executed) # Print out the best job results write_best_job(expt_dir, best_val, best_job, expt_grid) if n_complete >= options.max_finished_jobs: log("Maximum number of finished jobs (%d) reached." "Exiting" % options.max_finished_jobs) return False if n_candidates == 0: log("There are no candidates left. Exiting.") return False if n_pending >= options.max_concurrent: log("Maximum number of jobs (%d) pending." % (options.max_concurrent)) return True else: # start a bunch of candidate jobs if possible #to_start = min(options.max_concurrent - n_pending, n_candidates) #log("Trying to start %d jobs" % (to_start)) #for i in xrange(to_start): # Ask the chooser to pick the next candidate log("Choosing next candidate... ") job_id, ei = chooser.next(grid, values, durations, candidates, pending, complete) log("Expected improvement: %.6f" % ei) print ">>>>>>>", n_executed, ei if ei < config.EI and n_executed >= config.MIN_ACCEPTED_RUNS: config.strikes += 1 if config.strikes > 0: return False else: config.strikes = 0 # If the job_id is a tuple, then the chooser picked a new job. # We have to add this to our grid if isinstance(job_id, tuple): (job_id, candidate) = job_id job_id = expt_grid.add_to_grid(candidate) log("selected job %d from the grid." % (job_id)) # Convert this back into an interpretable job and add metadata. job = Job() job.id = job_id job.expt_dir = expt_dir job.name = expt.name job.language = expt.language job.status = 'submitted' job.submit_t = int(time.time()) job.param.extend(expt_grid.get_params(job_id)) #TODO: (@omid) check if the job has been previously completed; if so # mark the job as completed and use the cached value params = job_params(job) for key, val in params.items(): if isinstance(val, np.ndarray): val = val.tolist() if isinstance(val, list): val = frozenset(val) params[key] = val params = frozenset(params.items()) if params in jobs_executed: jid = jobs_executed[params] print ">>>> Bypassing job execution." for stat in ['status', 'values', 'durs']: dic = getattr(expt_grid, stat) dic[job_id] = dic[jid] expt_grid._save_jobs() return True jobs_executed[params] = job_id save_job(job) pid = driver.submit_job(job) if pid != None: log("submitted - pid = %d" % (pid)) expt_grid.set_submitted(job_id, pid) else: log("Failed to submit job!") log("Deleting job file.") os.unlink(job_file_for(job)) return True
def attempt_yaml_to_hyper(expt_config, expt_dir, driver, options, count, hyperList, yamlsDirectoryPath, number,mode): log("\n" + "-" * 40) expt = load_experiment(expt_config) # Build the experiment grid. expt_grid = ExperimentGrid(expt_dir, expt.variable, options.grid_size, options.grid_seed) # Print out the current best function value. best_val, best_job = expt_grid.get_best() if best_job >= 0: log("Current best: %f (job %d)" % (best_val, best_job)) else: log("Current best: No results returned yet.") # Returns lists of indices. pending = expt_grid.get_pending() complete = expt_grid.get_complete() n_pending = pending.shape[0] n_complete = complete.shape[0] log(" %d pending %d complete" % ( n_pending, n_complete)) if n_complete < count: print(" %d count %d complete" %( count, n_complete)) global Flag Flag = False return True else: global Flag Flag = True # Verify that pending jobs are actually running, and add them back to the # candidate set if they have crashed or gotten lost. for job_id in pending: proc_id = expt_grid.get_proc_id(job_id) if not driver.is_proc_alive(job_id, proc_id): log("Set job %d back to pending status." % (job_id)) expt_grid.set_candidate(job_id) # Print out the best job results write_best_job(expt_dir, best_val, best_job, expt_grid) if n_complete >= number: log("All yaml files are read and run in full data set." "Exiting" % options.max_finished_jobs) return False else: # choose the job-id to be the next job that read from hyper_yaml files job_id = count log("selected job %d from the grid." % (job_id)) # Convert this back into an interpretable job and add metadata. job = Job() job.id = job_id job.expt_dir = expt_dir job.name = expt.name job.language = expt.language job.status = 'submitted' job.submit_t = int(time.time()) # todo: read the line after the learn-rate in yamls file and reconding to the hyperopt.yaml file params = [] travelFile = listdir( yamlsDirectoryPath)[ count] filePath = join(yamlsDirectoryPath, travelFile) if mode== "get_hyper_from_output": if isfile( filePath): print("the path is " + travelFile) with open(filePath, 'r') as fin: for line in fin: if 'spear_wrapper params are:' in line: for i in xrange(len(hyperList)): hyper_param = hyperList[i] hyper_split = hyper_param.split() if hyper_split[1] in line: dic = ((line.split(hyper_split[1])[1]).split(": array([")[1]).split("])")[0]; param = Parameter() param.name = hyper_split[1] type = hyper_split[2] if type == "FLOAT": param.dbl_val.append (float(dic)) elif type == "INT": param.int_val.append (int(dic)) elif type == "ENUM": param.str_val.append (str(dic)) else: param.str_val.append (str(dic)) params.append(param) break param = Parameter() param.name = "experiment_dir" param.str_val.append (expt_dir) params.append(param) param = Parameter() param.name = "pre_output_file_name" param.str_val.append (travelFile) params.append(param) job.param.extend(params) else : job.param.extend(params) else: if isfile( filePath): print("the path is " + travelFile) i = 0 with open(filePath, 'r') as fin: for line in fin: if(i< hyperList.__len__()): hyper_param = hyperList[i] hyper_split = hyper_param.split() if hyper_split[0] in line: dic = [k.strip(",") for k in line.split()][1] param = Parameter() param.name = hyper_split[1] type = hyper_split[2] if type == "FLOAT": param.dbl_val.append (float(dic)) elif type == "INT": param.int_val.append (int(dic)) elif type == "ENUM": param.str_val.append (str(dic)) else: param.str_val.append (str(dic)) params.append(param) i = i+1 param = Parameter() param.name = "experiment_dir" param.str_val.append (expt_dir) params.append(param) job.param.extend(params) else : job.param.extend(params) save_job(job) pid = driver.submit_job(job) if pid != None: log("submitted - pid = %d" % (pid)) expt_grid.set_submitted(job_id, pid) else: log("Failed to submit job!") log("Deleting job file.") os.unlink(job_file_for(job)) return True