def _create_spearmint_parameters(self): self._create_mongo_instance() self.options, self.exp_dir = get_options(self.work_dir) self.resources = parse_resources_from_config(self.options) self.chooser_module = importlib.import_module('spearmint.choosers.' + self.options['chooser']) self.chooser = self.chooser_module.init(self.options) self.experiment_name = self.options.get('experiment-name', 'unnamed_experiment') self.db_address = self.options['database']['address'] self.db = MongoDB(database_address=self.db_address)
def main(): options, expt_dir = get_options() resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) while True: for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here remove_broken_jobs(db, jobs, experiment_name, resources) while resource.acceptingJobs(jobs): # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5))
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] directory") parser.add_option("--config", dest="config_file", help="Configuration file name.", type="string", default="config.json") (commandline_kwargs, args) = parser.parse_args() # Read in the config file expt_dir = os.path.realpath(args[0]) if not os.path.isdir(expt_dir): raise Exception("Cannot find directory %s" % expt_dir) expt_file = os.path.join(expt_dir, commandline_kwargs.config_file) try: with open(expt_file, 'r') as f: options = json.load(f, object_pairs_hook=OrderedDict) except: raise Exception("config.json did not load properly. Perhaps a spurious comma?") options["config"] = commandline_kwargs.config_file resources = parse_resources_from_config(options) # Set sensible defaults for options options['chooser'] = options.get('chooser', 'default_chooser') if 'tasks' not in options: options['tasks'] = {'main' : {'type' : 'OBJECTIVE', 'likelihood' : options.get('likelihood', 'GAUSSIAN')}} experiment_name = options.get("experiment-name", 'unnamed-experiment') # Set DB address db_address = parse_db_address(options) if 'database' not in options: options['database'] = {'name': 'spearmint', 'address': db_address} else: options['database']['address'] = db_address if not os.path.exists(expt_dir): sys.stderr.write("Cannot find experiment directory '%s'. " "Aborting.\n" % (expt_dir)) sys.exit(-1) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) # Connect to the database sys.stderr.write('Using database at %s.\n' % db_address) db_address = options['database']['address'] db = MongoDB(database_address=db_address) while True: for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5))
def main(): options, expt_dir = get_options() resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) while True: for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5))
def main(args=None): options, expt_dir = get_options(args) resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') resets = options.get("resets", []) job_id_offset = 0 current_phase = 0 if resets: experiment_name += '__' + str(current_phase) print 'STARTING PHASE ' + str(current_phase + 1) + ' (' + experiment_name + ')' # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) while True: pause = False if resets: jobs = load_jobs(db, experiment_name) num_pending_jobs = sum( [job['status'] == 'pending' for job in jobs]) num_finished_jobs = sum( [job['status'] == 'complete' for job in jobs]) if num_finished_jobs == resets[ current_phase] and num_pending_jobs == 0: job_id_offset += resets[current_phase] current_phase += 1 new_experiment_name = options.get( "experiment-name", 'unnamed-experiment') + '__' + str(current_phase) print 'STARTING PHASE ' + str( current_phase + 1) + ' (' + new_experiment_name + ')' old_hypers = load_hypers(db, experiment_name) save_hypers(old_hypers, db, new_experiment_name) experiment_name = new_experiment_name if num_finished_jobs + num_pending_jobs >= resets[current_phase]: pause = True for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs) and not pause: # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) print 'Found', len(jobs), 'jobs in db' # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, experiment_name, expt_dir, options, resource_name, job_id_offset) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources) or pause: time.sleep(options.get('polling-time', 5))
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] directory") parser.add_option("--config", dest="config_file", help="Configuration file name.", type="string", default="config.json") (commandline_kwargs, args) = parser.parse_args() # Read in the config file expt_dir = os.path.realpath(args[0]) if not os.path.isdir(expt_dir): raise Exception("Cannot find directory %s" % expt_dir) expt_file = os.path.join(expt_dir, commandline_kwargs.config_file) try: with open(expt_file, 'r') as f: options = json.load(f, object_pairs_hook=OrderedDict) except: raise Exception( "config.json did not load properly. Perhaps a spurious comma?") options["config"] = commandline_kwargs.config_file resources = parse_resources_from_config(options) # Set sensible defaults for options options['chooser'] = options.get('chooser', 'default_chooser') options['tasks'] = options.get( 'tasks', {'main': { 'type': 'OBJECTIVE', 'likelihood': 'GAUSSIAN' }}) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Set DB address db_address = parse_db_address(options) if 'database' not in options: options['database'] = {'name': 'spearmint', 'address': db_address} else: options['database']['address'] = db_address if not os.path.exists(expt_dir): sys.stderr.write("Cannot find experiment directory '%s'. " "Aborting.\n" % (expt_dir)) sys.exit(-1) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) # Connect to the database sys.stderr.write('Using database at %s.\n' % db_address) db_address = options['database']['address'] db = MongoDB(database_address=db_address) while True: for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5))
def main(): options, expt_dir = get_options() resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) # Setting up record for convergence past_best = [] converg_num = 20 startTraining = time.time() while stoppingCriterion(past_best, converg_num): for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) #print jobs[0]['values']['main'] #resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # Record current best best_val, best_input = chooser.get_best() past_best.append(best_val) past_best = [x for x in past_best if x is not None] #filter out Nones if len(past_best) > converg_num: past_best.pop(0) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): print "Sleeping..." time.sleep(options.get('polling-time', 5)) endTraining = time.time() trainingTime = endTraining - startTraining # After training, test best results runBestParams(5000, chooser, db, experiment_name, trainingTime)
def main(args=None): options, expt_dir = get_options(args) resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') resets = options.get("resets", []) job_id_offset = 0 current_phase = 0 if resets: experiment_name += '__' + str(current_phase) print 'STARTING PHASE ' + str(current_phase + 1) + ' (' + experiment_name + ')' # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) while True: pause = False if resets: jobs = load_jobs(db, experiment_name) num_pending_jobs = sum([job['status'] == 'pending' for job in jobs]) num_finished_jobs = sum([job['status'] == 'complete' for job in jobs]) if num_finished_jobs == resets[current_phase] and num_pending_jobs == 0: job_id_offset += resets[current_phase] current_phase += 1 new_experiment_name = options.get("experiment-name", 'unnamed-experiment') + '__' + str(current_phase) print 'STARTING PHASE ' + str(current_phase + 1) + ' (' + new_experiment_name + ')' old_hypers = load_hypers(db, experiment_name) save_hypers(old_hypers, db, new_experiment_name) experiment_name = new_experiment_name if num_finished_jobs + num_pending_jobs >= resets[current_phase]: pause = True for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs) and not pause: # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) print 'Found', len(jobs), 'jobs in db' # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, experiment_name, expt_dir, options, resource_name, job_id_offset) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources) or pause: time.sleep(options.get('polling-time', 5))
def main(): options, expt_dir = get_options() resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) # np.random.seed(0x6b6c26b2) hack_iter = 0 while hack_iter < 50: for resource_name, resource in resources.iteritems(): # TODO:: (moonkey) might be in vain, as the jobs will be loaded later in the inner loop jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): ################ HACK BEGIN ################ # This is where each rounds begins # If we are running EM or other methods, we may get extra points and objective values # Here is the point where we save them into the database, to fool the algorithm # to treat them as if they are sampled by BO, and utilize them to compute the GP and the # acquisition functions. # We are assuming the data are written in json format at the folder of 'config.json' file. hack_iter += 1 sys.stderr.write('###########hack_iter:' + str(hack_iter) + "###########\n") em_hack.add_historical_points_to_db(db, experiment_name, expt_dir) # # (moonkey) towards removing the randomness # np.random.seed(0x6b6c26b2) # if hack_iter == 3: # em_hack.add_historical_points_to_db(db, experiment_name, expt_dir) # continue ################ HACK END ################ # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so # some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 0))
def main(args): options, expt_dir = get_options(args) print(options) resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Connect to the database db_address = options['database']['address'] sys.stderr.write('Using database at %s.\n' % db_address) db = MongoDB(database_address=db_address) # Changed the loop so that it's not for forever budget = options.get("budget", 20) count = options.get("count", 0) ei_threshold = options.get("ei", 0.10) max_budget = options.get("maxbudget", 10) while count < budget or (chooser.ei >= ei_threshold and count < max_budget): for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) #resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): if count < budget or (chooser.ei >= ei_threshold and count < max_budget): sys.stderr.write("Proceeding to next experiment\n") # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) count += 1 else: #count += 1 break #sys.stderr.write('Waiting for a prior job to finish\n') # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5)) while tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5)) #best_input, best_val = chooser.best() # print(chooser.task_group.paramify(chooser.best_location.flatten())) print(chooser.best_value) return chooser.best_jobid, chooser.best_value, count
def main(): options, expt_dir = get_options() resources = parse_resources_from_config(options) # Load up the chooser. chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(options) experiment_name = options.get("experiment-name", 'unnamed-experiment') # Connect to the database db_address = options['database']['address'] db_name = options['database']['name'] sys.stderr.write('Using database %s at %s.\n' % (db_name, db_address)) db = MongoDB(database_address=db_address, database_name=db_name) suggest_file = os.path.join(expt_dir , experiment_name + ".suggest") suggest_idx = 0 while True: for resource_name, resource in resources.iteritems(): jobs = load_jobs(db, experiment_name) # resource.printStatus(jobs) # If the resource is currently accepting more jobs # TODO: here cost will eventually also be considered: even if the # resource is not full, we might wait because of cost incurred # Note: I chose to fill up one resource and them move on to the next # You could also do it the other way, by changing "while" to "if" here while resource.acceptingJobs(jobs): #db['rnn_8.jobs'].remove({status:'new'}) # Load jobs from DB # (move out of one or both loops?) would need to pass into load_tasks jobs = load_jobs(db, experiment_name) # Remove any broken jobs from pending. remove_broken_jobs(db, jobs, experiment_name, resources) # Get a suggestion for the next job suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name) try: # Check if the file for the manual suggestions exists if os.path.isfile(suggest_file): suggest_params = [] with open(suggest_file,'r') as csvfile: # There should not be a blank line in the beginning of the file! reader = csv.DictReader(csvfile) # Concatenate all the suggestions in the file for row in reader: suggest_params = suggest_params + [row] # If a new line is added to the file we overwrite the suggested jobs with these values if suggest_idx < len(suggest_params): print "--- Using manual suggestion instead of the one coming from the GP! ---" next_suggestion = suggest_params[suggest_idx] for key, value in next_suggestion.iteritems(): if isinstance(value,str): value=value.strip() suggested_job['params'][key.strip()]['values'][0] = value suggested_job['manual'] = 1 print "%s: %s" %(key.strip(),value) suggest_idx = suggest_idx + 1 except: print "--- Problem using the manual suggestion file! Back to the GP suggestion.. ---" # Submit the job to the appropriate resource process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, db_name, expt_dir) # Set the status of the job appropriately (successfully submitted or not) if process_id is None: suggested_job['status'] = 'broken' save_job(suggested_job, db, experiment_name) else: suggested_job['status'] = 'pending' suggested_job['proc_id'] = process_id save_job(suggested_job, db, experiment_name) jobs = load_jobs(db, experiment_name) # Print out the status of the resources # resource.printStatus(jobs) print_resources_status(resources.values(), jobs) # If no resources are accepting jobs, sleep # (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached) if tired(db, experiment_name, resources): time.sleep(options.get('polling-time', 5))