def _create_spearmint_parameters(self):
self._create_mongo_instance()
self.options, self.exp_dir = get_options(self.work_dir)
self.resources = parse_resources_from_config(self.options)
self.chooser_module = importlib.import_module('spearmint.choosers.' +
self.options['chooser'])
self.chooser = self.chooser_module.init(self.options)
self.experiment_name = self.options.get('experiment-name',
'unnamed_experiment')
self.db_address = self.options['database']['address']
self.db = MongoDB(database_address=self.db_address)
def main():
options, expt_dir = get_options()
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
while True:
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
remove_broken_jobs(db, jobs, experiment_name, resources)
while resource.acceptingJobs(jobs):
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] directory")
parser.add_option("--config", dest="config_file",
help="Configuration file name.",
type="string", default="config.json")
(commandline_kwargs, args) = parser.parse_args()
# Read in the config file
expt_dir = os.path.realpath(args[0])
if not os.path.isdir(expt_dir):
raise Exception("Cannot find directory %s" % expt_dir)
expt_file = os.path.join(expt_dir, commandline_kwargs.config_file)
try:
with open(expt_file, 'r') as f:
options = json.load(f, object_pairs_hook=OrderedDict)
except:
raise Exception("config.json did not load properly. Perhaps a spurious comma?")
options["config"] = commandline_kwargs.config_file
resources = parse_resources_from_config(options)
# Set sensible defaults for options
options['chooser'] = options.get('chooser', 'default_chooser')
if 'tasks' not in options:
options['tasks'] = {'main' : {'type' : 'OBJECTIVE', 'likelihood' : options.get('likelihood', 'GAUSSIAN')}}
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Set DB address
db_address = parse_db_address(options)
if 'database' not in options:
options['database'] = {'name': 'spearmint', 'address': db_address}
else:
options['database']['address'] = db_address
if not os.path.exists(expt_dir):
sys.stderr.write("Cannot find experiment directory '%s'. "
"Aborting.\n" % (expt_dir))
sys.exit(-1)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
# Connect to the database
sys.stderr.write('Using database at %s.\n' % db_address)
db_address = options['database']['address']
db = MongoDB(database_address=db_address)
while True:
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
def main():
options, expt_dir = get_options()
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
while True:
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
def main(args=None):
options, expt_dir = get_options(args)
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' +
options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
resets = options.get("resets", [])
job_id_offset = 0
current_phase = 0
if resets:
experiment_name += '__' + str(current_phase)
print 'STARTING PHASE ' + str(current_phase +
1) + ' (' + experiment_name + ')'
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
while True:
pause = False
if resets:
jobs = load_jobs(db, experiment_name)
num_pending_jobs = sum(
[job['status'] == 'pending' for job in jobs])
num_finished_jobs = sum(
[job['status'] == 'complete' for job in jobs])
if num_finished_jobs == resets[
current_phase] and num_pending_jobs == 0:
job_id_offset += resets[current_phase]
current_phase += 1
new_experiment_name = options.get(
"experiment-name",
'unnamed-experiment') + '__' + str(current_phase)
print 'STARTING PHASE ' + str(
current_phase + 1) + ' (' + new_experiment_name + ')'
old_hypers = load_hypers(db, experiment_name)
save_hypers(old_hypers, db, new_experiment_name)
experiment_name = new_experiment_name
if num_finished_jobs + num_pending_jobs >= resets[current_phase]:
pause = True
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs) and not pause:
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
print 'Found', len(jobs), 'jobs in db'
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db,
experiment_name, expt_dir,
options, resource_name,
job_id_offset)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name,
suggested_job,
db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources) or pause:
time.sleep(options.get('polling-time', 5))
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] directory")
parser.add_option("--config",
dest="config_file",
help="Configuration file name.",
type="string",
default="config.json")
(commandline_kwargs, args) = parser.parse_args()
# Read in the config file
expt_dir = os.path.realpath(args[0])
if not os.path.isdir(expt_dir):
raise Exception("Cannot find directory %s" % expt_dir)
expt_file = os.path.join(expt_dir, commandline_kwargs.config_file)
try:
with open(expt_file, 'r') as f:
options = json.load(f, object_pairs_hook=OrderedDict)
except:
raise Exception(
"config.json did not load properly. Perhaps a spurious comma?")
options["config"] = commandline_kwargs.config_file
resources = parse_resources_from_config(options)
# Set sensible defaults for options
options['chooser'] = options.get('chooser', 'default_chooser')
options['tasks'] = options.get(
'tasks', {'main': {
'type': 'OBJECTIVE',
'likelihood': 'GAUSSIAN'
}})
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Set DB address
db_address = parse_db_address(options)
if 'database' not in options:
options['database'] = {'name': 'spearmint', 'address': db_address}
else:
options['database']['address'] = db_address
if not os.path.exists(expt_dir):
sys.stderr.write("Cannot find experiment directory '%s'. "
"Aborting.\n" % (expt_dir))
sys.exit(-1)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' +
options['chooser'])
chooser = chooser_module.init(options)
# Connect to the database
sys.stderr.write('Using database at %s.\n' % db_address)
db_address = options['database']['address']
db = MongoDB(database_address=db_address)
while True:
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db,
expt_dir, options,
resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name,
suggested_job,
db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
def main():
options, expt_dir = get_options()
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' +
options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
# Setting up record for convergence
past_best = []
converg_num = 20
startTraining = time.time()
while stoppingCriterion(past_best, converg_num):
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
#print jobs[0]['values']['main']
#resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db,
expt_dir, options,
resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name,
suggested_job,
db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# Record current best
best_val, best_input = chooser.get_best()
past_best.append(best_val)
past_best = [x for x in past_best
if x is not None] #filter out Nones
if len(past_best) > converg_num:
past_best.pop(0)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
print "Sleeping..."
time.sleep(options.get('polling-time', 5))
endTraining = time.time()
trainingTime = endTraining - startTraining
# After training, test best results
runBestParams(5000, chooser, db, experiment_name, trainingTime)
def main(args=None):
options, expt_dir = get_options(args)
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
resets = options.get("resets", [])
job_id_offset = 0
current_phase = 0
if resets:
experiment_name += '__' + str(current_phase)
print 'STARTING PHASE ' + str(current_phase + 1) + ' (' + experiment_name + ')'
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
while True:
pause = False
if resets:
jobs = load_jobs(db, experiment_name)
num_pending_jobs = sum([job['status'] == 'pending' for job in jobs])
num_finished_jobs = sum([job['status'] == 'complete' for job in jobs])
if num_finished_jobs == resets[current_phase] and num_pending_jobs == 0:
job_id_offset += resets[current_phase]
current_phase += 1
new_experiment_name = options.get("experiment-name", 'unnamed-experiment') + '__' + str(current_phase)
print 'STARTING PHASE ' + str(current_phase + 1) + ' (' + new_experiment_name + ')'
old_hypers = load_hypers(db, experiment_name)
save_hypers(old_hypers, db, new_experiment_name)
experiment_name = new_experiment_name
if num_finished_jobs + num_pending_jobs >= resets[current_phase]:
pause = True
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs) and not pause:
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
print 'Found', len(jobs), 'jobs in db'
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, experiment_name, expt_dir, options,
resource_name, job_id_offset)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources) or pause:
time.sleep(options.get('polling-time', 5))
def main():
options, expt_dir = get_options()
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
# np.random.seed(0x6b6c26b2)
hack_iter = 0
while hack_iter < 50:
for resource_name, resource in resources.iteritems():
# TODO:: (moonkey) might be in vain, as the jobs will be loaded later in the inner loop
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
################ HACK BEGIN ################
# This is where each rounds begins
# If we are running EM or other methods, we may get extra points and objective values
# Here is the point where we save them into the database, to fool the algorithm
# to treat them as if they are sampled by BO, and utilize them to compute the GP and the
# acquisition functions.
# We are assuming the data are written in json format at the folder of 'config.json' file.
hack_iter += 1
sys.stderr.write('###########hack_iter:' + str(hack_iter) + "###########\n")
em_hack.add_historical_points_to_db(db, experiment_name, expt_dir)
# # (moonkey) towards removing the randomness
# np.random.seed(0x6b6c26b2)
# if hack_iter == 3:
# em_hack.add_historical_points_to_db(db, experiment_name, expt_dir)
# continue
################ HACK END ################
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so
# some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 0))
def main(args):
options, expt_dir = get_options(args)
print(options)
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
sys.stderr.write('Using database at %s.\n' % db_address)
db = MongoDB(database_address=db_address)
# Changed the loop so that it's not for forever
budget = options.get("budget", 20)
count = options.get("count", 0)
ei_threshold = options.get("ei", 0.10)
max_budget = options.get("maxbudget", 10)
while count < budget or (chooser.ei >= ei_threshold and count < max_budget):
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
#resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
if count < budget or (chooser.ei >= ei_threshold and count < max_budget):
sys.stderr.write("Proceeding to next experiment\n")
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name)
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
count += 1
else:
#count += 1
break
#sys.stderr.write('Waiting for a prior job to finish\n')
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
while tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
#best_input, best_val = chooser.best()
# print(chooser.task_group.paramify(chooser.best_location.flatten()))
print(chooser.best_value)
return chooser.best_jobid, chooser.best_value, count
def main():
options, expt_dir = get_options()
resources = parse_resources_from_config(options)
# Load up the chooser.
chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser'])
chooser = chooser_module.init(options)
experiment_name = options.get("experiment-name", 'unnamed-experiment')
# Connect to the database
db_address = options['database']['address']
db_name = options['database']['name']
sys.stderr.write('Using database %s at %s.\n' % (db_name, db_address))
db = MongoDB(database_address=db_address, database_name=db_name)
suggest_file = os.path.join(expt_dir , experiment_name + ".suggest")
suggest_idx = 0
while True:
for resource_name, resource in resources.iteritems():
jobs = load_jobs(db, experiment_name)
# resource.printStatus(jobs)
# If the resource is currently accepting more jobs
# TODO: here cost will eventually also be considered: even if the
# resource is not full, we might wait because of cost incurred
# Note: I chose to fill up one resource and them move on to the next
# You could also do it the other way, by changing "while" to "if" here
while resource.acceptingJobs(jobs):
#db['rnn_8.jobs'].remove({status:'new'})
# Load jobs from DB
# (move out of one or both loops?) would need to pass into load_tasks
jobs = load_jobs(db, experiment_name)
# Remove any broken jobs from pending.
remove_broken_jobs(db, jobs, experiment_name, resources)
# Get a suggestion for the next job
suggested_job = get_suggestion(chooser, resource.tasks, db, expt_dir, options, resource_name)
try:
# Check if the file for the manual suggestions exists
if os.path.isfile(suggest_file):
suggest_params = []
with open(suggest_file,'r') as csvfile:
# There should not be a blank line in the beginning of the file!
reader = csv.DictReader(csvfile)
# Concatenate all the suggestions in the file
for row in reader:
suggest_params = suggest_params + [row]
# If a new line is added to the file we overwrite the suggested jobs with these values
if suggest_idx < len(suggest_params):
print "--- Using manual suggestion instead of the one coming from the GP! ---"
next_suggestion = suggest_params[suggest_idx]
for key, value in next_suggestion.iteritems():
if isinstance(value,str):
value=value.strip()
suggested_job['params'][key.strip()]['values'][0] = value
suggested_job['manual'] = 1
print "%s: %s" %(key.strip(),value)
suggest_idx = suggest_idx + 1
except:
print "--- Problem using the manual suggestion file! Back to the GP suggestion.. ---"
# Submit the job to the appropriate resource
process_id = resource.attemptDispatch(experiment_name, suggested_job, db_address, db_name, expt_dir)
# Set the status of the job appropriately (successfully submitted or not)
if process_id is None:
suggested_job['status'] = 'broken'
save_job(suggested_job, db, experiment_name)
else:
suggested_job['status'] = 'pending'
suggested_job['proc_id'] = process_id
save_job(suggested_job, db, experiment_name)
jobs = load_jobs(db, experiment_name)
# Print out the status of the resources
# resource.printStatus(jobs)
print_resources_status(resources.values(), jobs)
# If no resources are accepting jobs, sleep
# (they might be accepting if suggest takes a while and so some jobs already finished by the time this point is reached)
if tired(db, experiment_name, resources):
time.sleep(options.get('polling-time', 5))
