def set_jobs(name, status, new_status):
experiments = experiment_scheduler.load_experiments(
cluster=None, filter_eq_dct=dict(name=name))
if len(experiments) == 0:
print "No experiments in database %s" % get_db_string("experiments")
return
experiment = experiments[0]
if status == "pending":
jobs = job_scheduler.load_pending_jobs(experiment["table"])
elif status == "broken":
jobs = job_scheduler.load_broken_jobs(experiment["table"])
elif status == "completed":
jobs = job_scheduler.load_completed_jobs(experiment["table"])
elif status == "running":
jobs = job_scheduler.load_running_jobs(experiment["table"])
if new_status == "pending":
sql_new_status = sql.START
new_status = "pending"
elif new_status == "running":
sql_new_status = sql.RUNNING
elif new_status == "completed":
sql_new_status = sql.COMPLETE
print "Setting %s jobs to %s status..." % (status, new_status)
job_scheduler.update_jobs(experiment["table"], jobs,
expand({sql.STATUS: sql_new_status,
'proc_status': new_status}))
def launch(cluster, limit, experiment):
if cluster is None:
raise ValueError("cluster must be specified for launch option")
if experiment:
filter_eq_dct = dict(name=experiment)
else:
filter_eq_dct = None
experiments = experiment_scheduler.load_experiments(
cluster, filter_eq_dct=filter_eq_dct)
random.shuffle(experiments)
if len(experiments) == 0:
print "No experiments in database %s" % get_db_string("experiments")
return
for experiment in experiments:
nb_of_jobs_to_launch = monitor_single_experiment(cluster, experiment)
if nb_of_jobs_to_launch > 0:
if limit and nb_of_jobs_to_launch > limit:
print "would submit %d new jobs" % nb_of_jobs_to_launch
nb_of_jobs_to_launch = limit
print "submitting %d new jobs" % nb_of_jobs_to_launch
job_scheduler.submit_job(cluster, experiment, nb_of_jobs_to_launch)
else:
print "no job to launch"
print "\n"
time.sleep(15)
def plot_experiment(experiment_name):
experiments = experiment_scheduler.load_experiments(
cluster=None, filter_eq_dct=dict(name=experiment_name))
if len(experiments) == 0:
print "No experiments in database %s" % get_db_string("experiments")
experiment = experiments[0]
jobs = job_scheduler.load_jobs(experiment['table'])
return plot(jobs)
def monitor(cluster):
if cluster is None:
raise ValueError("cluster must be specified for monitoring")
experiments = experiment_scheduler.load_experiments(cluster)
if len(experiments) == 0:
print "No experiments in database %s" % get_db_string("experiments")
return
for experiment in experiments:
nb_of_jobs_to_launch = monitor_single_experiment(cluster, experiment)
print "would submit %d new jobs\n" % nb_of_jobs_to_launch
def remove_experiment(name):
experiments = experiment_scheduler.load_experiments(
cluster=None, filter_eq_dct=dict(name=name))
if len(experiments) == 0:
print "No experiments in database %s" % get_db_string("experiments")
return
experiment = experiments[0]
table_name = experiment["table"]
if query_yes_no("Do you really want to delete experiment %s?" % bold(name)):
print "Deleting %s..." % name
experiment_scheduler.delete_experiments([experiment])
if query_yes_no("Do you want to delete corresponding jobs?"):
jobs = job_scheduler.load_jobs(table_name)
print "Deleting %d jobs..." % len(jobs)
job_scheduler.delete_jobs(table_name, jobs)
def list_experiments(cluster):
experiments = experiment_scheduler.load_experiments(cluster)
if len(experiments) == 0:
print "No experiments in database %s" % get_db_string("experiments")
for experiment in experiments:
print "Name:", bold(experiment["name"])
print " Id: %d" % experiment["jobman"]["id"]
for key in ["table", "gpu"]:
print " %s = %s" % (key, str(experiment[key]))
for name, cluster in experiment["clusters"].iteritems():
print " %s:" % name
for key, value in sorted(cluster.iteritems()):
print " %s = %s" % (key, value)
list_jobs(experiment["table"])
print