def build_submit_run(self):
title = "Batch # %d resubmitted" % (self.my_batch.batch_id)
with self.tag("html"):
with self.tag("head"):
with self.tag("title"):
self.text(title)
with self.tag("style"):
self.doc.asis(StyleSheet.BATCHPROFILER_STYLE)
with self.tag("body"):
with self.tag("h1"):
self.text(title)
with self.tag("div"):
self.text("Your batch # %d has been resubmitted." %
self.my_batch.batch_id)
with self.tag("table", klass="run-table"):
with self.tag("tr"):
with self.tag("th"):
self.text("First")
with self.tag("th"):
self.text("Last")
with self.tag("th"):
self.text("Job #")
with self.tag("th"):
self.text("Task #")
kwds = {}
run_id = BATCHPROFILER_VARIABLES[RUN_ID]
submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN]
if submit_run == RunBatch.JS_ALL:
runs = self.my_batch.select_runs()
elif submit_run == RunBatch.JS_ONE:
runs = [RunBatch.BPRun.select(run_id)]
else:
if submit_run == RunBatch.JS_INCOMPLETE:
statuses = RunBatch.INCOMPLETE_STATUSES
else:
statuses = [submit_run]
tasks = self.my_batch.select_tasks_by_state(statuses)
runs = [task.batch_array_task.run for task in tasks]
tasks = RunBatch.run(self.my_batch, runs)
for task in tasks:
assert isinstance(task, RunBatch.BPJobTask)
run = task.batch_array_task.run
assert isinstance(run, RunBatch.BPRun)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(task.job.job_id))
with self.tag("td"):
self.text(str(task.batch_array_task.task_id))
def build_submit_run(self):
title = "Batch # %d resubmitted"%(self.my_batch.batch_id)
with self.tag("html"):
with self.tag("head"):
with self.tag("title"):
self.text(title)
with self.tag("style"):
self.doc.asis(StyleSheet.BATCHPROFILER_STYLE)
with self.tag("body"):
with self.tag("h1"):
self.text(title)
with self.tag("div"):
self.text("Your batch # %d has been resubmitted." %
self.my_batch.batch_id)
with self.tag("table", klass="run-table"):
with self.tag("tr"):
with self.tag("th"):
self.text("First")
with self.tag("th"):
self.text("Last")
with self.tag("th"):
self.text("Job #")
with self.tag("th"):
self.text("Task #")
kwds = {}
run_id = BATCHPROFILER_VARIABLES[RUN_ID]
submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN]
if submit_run == RunBatch.JS_ALL:
runs = self.my_batch.select_runs()
elif submit_run == RunBatch.JS_ONE:
runs = [RunBatch.BPRun.select(run_id)]
else:
if submit_run == RunBatch.JS_INCOMPLETE:
statuses = RunBatch.INCOMPLETE_STATUSES
else:
statuses = [submit_run]
tasks = self.my_batch.select_tasks_by_state(statuses)
runs = [task.batch_array_task.run for task in tasks]
tasks = RunBatch.run(self.my_batch, runs)
for task in tasks:
assert isinstance(task, RunBatch.BPJobTask)
run = task.batch_array_task.run
assert isinstance(run, RunBatch.BPRun)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(task.job.job_id))
with self.tag("td"):
self.text(str(task.batch_array_task.task_id))
def run_sql_file(batch_id, sql_filename):
"""Use the mysql command line to run the given SQL script
batch_id - sql_file is associated with this batch
sql_path - path and file of the sql script
queue - run job on this queue
email - who to email when done
returns the RunBatch.BPJob
"""
cwd = os.path.dirname(__file__)
batch = RunBatch.BPBatch.select(batch_id)
run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_filename)
if run is None:
sql_path = os.path.join(RunBatch.batch_script_directory(batch),
sql_filename)
cmd = "%s -b %d -i %s"%(os.path.join(cwd, "sql_jobs.py"),
batch_id, sql_path)
run = RunBatch.BPSQLRun.create(batch, sql_filename, cmd)
return RunBatch.run(batch, [run], cwd=cwd)
def build_submit_batch(self):
'''Build the webpage for handling a submitted batch
Also do the batch submission
'''
batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE])
if self.has_groups:
first_last = np.hstack(
[[True], self.group_numbers[1:] != self.group_numbers[:-1],
[True]])
gn = self.group_numbers[first_last[:-1]]
first = self.image_numbers[first_last[:-1]]
last = self.image_numbers[first_last[1:]]
else:
first = self.image_numbers[::batch_size]
last = self.image_numbers[(batch_size - 1)::batch_size]
if len(last) < len(first):
last = np.hstack([last, self.image_numbers[-1]])
runs = [(f, l) for f, l in zip(first, last)]
#
# Put it in the database
#
write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0
with RunBatch.bpcursor() as cursor:
batch = RunBatch.BPBatch.create(
cursor,
email=BATCHPROFILER_DEFAULTS[EMAIL],
data_dir=BATCHPROFILER_DEFAULTS[DATA_DIR],
queue=BATCHPROFILER_DEFAULTS[QUEUE],
batch_size=BATCHPROFILER_DEFAULTS[BATCH_SIZE],
write_data=write_data,
timeout=60,
cpcluster=self.cpcluster,
project=BATCHPROFILER_DEFAULTS[PROJECT],
memory_limit=BATCHPROFILER_DEFAULTS[MEMORY_LIMIT],
priority=BATCHPROFILER_DEFAULTS[PRIORITY])
bpruns = []
for bstart, bend in runs:
cmd = RunBatch.cellprofiler_command(batch, bstart, bend)
run = RunBatch.BPRun.create(cursor, batch, bstart, bend, cmd)
bpruns.append(run)
RunBatch.run(batch, bpruns)
vb_url = BATCHPROFILER_DEFAULTS[URL] + \
"/ViewBatch.py?batch_id=%d" % batch.batch_id
self.send_batch_submission_email(batch, vb_url)
task_list = batch.select_tasks()
with self.tag("head"):
with self.tag("title"):
self.text("Batch #%d" % batch.batch_id)
with self.tag("style", type="text/css"):
self.doc.asis("""
table {
border-spacing: 0px;
border-collapse: collapse;
}
td {
text-align: left;
vertical-align: baseline;
padding: 0.1em 0.5em;
border: 1px solid #666666;
}""")
with self.tag("body"):
with self.tag("h1"):
self.text("Results for batch #")
with self.tag("a", href=vb_url):
self.text(str(batch.batch_id))
with self.tag("table"):
with self.tag("thead"):
with self.tag("tr"):
with self.tag("th"):
self.text("First image set")
with self.tag("th"):
self.text("Last image set")
with self.tag("th"):
self.text("job #")
with self.tag("th"):
self.text("task #")
for task_status in task_list:
assert isinstance(task_status, RunBatch.BPJobTaskStatus)
task = task_status.job_task
assert isinstance(task, RunBatch.BPJobTask)
job = task.job
assert isinstance(job, RunBatch.BPJob)
bat = task.batch_array_task
assert isinstance(bat, RunBatch.BPBatchArrayTask)
task_id = bat.task_id
run = bat.run
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
self.text(str(task_id))
def build_submit_batch(self):
'''Build the webpage for handling a submitted batch
Also do the batch submission
'''
batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE])
if self.has_groups:
first_last = np.hstack(
[[True],
self.group_numbers[1:] != self.group_numbers[:-1],
[True]])
gn = self.group_numbers[first_last[:-1]]
first = self.image_numbers[first_last[:-1]]
last = self.image_numbers[first_last[1:]]
else:
first = self.image_numbers[::batch_size]
last = self.image_numbers[(batch_size-1)::batch_size]
if len(last) < len(first):
last = np.hstack([last, self.image_numbers[-1]])
runs = [(f, l) for f, l in zip(first, last)]
#
# Put it in the database
#
write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0
with RunBatch.bpcursor() as cursor:
batch = RunBatch.BPBatch.create(
cursor,
email = BATCHPROFILER_DEFAULTS[EMAIL],
data_dir = BATCHPROFILER_DEFAULTS[DATA_DIR],
queue = BATCHPROFILER_DEFAULTS[QUEUE],
batch_size = BATCHPROFILER_DEFAULTS[BATCH_SIZE],
write_data = write_data,
timeout = 60,
cpcluster = self.cpcluster,
project = BATCHPROFILER_DEFAULTS[PROJECT],
memory_limit = BATCHPROFILER_DEFAULTS[MEMORY_LIMIT],
priority = BATCHPROFILER_DEFAULTS[PRIORITY])
bpruns = []
for bstart, bend in runs:
cmd = RunBatch.cellprofiler_command(batch, bstart, bend)
run = RunBatch.BPRun.create(cursor, batch, bstart, bend, cmd)
bpruns.append(run)
RunBatch.run(batch, bpruns)
vb_url = BATCHPROFILER_DEFAULTS[URL] + \
"/ViewBatch.py?batch_id=%d" % batch.batch_id
self.send_batch_submission_email(batch, vb_url)
task_list = batch.select_tasks()
with self.tag("head"):
with self.tag("title"):
self.text("Batch #%d" % batch.batch_id)
with self.tag("style", type="text/css"):
self.doc.asis("""
table {
border-spacing: 0px;
border-collapse: collapse;
}
td {
text-align: left;
vertical-align: baseline;
padding: 0.1em 0.5em;
border: 1px solid #666666;
}""")
with self.tag("body"):
with self.tag("h1"):
self.text("Results for batch #")
with self.tag("a", href=vb_url):
self.text(str(batch.batch_id))
with self.tag("table"):
with self.tag("thead"):
with self.tag("tr"):
with self.tag("th"):
self.text("First image set")
with self.tag("th"):
self.text("Last image set")
with self.tag("th"):
self.text("job #")
with self.tag("th"):
self.text("task #")
for task_status in task_list:
assert isinstance(task_status, RunBatch.BPJobTaskStatus)
task = task_status.job_task
assert isinstance(task, RunBatch.BPJobTask)
job = task.job
assert isinstance(job, RunBatch.BPJob)
bat = task.batch_array_task
assert isinstance(bat, RunBatch.BPBatchArrayTask)
task_id = bat.task_id
run = bat.run
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
self.text(str(task_id))
