def build_text_file_table_cell(self, task):
'''Build a table cell containing links to the stdout/err output'''
bat = task.batch_array_task
text_path = RunBatch.batch_array_task_text_file_path(bat)
text_file = os.path.basename(text_path)
err_path = RunBatch.batch_array_task_err_file_path(bat)
err_file = os.path.basename(err_path)
with self.tag("td", style="text-align: left"):
for ft, path, filename in ((FT_TEXT_FILE, text_path, text_file),
(FT_ERR_FILE, err_path, err_file)):
with self.tag("div"):
if os.path.isfile(path):
with self.tag(
"a",
style="text-align: left",
href='ViewTextFile.py?%s=%d&%s=%d&%s=%s' %
(BATCH_ARRAY_ID, bat.batch_array.batch_array_id,
TASK_ID, bat.task_id, FILE_TYPE, ft),
title=path):
self.text(filename)
else:
with self.tag("span",
style="text-align: left",
title='Text file not available'):
self.text(filename)
def build_text_file_table_cell(self, task):
'''Build a table cell containing links to the stdout/err output'''
bat = task.batch_array_task
text_path = RunBatch.batch_array_task_text_file_path(bat)
text_file = os.path.basename(text_path)
err_path = RunBatch.batch_array_task_err_file_path(bat)
err_file = os.path.basename(err_path)
with self.tag("td", style="text-align: left"):
for ft, path, filename in (
(FT_TEXT_FILE, text_path, text_file),
(FT_ERR_FILE, err_path, err_file)):
with self.tag("div"):
if os.path.isfile(path):
with self.tag(
"a", style="text-align: left",
href='ViewTextFile.py?%s=%d&%s=%d&%s=%s' %
(BATCH_ARRAY_ID, bat.batch_array.batch_array_id,
TASK_ID, bat.task_id, FILE_TYPE, ft),
title=path):
self.text(filename)
else:
with self.tag(
"span",
style="text-align: left",
title='Text file not available'):
self.text(filename)
def delete_run(my_batch, my_run):
if delete_action in (A_DELETE_ALL, A_DELETE_TEXT):
remove_if_exists(RunBatch.run_text_file_path(my_batch, my_run))
remove_if_exists(RunBatch.run_err_file_path(my_batch, my_run))
if delete_action in (A_DELETE_ALL, A_DELETE_OUTPUT):
remove_if_exists(RunBatch.run_out_file_path(my_batch, my_run))
def delete_run(my_batch, my_run):
if delete_action in (A_DELETE_ALL, A_DELETE_TEXT):
for bat in RunBatch.BPBatchArrayTask.select_by_run(my_run):
remove_if_exists(RunBatch.batch_array_task_text_file_path(bat))
remove_if_exists(RunBatch.batch_array_task_err_file_path(bat))
if delete_action in (A_DELETE_ALL, A_DELETE_OUTPUT):
remove_if_exists(RunBatch.run_out_file_path(my_batch, my_run))
def delete_run(my_batch, my_run):
if delete_action in (A_DELETE_ALL, A_DELETE_TEXT):
for bat in RunBatch.BPBatchArrayTask.select_by_run(my_run):
remove_if_exists(RunBatch.batch_array_task_text_file_path(bat))
remove_if_exists(RunBatch.batch_array_task_err_file_path(bat))
if delete_action in (A_DELETE_ALL, A_DELETE_OUTPUT):
remove_if_exists(RunBatch.run_out_file_path(my_batch, my_run))
def do_run_id(run_id):
my_run = RunBatch.BPRunBase.select(run_id)
my_batch = RunBatch.BPBatch()
my_batch.select(my_run.batch_id)
file_type = BATCHPROFILER_VARIABLES[FILE_TYPE]
if file_type == FT_TEXT_FILE:
show_file(RunBatch.run_text_file_path(my_batch, my_run))
elif file_type == FT_ERR_FILE:
show_file(RunBatch.run_err_file_path(my_batch, my_run))
elif file_type == FT_OUT_FILE:
download_attachment(
RunBatch.run_out_file(my_batch, my_run),
RunBatch.run_out_file_path(my_batch, my_run))
else:
show_help("Unknown file type: %s" % file_type)
def do_it(batch_array_id, task_id):
batch_array = RunBatch.BPBatchArray.select(batch_array_id)
bat = RunBatch.BPBatchArrayTask.select_by_batch_array_and_task_id(
batch_array, task_id)
if bat is None:
show_help()
return
run = bat.run
my_batch = run.batch
file_type = BATCHPROFILER_VARIABLES[FILE_TYPE]
if file_type == FT_TEXT_FILE:
show_file(RunBatch.batch_array_task_text_file_path(bat))
elif file_type == FT_ERR_FILE:
show_file(RunBatch.batch_array_task_err_file_path(bat))
elif file_type == FT_OUT_FILE:
download_attachment(RunBatch.run_out_file(my_batch, my_run),
RunBatch.run_out_file_path(my_batch, my_run))
else:
show_help("Unknown file type: %s" % file_type)
def handle_post():
batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID]
my_batch = RunBatch.BPBatch()
my_batch.select(batch_id)
txt_output_dir = RunBatch.text_file_directory(my_batch)
job_script_dir = RunBatch.script_file_directory(my_batch)
what_i_did = {}
maybe_chmod(txt_output_dir, 0777, what_i_did)
maybe_chmod(job_script_dir, 0777, what_i_did)
for run in my_batch.select_runs():
for path in [RunBatch.run_text_file_path(my_batch, run),
RunBatch.run_out_file_path(my_batch, run),
RunBatch.run_err_file_path(my_batch, run)]:
maybe_chmod(path, 0644, what_i_did)
print "Content-Type: application/json"
print
print json.dumps(what_i_did)
def run_sql_file(batch_id, sql_filename):
"""Use the mysql command line to run the given SQL script
batch_id - sql_file is associated with this batch
sql_path - path and file of the sql script
queue - run job on this queue
email - who to email when done
returns the RunBatch.BPJob
"""
cwd = os.path.dirname(__file__)
batch = RunBatch.BPBatch.select(batch_id)
run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_filename)
if run is None:
sql_path = os.path.join(RunBatch.batch_script_directory(batch),
sql_filename)
cmd = "%s -b %d -i %s"%(os.path.join(cwd, "sql_jobs.py"),
batch_id, sql_path)
run = RunBatch.BPSQLRun.create(batch, sql_filename, cmd)
return RunBatch.run(batch, [run], cwd=cwd)
def do_it(batch_array_id, task_id):
batch_array = RunBatch.BPBatchArray.select(batch_array_id)
bat = RunBatch.BPBatchArrayTask.select_by_batch_array_and_task_id(
batch_array, task_id)
if bat is None:
show_help()
return
run = bat.run
my_batch = run.batch
file_type = BATCHPROFILER_VARIABLES[FILE_TYPE]
if file_type == FT_TEXT_FILE:
show_file(RunBatch.batch_array_task_text_file_path(bat))
elif file_type == FT_ERR_FILE:
show_file(RunBatch.batch_array_task_err_file_path(bat))
elif file_type == FT_OUT_FILE:
download_attachment(
RunBatch.run_out_file(my_batch, my_run),
RunBatch.run_out_file_path(my_batch, my_run))
else:
show_help("Unknown file type: %s" % file_type)
def build_submit_run(self):
title = "Batch # %d resubmitted"%(self.my_batch.batch_id)
with self.tag("html"):
with self.tag("head"):
with self.tag("title"):
self.text(title)
with self.tag("style"):
self.doc.asis(StyleSheet.BATCHPROFILER_STYLE)
with self.tag("body"):
with self.tag("h1"):
self.text(title)
with self.tag("div"):
self.text("Your batch # %d has been resubmitted." %
self.my_batch.batch_id)
with self.tag("table", klass="run-table"):
with self.tag("tr"):
with self.tag("th"):
self.text("First")
with self.tag("th"):
self.text("Last")
with self.tag("th"):
self.text("Job #")
with self.tag("th"):
self.text("Task #")
kwds = {}
run_id = BATCHPROFILER_VARIABLES[RUN_ID]
submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN]
if submit_run == RunBatch.JS_ALL:
runs = self.my_batch.select_runs()
elif submit_run == RunBatch.JS_ONE:
runs = [RunBatch.BPRun.select(run_id)]
else:
if submit_run == RunBatch.JS_INCOMPLETE:
statuses = RunBatch.INCOMPLETE_STATUSES
else:
statuses = [submit_run]
tasks = self.my_batch.select_tasks_by_state(statuses)
runs = [task.batch_array_task.run for task in tasks]
tasks = RunBatch.run(self.my_batch, runs)
for task in tasks:
assert isinstance(task, RunBatch.BPJobTask)
run = task.batch_array_task.run
assert isinstance(run, RunBatch.BPRun)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(task.job.job_id))
with self.tag("td"):
self.text(str(task.batch_array_task.task_id))
def build_submit_run(self):
title = "Batch # %d resubmitted" % (self.my_batch.batch_id)
with self.tag("html"):
with self.tag("head"):
with self.tag("title"):
self.text(title)
with self.tag("style"):
self.doc.asis(StyleSheet.BATCHPROFILER_STYLE)
with self.tag("body"):
with self.tag("h1"):
self.text(title)
with self.tag("div"):
self.text("Your batch # %d has been resubmitted." %
self.my_batch.batch_id)
with self.tag("table", klass="run-table"):
with self.tag("tr"):
with self.tag("th"):
self.text("First")
with self.tag("th"):
self.text("Last")
with self.tag("th"):
self.text("Job #")
with self.tag("th"):
self.text("Task #")
kwds = {}
run_id = BATCHPROFILER_VARIABLES[RUN_ID]
submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN]
if submit_run == RunBatch.JS_ALL:
runs = self.my_batch.select_runs()
elif submit_run == RunBatch.JS_ONE:
runs = [RunBatch.BPRun.select(run_id)]
else:
if submit_run == RunBatch.JS_INCOMPLETE:
statuses = RunBatch.INCOMPLETE_STATUSES
else:
statuses = [submit_run]
tasks = self.my_batch.select_tasks_by_state(statuses)
runs = [task.batch_array_task.run for task in tasks]
tasks = RunBatch.run(self.my_batch, runs)
for task in tasks:
assert isinstance(task, RunBatch.BPJobTask)
run = task.batch_array_task.run
assert isinstance(run, RunBatch.BPRun)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(task.job.job_id))
with self.tag("td"):
self.text(str(task.batch_array_task.task_id))
def build_submit_run(self):
title = "Batch # %d resubmitted"%(self.my_batch.batch_id)
with self.tag("html"):
with self.tag("head"):
with self.tag("title"):
self.text(title)
with self.tag("style"):
self.doc.asis(StyleSheet.BATCHPROFILER_STYLE)
with self.tag("body"):
with self.tag("h1"):
self.text(title)
with self.tag("div"):
self.text("Your batch # %d has been resubmitted." %
self.my_batch.batch_id)
with self.tag("table", klass="run-table"):
with self.tag("tr"):
with self.tag("th"):
self.text("First")
with self.tag("th"):
self.text("Last")
with self.tag("th"):
self.text("Job #")
kwds = {}
run_id = BATCHPROFILER_VARIABLES[RUN_ID]
submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN]
if submit_run == RESUBMIT:
submit_run = None
if run_id is not None:
kwds["by_run"] = run_id
if submit_run == RunBatch.JS_INCOMPLETE:
kwds["by_status"] = RunBatch.INCOMPLETE_STATUSES
elif submit_run in RunBatch.INCOMPLETE_STATUSES:
kwds["by_status"] = [submit_run]
jobs = self.my_batch.select_jobs(**kwds)
for run, job, status in jobs:
assert isinstance(run, RunBatch.BPRun)
assert isinstance(job, RunBatch.BPJob)
if run.run_id == self.run_id or \
(submit_run == RunBatch.JS_INCOMPLETE and
status in RunBatch.INCOMPLETE_STATUSES) or \
status == submit_run or \
submit_run == RunBatch.JS_ALL:
new_job = RunBatch.run_one(self.my_batch, run)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(new_job.job_id))
def build_text_file_table_cell(self, run):
'''Build a table cell containing links to the stdout/err output'''
text_file = RunBatch.run_text_file(run)
text_path = RunBatch.run_text_file_path(self.my_batch, run)
err_file = RunBatch.run_err_file(run)
err_path = RunBatch.run_err_file_path(self.my_batch, run)
with self.tag("td", style="text-align: left"):
for ft, path, filename in (
(FT_TEXT_FILE, text_path, text_file),
(FT_ERR_FILE, err_path, err_file)):
with self.tag("div"):
if os.path.isfile(path):
with self.tag(
"a", style="text-align: left",
href='ViewTextFile.py?run_id=%d&%s=%s' %
(run.run_id, FILE_TYPE, ft),
title=path):
self.text(filename)
else:
with self.tag(
"span",
style="text-align: left",
title='Text file not available'):
self.text(filename)
def sql_file_job_and_status(batch_id, sql_file):
"""Return the latest job ID associated with the batch and sql path
batch_id - batch id associated with the submission
sql_path - path to the sql script submitted
returns latest job or None if not submitted
"""
batch = RunBatch.select(batch_id)
run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_file)
if run is None:
return None, None, None
result = RunBatch.BPJobTask.select_by_run(run)
if len(result) == 0:
return None, None, None
return run, result[0], RunBatch.BPJobTaskStatus.select_by_job_task(job_task)
def sql_file_job_and_status(batch_id, sql_file):
"""Return the latest job ID associated with the batch and sql path
batch_id - batch id associated with the submission
sql_path - path to the sql script submitted
returns latest job or None if not submitted
"""
batch = RunBatch.select(batch_id)
run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_file)
if run is None:
return None, None, None
result = RunBatch.BPJobTask.select_by_run(run)
if len(result) == 0:
return None, None, None
return run, result[0], RunBatch.BPJobTaskStatus.select_by_job_task(
job_task)
def handle_post():
batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID]
my_batch = RunBatch.BPBatch()
my_batch.select(batch_id)
txt_output_dir = RunBatch.text_file_directory(my_batch)
job_script_dir = RunBatch.script_file_directory(my_batch)
what_i_did = {}
maybe_chmod(txt_output_dir, 0777, what_i_did)
maybe_chmod(job_script_dir, 0777, what_i_did)
for run in my_batch.select_runs():
for path in [
RunBatch.run_text_file_path(my_batch, run),
RunBatch.run_out_file_path(my_batch, run),
RunBatch.run_err_file_path(my_batch, run)
]:
maybe_chmod(path, 0644, what_i_did)
print "Content-Type: application/json"
print
print json.dumps(what_i_did)
cgitb.enable()
import RunBatch
import cgi
import re
import os
import os.path
import subprocess
import sql_jobs
form = cgi.FieldStorage()
batch_id = int(form["batch_id"].value)
sql_script = form["sql_script"].value
output_file = form["output_file"].value
queue = (form.has_key("queue") and form["queue"].value) or None
my_batch = RunBatch.LoadBatch(batch_id)
re_load_line = re.compile(
"'(.+?)[0-9]+_[0-9]+_(Image|Object).CSV'\sREPLACE\sINTO\sTABLE\s([A-Za-z0-9_]+)\s"
)
re_ignore_line = re.compile("SHOW WARNINGS;")
table_lines = []
image_prefix = None
object_prefix = None
sql_script_file = open(my_batch["data_dir"] + os.sep + sql_script, "r")
in_table_defs = True
try:
for line in sql_script_file:
match = re_load_line.search(line)
if match:
in_table_defs = False
cgitb.enable()
import RunBatch
import StyleSheet
import cgi
import os
import os.path
def remove_if_exists(path):
if os.path.exists(path):
os.remove(path)
form = cgi.FieldStorage()
delete_action = form["delete_action"].value
if form.has_key("run_id"):
run_id = int(form["run_id"].value)
my_batch,my_run = RunBatch.LoadRun(run_id)
if ((delete_action.upper() == "ALL") or (delete_action.upper() == "TEXT")):
remove_if_exists(RunBatch.RunTextFilePath(my_batch, my_run))
if (delete_action.upper() == "ALL") or (delete_action.upper() == "OUTPUT"):
remove_if_exists(RunBatch.RunOutFilePath(my_batch, my_run))
if (delete_action.upper() == "ALL") or (delete_action.upper() == "DONE"):
remove_if_exists(RunBatch.RunDoneFilePath(my_batch, my_run))
elif form.has_key("batch_id"):
batch_id = int(form["batch_id"].value)
my_batch = RunBatch.LoadBatch(batch_id)
for my_run in my_batch["runs"]:
if ((delete_action.upper() == "ALL") or (delete_action.upper() == "TEXT")):
remove_if_exists(RunBatch.RunTextFilePath(my_batch, my_run))
def build_database_scripts(self):
sql_files = []
kill_db_action = 'confirm("Are you sure you want to kill the database upload?")'
upload_action = 'confirm("Are you sure you want to upload to the database using %s?")'
for filename in os.listdir(self.my_batch.data_dir):
if filename.upper().endswith(".SQL"):
sql_files.append(filename)
if len(sql_files) == 0:
return
with self.tag("div", style='clear:both; padding-top:10px'):
with self.tag("h2"):
self.text("Database scripts")
with self.tag("table", klass="run_table"):
with self.tag("tr"):
for caption in ("Script file", "Action", "Last job id",
"Status", "Output"):
with self.tag("th"):
self.text(caption)
for filename in sql_files:
if filename.startswith('batch_'):
continue
run, job, status = sql_jobs.sql_file_job_and_status(
self.my_batch.batch_id,
RunBatch.batch_script_file(filename))
with self.tag("tr"):
with self.tag("td"):
self.text(filename)
with self.tag("td", style="text-align:left"):
run_button = True
output_file = filename[:-3]+'out'
output_path = os.path.join(
self.my_batch.data_dir,output_file)
if job is not None and status in (
RunBatch.JS_RUNNING, RunBatch.JS_SUBMITTED):
#
# A kill button for jobs that are killable
with self.tag("div"):
with self.tag("form",
action="KillJobs.py",
method="POST",
target="KillJob"):
self.doc.input(type='hidden',
name=JOB_ID,
value=str(job.job_id))
self.doc.stag(
"input",
type='submit', value="Kill",
name="Kill_db_button",
onclick=kill_db_action)
with self.tag("div"):
with self.tag("form",
action = "UploadToDatabase.py",
method = "POST"):
self.doc.input(
type='hidden',
name=SQL_SCRIPT,
value=filename)
self.doc.input(type='hidden',
name=OUTPUT_FILE,
value = output_path)
self.doc.input(type='hidden',
name=BATCH_ID)
with self.tag(
"span", style='white-space:nowrap'):
self.doc.stag(
"input",
type="submit",
value="Run",
onclick=upload_action % filename)
self.doc.asis(" Queue:")
with self.doc.select(name=QUEUE):
for queue_name in\
bputilities.get_queues():
with self.doc.option(
value=queue_name):
self.text(queue_name)
if job is None:
with self.tag("td", colspan="4"):
self.text("not run")
else:
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
if status != RunBatch.JS_DONE:
self.text(status)
else:
run_time = RunBatch.GetCPUTime(
self.my_batch, run)
self.text("Complete(%.2f sec)" % run_time)
self.build_text_file_table_cell(run)
def build_job_table_body(self):
with self.tag("tbody"):
for task_status in self.tasks:
assert isinstance(task_status, RunBatch.BPJobTaskStatus)
task = task_status.job_task
status = task_status.status
assert isinstance(task, RunBatch.BPJobTask)
run = task.batch_array_task.run
assert isinstance(run, RunBatch.BPRun)
job = task.job
assert isinstance(job, RunBatch.BPJob)
out_file = RunBatch.run_out_file(self.my_batch, run)
out_path = RunBatch.run_out_file_path(self.my_batch, run)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
self.text(str(task.batch_array_task.task_id))
if status == RunBatch.JS_DONE:
with self.tag("td"):
cpu = task.get_runtime()
if cpu is not None:
with self.tag("div", **{
"class":"success_message"}):
self.text("Complete (%.2f sec)" % cpu)
else:
with self.tag("div", **{
"class":"error_message"}):
self.text("Error: see log")
with self.tag("div"):
self.build_resubmit_run_form(run)
else:
with self.tag("td", style='color:red'):
with self.tag("div"):
self.text(status.lower().capitalize())
with self.tag("div"):
self.build_resubmit_run_form(run)
self.build_text_file_table_cell(task)
if self.my_batch.wants_measurements_file:
with self.tag("td"):
if os.path.isfile(out_path):
with self.tag(
"a",
href='ViewTextFile.py?run_id=%d&%s=%s' %
(run.run_id, FILE_TYPE, FT_OUT_FILE),
title=out_path):
self.text(out_file)
else:
with self.tag("span",
title='Output file not available'):
self.text(out_file)
#
# This cell contains the form that deletes things
#
with self.tag("td"):
with self.tag("form", action='DeleteFile.py', method='POST'):
self.doc.input(
type="hidden", name=RUN_ID,
value = str(run.run_id))
text_file = os.path.basename(
RunBatch.batch_array_task_text_file_path(
task.batch_array_task))
for action, filename in (
(A_DELETE_TEXT, text_file),
(A_DELETE_OUTPUT, RunBatch.run_out_file(self.my_batch, run)),
(A_DELETE_ALL, "all files for this run")):
self.doc.stag(
"input",
type="submit", name=K_DELETE_ACTION,
value=action,
title='Delete file %s' % filename,
onclick='return confirm("Do you really want'
'to delete %s?")' % filename)
def build_database_scripts(self):
sql_files = []
kill_db_action = 'confirm("Are you sure you want to kill the database upload?")'
upload_action = 'confirm("Are you sure you want to upload to the database using %s?")'
for filename in os.listdir(self.my_batch.data_dir):
if filename.upper().endswith(".SQL"):
sql_files.append(filename)
if len(sql_files) == 0:
return
with self.tag("div", style='clear:both; padding-top:10px'):
with self.tag("h2"):
self.text("Database scripts")
with self.tag("table", klass="run_table"):
with self.tag("tr"):
for caption in ("Script file", "Action", "Last job id",
"Status", "Output"):
with self.tag("th"):
self.text(caption)
for filename in sql_files:
if filename.startswith('batch_'):
continue
run, task, task_status = sql_jobs.sql_file_job_and_status(
self.my_batch.batch_id,
RunBatch.batch_script_file(filename))
status = None if task_status is None else task_status.status
with self.tag("tr"):
with self.tag("td"):
self.text(filename)
with self.tag("td", style="text-align:left"):
run_button = True
output_file = filename[:-3] + 'out'
output_path = os.path.join(self.my_batch.data_dir,
output_file)
if task is not None and task_status.status in (
RunBatch.JS_RUNNING,
RunBatch.JS_SUBMITTED):
#
# A kill button for jobs that are killable
with self.tag("div"):
with self.tag("form",
action="KillJobs.py",
method="POST",
target="KillJob"):
self.doc.input(type='hidden',
name=JOB_ID,
value=str(job.job_id))
self.doc.stag("input",
type='submit',
value="Kill",
name="Kill_db_button",
onclick=kill_db_action)
with self.tag("div"):
with self.tag("form",
action="UploadToDatabase.py",
method="POST"):
self.doc.input(type='hidden',
name=SQL_SCRIPT,
value=filename)
self.doc.input(type='hidden',
name=OUTPUT_FILE,
value=output_path)
self.doc.input(type='hidden',
name=BATCH_ID)
with self.tag("span",
style='white-space:nowrap'):
self.doc.stag("input",
type="submit",
value="Run",
onclick=upload_action %
filename)
self.doc.asis(" Queue:")
with self.doc.select(name=QUEUE):
for queue_name in\
bputilities.get_queues():
with self.doc.option(
value=queue_name):
self.text(queue_name)
if task is None:
with self.tag("td", colspan="4"):
self.text("not run")
else:
with self.tag("td"):
self.text(str(task.job.job_id))
with self.tag("td"):
if status != RunBatch.JS_DONE:
self.text(status)
else:
run_time = \
task_status.created - task.job.created
self.text("Complete(%.2f sec)" %
run_time.total_seconds())
self.build_text_file_table_cell(task)
re_file_name = re.compile("^(.+?)[0-9]+_[0-9]+_(Image|Object).CSV$")
image_files = []
object_files = []
for file_name in os.listdir(my_batch.data_dir):
match = re_file_name.search(file_name)
if match:
if (image_prefix and match.groups(1)[0] == image_prefix
and match.groups(1)[1] == 'Image'):
image_files.append(file_name)
elif (object_prefix and
match.groups(1)[0] == object_prefix and
match.groups(1)[1] == 'Object'):
object_files.append(file_name)
batch_script_file = RunBatch.batch_script_file(sql_script)
batch_script_dir = RunBatch.batch_script_directory(my_batch)
if not os.path.isdir(batch_script_dir):
os.makedirs(batch_script_dir)
batch_script_path = RunBatch.batch_script_path(my_batch, sql_script)
sql_script_file = open(batch_script_path, "w")
try:
sql_script_file.writelines(table_lines)
for file_name in image_files:
path_name = os.path.join(my_batch.data_dir, file_name)
sql_script_file.write("""SELECT 'Loading %(file_name)s into %(image_table)s';"""%(globals()))
sql_script_file.write("""LOAD DATA LOCAL INFILE '%(path_name)s' REPLACE INTO TABLE %(image_table)s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"';
SHOW WARNINGS;
"""%(globals()))
for file_name in object_files:
path_name = os.path.join(my_batch.data_dir, file_name)
import cgi
import os, sys
import bputilities
from bpformdata import *
import RunBatch
from cellprofiler.utilities.hdf5_dict import HDF5Dict
M_USER_PIPELINE = "Pipeline_UserPipeline"
M_PIPELINE = "Pipeline_Pipeline"
F_BATCH_DATA_H5 = "Batch_data.h5"
EXPERIMENT = "Experiment"
with bputilities.CellProfilerContext():
batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID]
my_batch = RunBatch.BPBatch.select(batch_id)
path = RunBatch.batch_data_file_path(my_batch)
h = HDF5Dict(path, mode="r")
if M_USER_PIPELINE in h.second_level_names(EXPERIMENT):
feature = M_USER_PIPELINE
else:
feature = M_PIPELINE
pipeline_text = h[EXPERIMENT, feature, 0][0].decode("unicode_escape")
h.close()
del h
if isinstance(pipeline_text, unicode):
pipeline_text = pipeline_text.encode("us-ascii")
print "Content-Type: text/plain"
print "Content-Length: %d" % len(pipeline_text)
print "Content-Disposition: attachment; filename=\"%d.cppipe\"" % batch_id
print
sys.stdout.write(pipeline_text)
#
import cgitb
cgitb.enable()
import RunBatch
import sql_jobs
import StyleSheet
import cgi
import os
import os.path
print "Content-Type: text/html"
print
form = cgi.FieldStorage()
batch_id = int(form["batch_id"].value)
my_batch = RunBatch.LoadBatch(batch_id)
jobs_by_state = {}
job_ids = [run["job_id"] for run in my_batch["runs"]]
job_dictionary = RunBatch.GetJobStatus(job_ids)
for run in my_batch["runs"]:
stat = "Unknown"
if os.path.isfile(RunBatch.RunDoneFilePath(my_batch, run)):
stat = "Complete"
elif run["job_id"] == None:
pass
else:
job_status = job_dictionary[run["job_id"]]
if job_status and job_status.has_key("STAT"):
stat = job_status["STAT"]
"""
#
# View the text file that is produced by bsub during batch processing
#
import cgitb
cgitb.enable()
import RunBatch
import cgi
import os
import os.path
import stat
if cgi.FieldStorage().has_key("run_id"):
run_id = int(cgi.FieldStorage()["run_id"].value)
my_batch, my_run = RunBatch.LoadRun(run_id)
text_file_path = RunBatch.RunTextFilePath(my_batch, my_run)
else:
text_file_path = cgi.FieldStorage()["file_name"].value
#
# This is a temporary work-around because files get created
# with the wrong permissions
#
if (os.stat(text_file_path)[0] & stat.S_IREAD) == 0:
os.chmod(text_file_path, 0644)
text_file = open(text_file_path, "r")
print "Content-Type: text/plain"
print
print text_file.read()
text_file.close()
import os
import subprocess
import sys
job_id = BATCHPROFILER_VARIABLES[JOB_ID]
task_id = BATCHPROFILER_VARIABLES[TASK_ID]
batch_id = BATCHPROFILER_VARIABLES[BATCH_ID]
if job_id is not None:
job = RunBatch.BPJob.select_by_job_id(job_id)
if job is None:
if task_id is None:
bputilities.kill_job(job_id)
else:
bputilities.kill_tasks(job_id, [task_id])
elif task_id is None:
RunBatch.kill_job(job)
else:
task = RunBatch.BPJobTask.select_by_task_id(job, task_id)
RunBatch.kill_task(task)
print"""
<html><head><title>Job %(job_id)d killed</title></head>
<body>Job %(job_id)d killed
</body>
</html>
"""%(globals())
elif batch_id is not None:
RunBatch.kill_batch(batch_id)
url = "ViewBatch.py?batch_id=%d"%(batch_id)
print "<html><head>"
print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
import os.path
import subprocess
import sql_jobs
#
# TODO: Move the logic that collects the load statements into sql_jobs
#
# TODO: use yattag to build the HTML
#
batch_id = BATCHPROFILER_VARIABLES[BATCH_ID]
sql_script = BATCHPROFILER_VARIABLES[SQL_SCRIPT]
output_file = BATCHPROFILER_VARIABLES[OUTPUT_FILE]
queue = BATCHPROFILER_VARIABLES[QUEUE]
my_batch = RunBatch.BPBatch()
my_batch.select(batch_id)
re_load_line = re.compile(
"'(.+?)[0-9]+_[0-9]+_(Image|Object).CSV'\sREPLACE\sINTO\sTABLE\s([A-Za-z0-9_]+)\s"
)
re_ignore_line = re.compile("SHOW WARNINGS;")
table_lines = []
image_prefix = None
object_prefix = None
sql_script_file = open(my_batch.data_dir + os.sep + sql_script, "r")
in_table_defs = True
try:
for line in sql_script_file:
match = re_load_line.search(line)
if match:
cgitb.enable()
import RunBatch
import bputilities
from bpformdata import *
import cgi
import subprocess
import sys
job_id = BATCHPROFILER_VARIABLES[JOB_ID]
batch_id = BATCHPROFILER_VARIABLES[BATCH_ID]
if job_id is not None:
job = RunBatch.BPJob.select(job_id)
if job is None:
bputilities.kill_job(job_id)
else:
RunBatch.kill_job(job)
print"""
<html><head><title>Job %(job_id)d killed</title></head>
<body>Job %(job_id)d killed
</body>
</html>
"""%(globals())
elif batch_id is not None:
RunBatch.kill_batch(batch_id)
url = "ViewBatch.py?batch_id=%d"%(batch_id)
print "<html><head>"
print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
print "</head>"
print "<body>This page should be redirected to <a href='%(url)s'/>%(url)s</a></body>"%(globals())
print "</html>"
Please see the AUTHORS file for credits.
Website: http://www.cellprofiler.org
"""
#
# Kill all jobs in a batch
#
import cgitb
cgitb.enable()
import RunBatch
import cgi
import os
form = cgi.FieldStorage()
batch_id = int(form["batch_id"].value)
my_batch = RunBatch.LoadBatch(batch_id)
status_dir = "%(data_dir)s/status/" % (my_batch)
txt_output_dir = "%(data_dir)s/txt_output/" % (my_batch)
if os.path.exists(status_dir):
os.chmod(status_dir, 0777)
if os.path.exists(txt_output_dir):
os.chmod(txt_output_dir, 0777)
for run in my_batch["runs"]:
for path in [
RunBatch.RunTextFilePath(my_batch, run),
RunBatch.RunDoneFilePath(my_batch, run),
RunBatch.RunOutFilePath(my_batch, run)
]:
if os.path.exists(path):
os.chmod(path, 0644)
def build_submit_batch(self):
'''Build the webpage for handling a submitted batch
Also do the batch submission
'''
batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE])
if self.has_groups:
first_last = np.hstack(
[[True],
self.group_numbers[1:] != self.group_numbers[:-1],
[True]])
gn = self.group_numbers[first_last[:-1]]
first = self.image_numbers[first_last[:-1]]
last = self.image_numbers[first_last[1:]]
else:
first = self.image_numbers[::batch_size]
last = self.image_numbers[(batch_size-1)::batch_size]
if len(last) < len(first):
last = np.hstack([last, self.image_numbers[-1]])
batch = RunBatch.BPBatch()
runs = [(f, l) for f, l in zip(first, last)]
#
# Put it in the database
#
write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0
batch.create(
email = BATCHPROFILER_DEFAULTS[EMAIL],
data_dir = BATCHPROFILER_DEFAULTS[DATA_DIR],
queue = BATCHPROFILER_DEFAULTS[QUEUE],
batch_size = BATCHPROFILER_DEFAULTS[BATCH_SIZE],
write_data = write_data,
timeout = 60,
cpcluster = self.cpcluster,
project = BATCHPROFILER_DEFAULTS[PROJECT],
memory_limit = BATCHPROFILER_DEFAULTS[MEMORY_LIMIT],
priority = BATCHPROFILER_DEFAULTS[PRIORITY],
runs = runs)
RunBatch.run_all(batch.batch_id)
vb_url = "ViewBatch.py?batch_id=%d" % batch.batch_id
self.send_batch_submission_email(batch, vb_url)
job_list = batch.select_jobs()
with self.tag("head"):
with self.tag("title"):
self.text("Batch #%d" % batch.batch_id)
with self.tag("style", type="text/css"):
self.doc.asis("""
table {
border-spacing: 0px;
border-collapse: collapse;
}
td {
text-align: left;
vertical-align: baseline;
padding: 0.1em 0.5em;
border: 1px solid #666666;
}""")
with self.tag("body"):
with self.tag("h1"):
self.text("Results for batch #")
with self.tag("a", href=vb_url):
self.text(str(batch.batch_id))
with self.tag("table"):
with self.tag("thead"):
with self.tag("tr"):
with self.tag("th"):
self.text("First image set")
with self.tag("th"):
self.text("Last image set")
with self.tag("th"):
self.text("job #")
for run, job, status in job_list:
assert isinstance(run, RunBatch.BPRun)
assert isinstance(job, RunBatch.BPJob)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
#
print "Content-Type: text/html"
print
import cgitb
cgitb.enable()
import RunBatch
import cgi
import subprocess
import sys
form = cgi.FieldStorage()
if form.has_key("job_id"):
import subprocess
job_id = int(form["job_id"].value)
run = {"job_id":job_id}
RunBatch.KillOne(run)
print"""
<html><head><title>Job %(job_id)d killed</title></head>
<body>Job %(job_id)d killed
</body>
</html>
"""%(globals())
elif form.has_key("batch_id"):
batch_id = int(form["batch_id"].value)
my_batch = RunBatch.LoadBatch(batch_id)
for run in my_batch["runs"]:
RunBatch.KillOne(run)
url = "ViewBatch.py?batch_id=%(batch_id)d"%(my_batch)
print "<html><head>"
print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
def build_submit_batch(self):
'''Build the webpage for handling a submitted batch
Also do the batch submission
'''
batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE])
if self.has_groups:
first_last = np.hstack(
[[True], self.group_numbers[1:] != self.group_numbers[:-1],
[True]])
gn = self.group_numbers[first_last[:-1]]
first = self.image_numbers[first_last[:-1]]
last = self.image_numbers[first_last[1:]]
else:
first = self.image_numbers[::batch_size]
last = self.image_numbers[(batch_size - 1)::batch_size]
if len(last) < len(first):
last = np.hstack([last, self.image_numbers[-1]])
runs = [(f, l) for f, l in zip(first, last)]
#
# Put it in the database
#
write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0
with RunBatch.bpcursor() as cursor:
batch = RunBatch.BPBatch.create(
cursor,
email=BATCHPROFILER_DEFAULTS[EMAIL],
data_dir=BATCHPROFILER_DEFAULTS[DATA_DIR],
queue=BATCHPROFILER_DEFAULTS[QUEUE],
batch_size=BATCHPROFILER_DEFAULTS[BATCH_SIZE],
write_data=write_data,
timeout=60,
cpcluster=self.cpcluster,
project=BATCHPROFILER_DEFAULTS[PROJECT],
memory_limit=BATCHPROFILER_DEFAULTS[MEMORY_LIMIT],
priority=BATCHPROFILER_DEFAULTS[PRIORITY])
bpruns = []
for bstart, bend in runs:
cmd = RunBatch.cellprofiler_command(batch, bstart, bend)
run = RunBatch.BPRun.create(cursor, batch, bstart, bend, cmd)
bpruns.append(run)
RunBatch.run(batch, bpruns)
vb_url = BATCHPROFILER_DEFAULTS[URL] + \
"/ViewBatch.py?batch_id=%d" % batch.batch_id
self.send_batch_submission_email(batch, vb_url)
task_list = batch.select_tasks()
with self.tag("head"):
with self.tag("title"):
self.text("Batch #%d" % batch.batch_id)
with self.tag("style", type="text/css"):
self.doc.asis("""
table {
border-spacing: 0px;
border-collapse: collapse;
}
td {
text-align: left;
vertical-align: baseline;
padding: 0.1em 0.5em;
border: 1px solid #666666;
}""")
with self.tag("body"):
with self.tag("h1"):
self.text("Results for batch #")
with self.tag("a", href=vb_url):
self.text(str(batch.batch_id))
with self.tag("table"):
with self.tag("thead"):
with self.tag("tr"):
with self.tag("th"):
self.text("First image set")
with self.tag("th"):
self.text("Last image set")
with self.tag("th"):
self.text("job #")
with self.tag("th"):
self.text("task #")
for task_status in task_list:
assert isinstance(task_status, RunBatch.BPJobTaskStatus)
task = task_status.job_task
assert isinstance(task, RunBatch.BPJobTask)
job = task.job
assert isinstance(job, RunBatch.BPJob)
bat = task.batch_array_task
assert isinstance(bat, RunBatch.BPBatchArrayTask)
task_id = bat.task_id
run = bat.run
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
self.text(str(task_id))
"""
print "Content-Type: text/html"
print
import cgitb
cgitb.enable()
import RunBatch
import cgi
import subprocess
import sys
form = cgi.FieldStorage()
if form.has_key("job_id"):
import subprocess
job_id = int(form["job_id"].value)
run = {"job_id":job_id}
RunBatch.KillOne(run)
print"""
<html><head><title>Job %(job_id)d killed</title></head>
<body>Job %(job_id)d killed
</body>
</html>
"""%(globals())
elif form.has_key("batch_id"):
batch_id = int(form["batch_id"].value)
RunBatch.KillBatch(batch_id)
url = "ViewBatch.py?batch_id=%d"%(batch_id)
print "<html><head>"
print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
print "</head>"
print "<body>This page should be redirected to <a href='%(url)s'/>%(url)s</a></body>"%(globals())
def build_job_table_body(self):
with self.tag("tbody"):
for task_status in self.tasks:
assert isinstance(task_status, RunBatch.BPJobTaskStatus)
task = task_status.job_task
status = task_status.status
assert isinstance(task, RunBatch.BPJobTask)
run = task.batch_array_task.run
assert isinstance(run, RunBatch.BPRun)
job = task.job
assert isinstance(job, RunBatch.BPJob)
out_file = RunBatch.run_out_file(self.my_batch, run)
out_path = RunBatch.run_out_file_path(self.my_batch, run)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
self.text(str(task.batch_array_task.task_id))
if status == RunBatch.JS_DONE:
with self.tag("td"):
cpu = task_status.created - job.created
self.text("Complete (%.2f sec)" %
(cpu.total_seconds()))
else:
with self.tag("td", style='color:red'):
with self.tag("div"):
self.text(status.lower().capitalize())
with self.tag("div"):
with self.tag("form",
action="ViewBatch.py",
method="POST",
target="ResubmitWindow"):
self.doc.input(type="hidden",
name=BATCH_ID)
self.doc.input(type="hidden",
name=RUN_ID,
value=str(run.run_id))
self.doc.input(type="hidden",
name=SUBMIT_RUN,
value=RunBatch.JS_ONE)
self.doc.stag("input",
type='submit',
name="Resubmit_Button",
value=RESUBMIT)
self.build_text_file_table_cell(task)
if self.my_batch.wants_measurements_file:
with self.tag("td"):
if os.path.isfile(out_path):
with self.tag(
"a",
href='ViewTextFile.py?run_id=%d&%s=%s'
% (run.run_id, FILE_TYPE, FT_OUT_FILE),
title=out_path):
self.text(out_file)
else:
with self.tag(
"span",
title='Output file not available'):
self.text(out_file)
#
# This cell contains the form that deletes things
#
with self.tag("td"):
with self.tag("form",
action='DeleteFile.py',
method='POST'):
self.doc.input(type="hidden",
name=RUN_ID,
value=str(run.run_id))
text_file = os.path.basename(
RunBatch.batch_array_task_text_file_path(
task.batch_array_task))
for action, filename in (
(A_DELETE_TEXT, text_file),
(A_DELETE_OUTPUT,
RunBatch.run_out_file(self.my_batch, run)),
(A_DELETE_ALL, "all files for this run")):
self.doc.stag(
"input",
type="submit",
name=K_DELETE_ACTION,
value=action,
title='Delete file %s' % filename,
onclick='return confirm("Do you really want'
'to delete %s?")' % filename)
import os
import subprocess
import sys
job_id = BATCHPROFILER_VARIABLES[JOB_ID]
task_id = BATCHPROFILER_VARIABLES[TASK_ID]
batch_id = BATCHPROFILER_VARIABLES[BATCH_ID]
if job_id is not None:
job = RunBatch.BPJob.select_by_job_id(job_id)
if job is None:
if task_id is None:
bputilities.kill_job(job_id)
else:
bputilities.kill_tasks(job_id, [task_id])
elif task_id is None:
RunBatch.kill_job(job)
else:
task = RunBatch.BPJobTask.select_by_task_id(job, task_id)
RunBatch.kill_task(task)
print """
<html><head><title>Job %(job_id)d killed</title></head>
<body>Job %(job_id)d killed
</body>
</html>
""" % (
globals()
)
elif batch_id is not None:
RunBatch.kill_batch(batch_id)
url = "ViewBatch.py?batch_id=%d" % (batch_id)
if form_data.has_key("batch_size"):
batch_size = int(form_data["batch_size"].value)
for i in image_numbers[::batch_size]:
start = i
end = min(start + batch_size - 1, max(image_numbers))
status_file_name = ("%s/status/Batch_%d_to_%d_DONE.mat" %
(batch["data_dir"], start, end))
run = {
"start": start,
"end": end,
"group": None,
"status_file_name": status_file_name
}
batch["runs"].append(run)
batch_id = RunBatch.CreateBatchRun(batch)
email_text = []
email_text.append("<html>")
email_text.append("<head><title>Batch # %d</title>" % (batch_id))
email_text.append("<style type='text/css'>")
email_text.append("""
table {
border-spacing: 0px;
border-collapse: collapse;
}
td {
text-align: left;
vertical-align: baseline;
padding: 0.1em 0.5em;
border: 1px solid #666666;
def build_job_table_body(self):
with self.tag("tbody"):
for run, job, status in self.jobs:
assert isinstance(run, RunBatch.BPRun)
assert isinstance(job, RunBatch.BPJob)
out_file = RunBatch.run_out_file(self.my_batch, run)
out_path = RunBatch.run_out_file_path(self.my_batch, run)
with self.tag("tr"):
with self.tag("td"):
self.text(str(run.bstart))
with self.tag("td"):
self.text(str(run.bend))
with self.tag("td"):
self.text(str(job.job_id))
if status == RunBatch.JS_DONE:
with self.tag("td"):
cpu = RunBatch.GetCPUTime(self.my_batch, run)
self.text("Complete (%.2f sec)"%(cpu))
else:
with self.tag("td", style='color:red'):
with self.tag("div"):
self.text(status.lower().capitalize())
with self.tag("div"):
with self.tag("form",
action="ViewBatch.py",
method="POST",
target="ResubmitWindow"):
self.doc.input(
type="hidden", name=BATCH_ID)
self.doc.input(
type="hidden",
name=RUN_ID,
value=str(run.run_id))
self.doc.stag(
"input",
type='submit',
name=SUBMIT_RUN,
value=RESUBMIT)
self.build_text_file_table_cell(run)
if self.my_batch.wants_measurements_file:
with self.tag("td"):
if os.path.isfile(out_path):
with self.tag(
"a",
href='ViewTextFile.py?run_id=%d&%s=%s' %
(run.run_id, FILE_TYPE, FT_OUT_FILE),
title=out_path):
self.text(out_file)
else:
with self.tag("span",
title='Output file not available'):
self.text(out_file)
#
# This cell contains the form that deletes things
#
with self.tag("td"):
with self.tag("form", action='DeleteFile.py', method='POST'):
self.doc.input(
type="hidden", name=RUN_ID,
value = str(run.run_id))
for action, filename in (
(A_DELETE_TEXT, RunBatch.run_text_file(run)),
(A_DELETE_OUTPUT, RunBatch.run_out_file(self.my_batch, run)),
(A_DELETE_ALL, "all files for this run")):
self.doc.stag(
"input",
type="submit", name=K_DELETE_ACTION,
value=action,
title='Delete file %s' % filename,
onclick='return confirm("Do you really want'
'to delete %s?")' % filename)
