def build_text_file_table_cell(self, task): '''Build a table cell containing links to the stdout/err output''' bat = task.batch_array_task text_path = RunBatch.batch_array_task_text_file_path(bat) text_file = os.path.basename(text_path) err_path = RunBatch.batch_array_task_err_file_path(bat) err_file = os.path.basename(err_path) with self.tag("td", style="text-align: left"): for ft, path, filename in ((FT_TEXT_FILE, text_path, text_file), (FT_ERR_FILE, err_path, err_file)): with self.tag("div"): if os.path.isfile(path): with self.tag( "a", style="text-align: left", href='ViewTextFile.py?%s=%d&%s=%d&%s=%s' % (BATCH_ARRAY_ID, bat.batch_array.batch_array_id, TASK_ID, bat.task_id, FILE_TYPE, ft), title=path): self.text(filename) else: with self.tag("span", style="text-align: left", title='Text file not available'): self.text(filename)
def build_text_file_table_cell(self, task): '''Build a table cell containing links to the stdout/err output''' bat = task.batch_array_task text_path = RunBatch.batch_array_task_text_file_path(bat) text_file = os.path.basename(text_path) err_path = RunBatch.batch_array_task_err_file_path(bat) err_file = os.path.basename(err_path) with self.tag("td", style="text-align: left"): for ft, path, filename in ( (FT_TEXT_FILE, text_path, text_file), (FT_ERR_FILE, err_path, err_file)): with self.tag("div"): if os.path.isfile(path): with self.tag( "a", style="text-align: left", href='ViewTextFile.py?%s=%d&%s=%d&%s=%s' % (BATCH_ARRAY_ID, bat.batch_array.batch_array_id, TASK_ID, bat.task_id, FILE_TYPE, ft), title=path): self.text(filename) else: with self.tag( "span", style="text-align: left", title='Text file not available'): self.text(filename)
def delete_run(my_batch, my_run): if delete_action in (A_DELETE_ALL, A_DELETE_TEXT): remove_if_exists(RunBatch.run_text_file_path(my_batch, my_run)) remove_if_exists(RunBatch.run_err_file_path(my_batch, my_run)) if delete_action in (A_DELETE_ALL, A_DELETE_OUTPUT): remove_if_exists(RunBatch.run_out_file_path(my_batch, my_run))
def delete_run(my_batch, my_run): if delete_action in (A_DELETE_ALL, A_DELETE_TEXT): for bat in RunBatch.BPBatchArrayTask.select_by_run(my_run): remove_if_exists(RunBatch.batch_array_task_text_file_path(bat)) remove_if_exists(RunBatch.batch_array_task_err_file_path(bat)) if delete_action in (A_DELETE_ALL, A_DELETE_OUTPUT): remove_if_exists(RunBatch.run_out_file_path(my_batch, my_run))
def do_run_id(run_id): my_run = RunBatch.BPRunBase.select(run_id) my_batch = RunBatch.BPBatch() my_batch.select(my_run.batch_id) file_type = BATCHPROFILER_VARIABLES[FILE_TYPE] if file_type == FT_TEXT_FILE: show_file(RunBatch.run_text_file_path(my_batch, my_run)) elif file_type == FT_ERR_FILE: show_file(RunBatch.run_err_file_path(my_batch, my_run)) elif file_type == FT_OUT_FILE: download_attachment( RunBatch.run_out_file(my_batch, my_run), RunBatch.run_out_file_path(my_batch, my_run)) else: show_help("Unknown file type: %s" % file_type)
def do_it(batch_array_id, task_id): batch_array = RunBatch.BPBatchArray.select(batch_array_id) bat = RunBatch.BPBatchArrayTask.select_by_batch_array_and_task_id( batch_array, task_id) if bat is None: show_help() return run = bat.run my_batch = run.batch file_type = BATCHPROFILER_VARIABLES[FILE_TYPE] if file_type == FT_TEXT_FILE: show_file(RunBatch.batch_array_task_text_file_path(bat)) elif file_type == FT_ERR_FILE: show_file(RunBatch.batch_array_task_err_file_path(bat)) elif file_type == FT_OUT_FILE: download_attachment(RunBatch.run_out_file(my_batch, my_run), RunBatch.run_out_file_path(my_batch, my_run)) else: show_help("Unknown file type: %s" % file_type)
def handle_post(): batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID] my_batch = RunBatch.BPBatch() my_batch.select(batch_id) txt_output_dir = RunBatch.text_file_directory(my_batch) job_script_dir = RunBatch.script_file_directory(my_batch) what_i_did = {} maybe_chmod(txt_output_dir, 0777, what_i_did) maybe_chmod(job_script_dir, 0777, what_i_did) for run in my_batch.select_runs(): for path in [RunBatch.run_text_file_path(my_batch, run), RunBatch.run_out_file_path(my_batch, run), RunBatch.run_err_file_path(my_batch, run)]: maybe_chmod(path, 0644, what_i_did) print "Content-Type: application/json" print print json.dumps(what_i_did)
def run_sql_file(batch_id, sql_filename): """Use the mysql command line to run the given SQL script batch_id - sql_file is associated with this batch sql_path - path and file of the sql script queue - run job on this queue email - who to email when done returns the RunBatch.BPJob """ cwd = os.path.dirname(__file__) batch = RunBatch.BPBatch.select(batch_id) run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_filename) if run is None: sql_path = os.path.join(RunBatch.batch_script_directory(batch), sql_filename) cmd = "%s -b %d -i %s"%(os.path.join(cwd, "sql_jobs.py"), batch_id, sql_path) run = RunBatch.BPSQLRun.create(batch, sql_filename, cmd) return RunBatch.run(batch, [run], cwd=cwd)
def do_it(batch_array_id, task_id): batch_array = RunBatch.BPBatchArray.select(batch_array_id) bat = RunBatch.BPBatchArrayTask.select_by_batch_array_and_task_id( batch_array, task_id) if bat is None: show_help() return run = bat.run my_batch = run.batch file_type = BATCHPROFILER_VARIABLES[FILE_TYPE] if file_type == FT_TEXT_FILE: show_file(RunBatch.batch_array_task_text_file_path(bat)) elif file_type == FT_ERR_FILE: show_file(RunBatch.batch_array_task_err_file_path(bat)) elif file_type == FT_OUT_FILE: download_attachment( RunBatch.run_out_file(my_batch, my_run), RunBatch.run_out_file_path(my_batch, my_run)) else: show_help("Unknown file type: %s" % file_type)
def build_submit_run(self): title = "Batch # %d resubmitted"%(self.my_batch.batch_id) with self.tag("html"): with self.tag("head"): with self.tag("title"): self.text(title) with self.tag("style"): self.doc.asis(StyleSheet.BATCHPROFILER_STYLE) with self.tag("body"): with self.tag("h1"): self.text(title) with self.tag("div"): self.text("Your batch # %d has been resubmitted." % self.my_batch.batch_id) with self.tag("table", klass="run-table"): with self.tag("tr"): with self.tag("th"): self.text("First") with self.tag("th"): self.text("Last") with self.tag("th"): self.text("Job #") with self.tag("th"): self.text("Task #") kwds = {} run_id = BATCHPROFILER_VARIABLES[RUN_ID] submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN] if submit_run == RunBatch.JS_ALL: runs = self.my_batch.select_runs() elif submit_run == RunBatch.JS_ONE: runs = [RunBatch.BPRun.select(run_id)] else: if submit_run == RunBatch.JS_INCOMPLETE: statuses = RunBatch.INCOMPLETE_STATUSES else: statuses = [submit_run] tasks = self.my_batch.select_tasks_by_state(statuses) runs = [task.batch_array_task.run for task in tasks] tasks = RunBatch.run(self.my_batch, runs) for task in tasks: assert isinstance(task, RunBatch.BPJobTask) run = task.batch_array_task.run assert isinstance(run, RunBatch.BPRun) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(task.job.job_id)) with self.tag("td"): self.text(str(task.batch_array_task.task_id))
def build_submit_run(self): title = "Batch # %d resubmitted" % (self.my_batch.batch_id) with self.tag("html"): with self.tag("head"): with self.tag("title"): self.text(title) with self.tag("style"): self.doc.asis(StyleSheet.BATCHPROFILER_STYLE) with self.tag("body"): with self.tag("h1"): self.text(title) with self.tag("div"): self.text("Your batch # %d has been resubmitted." % self.my_batch.batch_id) with self.tag("table", klass="run-table"): with self.tag("tr"): with self.tag("th"): self.text("First") with self.tag("th"): self.text("Last") with self.tag("th"): self.text("Job #") with self.tag("th"): self.text("Task #") kwds = {} run_id = BATCHPROFILER_VARIABLES[RUN_ID] submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN] if submit_run == RunBatch.JS_ALL: runs = self.my_batch.select_runs() elif submit_run == RunBatch.JS_ONE: runs = [RunBatch.BPRun.select(run_id)] else: if submit_run == RunBatch.JS_INCOMPLETE: statuses = RunBatch.INCOMPLETE_STATUSES else: statuses = [submit_run] tasks = self.my_batch.select_tasks_by_state(statuses) runs = [task.batch_array_task.run for task in tasks] tasks = RunBatch.run(self.my_batch, runs) for task in tasks: assert isinstance(task, RunBatch.BPJobTask) run = task.batch_array_task.run assert isinstance(run, RunBatch.BPRun) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(task.job.job_id)) with self.tag("td"): self.text(str(task.batch_array_task.task_id))
def build_submit_run(self): title = "Batch # %d resubmitted"%(self.my_batch.batch_id) with self.tag("html"): with self.tag("head"): with self.tag("title"): self.text(title) with self.tag("style"): self.doc.asis(StyleSheet.BATCHPROFILER_STYLE) with self.tag("body"): with self.tag("h1"): self.text(title) with self.tag("div"): self.text("Your batch # %d has been resubmitted." % self.my_batch.batch_id) with self.tag("table", klass="run-table"): with self.tag("tr"): with self.tag("th"): self.text("First") with self.tag("th"): self.text("Last") with self.tag("th"): self.text("Job #") kwds = {} run_id = BATCHPROFILER_VARIABLES[RUN_ID] submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN] if submit_run == RESUBMIT: submit_run = None if run_id is not None: kwds["by_run"] = run_id if submit_run == RunBatch.JS_INCOMPLETE: kwds["by_status"] = RunBatch.INCOMPLETE_STATUSES elif submit_run in RunBatch.INCOMPLETE_STATUSES: kwds["by_status"] = [submit_run] jobs = self.my_batch.select_jobs(**kwds) for run, job, status in jobs: assert isinstance(run, RunBatch.BPRun) assert isinstance(job, RunBatch.BPJob) if run.run_id == self.run_id or \ (submit_run == RunBatch.JS_INCOMPLETE and status in RunBatch.INCOMPLETE_STATUSES) or \ status == submit_run or \ submit_run == RunBatch.JS_ALL: new_job = RunBatch.run_one(self.my_batch, run) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(new_job.job_id))
def build_text_file_table_cell(self, run): '''Build a table cell containing links to the stdout/err output''' text_file = RunBatch.run_text_file(run) text_path = RunBatch.run_text_file_path(self.my_batch, run) err_file = RunBatch.run_err_file(run) err_path = RunBatch.run_err_file_path(self.my_batch, run) with self.tag("td", style="text-align: left"): for ft, path, filename in ( (FT_TEXT_FILE, text_path, text_file), (FT_ERR_FILE, err_path, err_file)): with self.tag("div"): if os.path.isfile(path): with self.tag( "a", style="text-align: left", href='ViewTextFile.py?run_id=%d&%s=%s' % (run.run_id, FILE_TYPE, ft), title=path): self.text(filename) else: with self.tag( "span", style="text-align: left", title='Text file not available'): self.text(filename)
def sql_file_job_and_status(batch_id, sql_file): """Return the latest job ID associated with the batch and sql path batch_id - batch id associated with the submission sql_path - path to the sql script submitted returns latest job or None if not submitted """ batch = RunBatch.select(batch_id) run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_file) if run is None: return None, None, None result = RunBatch.BPJobTask.select_by_run(run) if len(result) == 0: return None, None, None return run, result[0], RunBatch.BPJobTaskStatus.select_by_job_task(job_task)
def sql_file_job_and_status(batch_id, sql_file): """Return the latest job ID associated with the batch and sql path batch_id - batch id associated with the submission sql_path - path to the sql script submitted returns latest job or None if not submitted """ batch = RunBatch.select(batch_id) run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_file) if run is None: return None, None, None result = RunBatch.BPJobTask.select_by_run(run) if len(result) == 0: return None, None, None return run, result[0], RunBatch.BPJobTaskStatus.select_by_job_task( job_task)
def handle_post(): batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID] my_batch = RunBatch.BPBatch() my_batch.select(batch_id) txt_output_dir = RunBatch.text_file_directory(my_batch) job_script_dir = RunBatch.script_file_directory(my_batch) what_i_did = {} maybe_chmod(txt_output_dir, 0777, what_i_did) maybe_chmod(job_script_dir, 0777, what_i_did) for run in my_batch.select_runs(): for path in [ RunBatch.run_text_file_path(my_batch, run), RunBatch.run_out_file_path(my_batch, run), RunBatch.run_err_file_path(my_batch, run) ]: maybe_chmod(path, 0644, what_i_did) print "Content-Type: application/json" print print json.dumps(what_i_did)
cgitb.enable() import RunBatch import cgi import re import os import os.path import subprocess import sql_jobs form = cgi.FieldStorage() batch_id = int(form["batch_id"].value) sql_script = form["sql_script"].value output_file = form["output_file"].value queue = (form.has_key("queue") and form["queue"].value) or None my_batch = RunBatch.LoadBatch(batch_id) re_load_line = re.compile( "'(.+?)[0-9]+_[0-9]+_(Image|Object).CSV'\sREPLACE\sINTO\sTABLE\s([A-Za-z0-9_]+)\s" ) re_ignore_line = re.compile("SHOW WARNINGS;") table_lines = [] image_prefix = None object_prefix = None sql_script_file = open(my_batch["data_dir"] + os.sep + sql_script, "r") in_table_defs = True try: for line in sql_script_file: match = re_load_line.search(line) if match: in_table_defs = False
cgitb.enable() import RunBatch import StyleSheet import cgi import os import os.path def remove_if_exists(path): if os.path.exists(path): os.remove(path) form = cgi.FieldStorage() delete_action = form["delete_action"].value if form.has_key("run_id"): run_id = int(form["run_id"].value) my_batch,my_run = RunBatch.LoadRun(run_id) if ((delete_action.upper() == "ALL") or (delete_action.upper() == "TEXT")): remove_if_exists(RunBatch.RunTextFilePath(my_batch, my_run)) if (delete_action.upper() == "ALL") or (delete_action.upper() == "OUTPUT"): remove_if_exists(RunBatch.RunOutFilePath(my_batch, my_run)) if (delete_action.upper() == "ALL") or (delete_action.upper() == "DONE"): remove_if_exists(RunBatch.RunDoneFilePath(my_batch, my_run)) elif form.has_key("batch_id"): batch_id = int(form["batch_id"].value) my_batch = RunBatch.LoadBatch(batch_id) for my_run in my_batch["runs"]: if ((delete_action.upper() == "ALL") or (delete_action.upper() == "TEXT")): remove_if_exists(RunBatch.RunTextFilePath(my_batch, my_run))
def build_database_scripts(self): sql_files = [] kill_db_action = 'confirm("Are you sure you want to kill the database upload?")' upload_action = 'confirm("Are you sure you want to upload to the database using %s?")' for filename in os.listdir(self.my_batch.data_dir): if filename.upper().endswith(".SQL"): sql_files.append(filename) if len(sql_files) == 0: return with self.tag("div", style='clear:both; padding-top:10px'): with self.tag("h2"): self.text("Database scripts") with self.tag("table", klass="run_table"): with self.tag("tr"): for caption in ("Script file", "Action", "Last job id", "Status", "Output"): with self.tag("th"): self.text(caption) for filename in sql_files: if filename.startswith('batch_'): continue run, job, status = sql_jobs.sql_file_job_and_status( self.my_batch.batch_id, RunBatch.batch_script_file(filename)) with self.tag("tr"): with self.tag("td"): self.text(filename) with self.tag("td", style="text-align:left"): run_button = True output_file = filename[:-3]+'out' output_path = os.path.join( self.my_batch.data_dir,output_file) if job is not None and status in ( RunBatch.JS_RUNNING, RunBatch.JS_SUBMITTED): # # A kill button for jobs that are killable with self.tag("div"): with self.tag("form", action="KillJobs.py", method="POST", target="KillJob"): self.doc.input(type='hidden', name=JOB_ID, value=str(job.job_id)) self.doc.stag( "input", type='submit', value="Kill", name="Kill_db_button", onclick=kill_db_action) with self.tag("div"): with self.tag("form", action = "UploadToDatabase.py", method = "POST"): self.doc.input( type='hidden', name=SQL_SCRIPT, value=filename) self.doc.input(type='hidden', name=OUTPUT_FILE, value = output_path) self.doc.input(type='hidden', name=BATCH_ID) with self.tag( "span", style='white-space:nowrap'): self.doc.stag( "input", type="submit", value="Run", onclick=upload_action % filename) self.doc.asis(" Queue:") with self.doc.select(name=QUEUE): for queue_name in\ bputilities.get_queues(): with self.doc.option( value=queue_name): self.text(queue_name) if job is None: with self.tag("td", colspan="4"): self.text("not run") else: with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): if status != RunBatch.JS_DONE: self.text(status) else: run_time = RunBatch.GetCPUTime( self.my_batch, run) self.text("Complete(%.2f sec)" % run_time) self.build_text_file_table_cell(run)
def build_job_table_body(self): with self.tag("tbody"): for task_status in self.tasks: assert isinstance(task_status, RunBatch.BPJobTaskStatus) task = task_status.job_task status = task_status.status assert isinstance(task, RunBatch.BPJobTask) run = task.batch_array_task.run assert isinstance(run, RunBatch.BPRun) job = task.job assert isinstance(job, RunBatch.BPJob) out_file = RunBatch.run_out_file(self.my_batch, run) out_path = RunBatch.run_out_file_path(self.my_batch, run) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): self.text(str(task.batch_array_task.task_id)) if status == RunBatch.JS_DONE: with self.tag("td"): cpu = task.get_runtime() if cpu is not None: with self.tag("div", **{ "class":"success_message"}): self.text("Complete (%.2f sec)" % cpu) else: with self.tag("div", **{ "class":"error_message"}): self.text("Error: see log") with self.tag("div"): self.build_resubmit_run_form(run) else: with self.tag("td", style='color:red'): with self.tag("div"): self.text(status.lower().capitalize()) with self.tag("div"): self.build_resubmit_run_form(run) self.build_text_file_table_cell(task) if self.my_batch.wants_measurements_file: with self.tag("td"): if os.path.isfile(out_path): with self.tag( "a", href='ViewTextFile.py?run_id=%d&%s=%s' % (run.run_id, FILE_TYPE, FT_OUT_FILE), title=out_path): self.text(out_file) else: with self.tag("span", title='Output file not available'): self.text(out_file) # # This cell contains the form that deletes things # with self.tag("td"): with self.tag("form", action='DeleteFile.py', method='POST'): self.doc.input( type="hidden", name=RUN_ID, value = str(run.run_id)) text_file = os.path.basename( RunBatch.batch_array_task_text_file_path( task.batch_array_task)) for action, filename in ( (A_DELETE_TEXT, text_file), (A_DELETE_OUTPUT, RunBatch.run_out_file(self.my_batch, run)), (A_DELETE_ALL, "all files for this run")): self.doc.stag( "input", type="submit", name=K_DELETE_ACTION, value=action, title='Delete file %s' % filename, onclick='return confirm("Do you really want' 'to delete %s?")' % filename)
def build_database_scripts(self): sql_files = [] kill_db_action = 'confirm("Are you sure you want to kill the database upload?")' upload_action = 'confirm("Are you sure you want to upload to the database using %s?")' for filename in os.listdir(self.my_batch.data_dir): if filename.upper().endswith(".SQL"): sql_files.append(filename) if len(sql_files) == 0: return with self.tag("div", style='clear:both; padding-top:10px'): with self.tag("h2"): self.text("Database scripts") with self.tag("table", klass="run_table"): with self.tag("tr"): for caption in ("Script file", "Action", "Last job id", "Status", "Output"): with self.tag("th"): self.text(caption) for filename in sql_files: if filename.startswith('batch_'): continue run, task, task_status = sql_jobs.sql_file_job_and_status( self.my_batch.batch_id, RunBatch.batch_script_file(filename)) status = None if task_status is None else task_status.status with self.tag("tr"): with self.tag("td"): self.text(filename) with self.tag("td", style="text-align:left"): run_button = True output_file = filename[:-3] + 'out' output_path = os.path.join(self.my_batch.data_dir, output_file) if task is not None and task_status.status in ( RunBatch.JS_RUNNING, RunBatch.JS_SUBMITTED): # # A kill button for jobs that are killable with self.tag("div"): with self.tag("form", action="KillJobs.py", method="POST", target="KillJob"): self.doc.input(type='hidden', name=JOB_ID, value=str(job.job_id)) self.doc.stag("input", type='submit', value="Kill", name="Kill_db_button", onclick=kill_db_action) with self.tag("div"): with self.tag("form", action="UploadToDatabase.py", method="POST"): self.doc.input(type='hidden', name=SQL_SCRIPT, value=filename) self.doc.input(type='hidden', name=OUTPUT_FILE, value=output_path) self.doc.input(type='hidden', name=BATCH_ID) with self.tag("span", style='white-space:nowrap'): self.doc.stag("input", type="submit", value="Run", onclick=upload_action % filename) self.doc.asis(" Queue:") with self.doc.select(name=QUEUE): for queue_name in\ bputilities.get_queues(): with self.doc.option( value=queue_name): self.text(queue_name) if task is None: with self.tag("td", colspan="4"): self.text("not run") else: with self.tag("td"): self.text(str(task.job.job_id)) with self.tag("td"): if status != RunBatch.JS_DONE: self.text(status) else: run_time = \ task_status.created - task.job.created self.text("Complete(%.2f sec)" % run_time.total_seconds()) self.build_text_file_table_cell(task)
re_file_name = re.compile("^(.+?)[0-9]+_[0-9]+_(Image|Object).CSV$") image_files = [] object_files = [] for file_name in os.listdir(my_batch.data_dir): match = re_file_name.search(file_name) if match: if (image_prefix and match.groups(1)[0] == image_prefix and match.groups(1)[1] == 'Image'): image_files.append(file_name) elif (object_prefix and match.groups(1)[0] == object_prefix and match.groups(1)[1] == 'Object'): object_files.append(file_name) batch_script_file = RunBatch.batch_script_file(sql_script) batch_script_dir = RunBatch.batch_script_directory(my_batch) if not os.path.isdir(batch_script_dir): os.makedirs(batch_script_dir) batch_script_path = RunBatch.batch_script_path(my_batch, sql_script) sql_script_file = open(batch_script_path, "w") try: sql_script_file.writelines(table_lines) for file_name in image_files: path_name = os.path.join(my_batch.data_dir, file_name) sql_script_file.write("""SELECT 'Loading %(file_name)s into %(image_table)s';"""%(globals())) sql_script_file.write("""LOAD DATA LOCAL INFILE '%(path_name)s' REPLACE INTO TABLE %(image_table)s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'; SHOW WARNINGS; """%(globals())) for file_name in object_files: path_name = os.path.join(my_batch.data_dir, file_name)
import cgi import os, sys import bputilities from bpformdata import * import RunBatch from cellprofiler.utilities.hdf5_dict import HDF5Dict M_USER_PIPELINE = "Pipeline_UserPipeline" M_PIPELINE = "Pipeline_Pipeline" F_BATCH_DATA_H5 = "Batch_data.h5" EXPERIMENT = "Experiment" with bputilities.CellProfilerContext(): batch_id = BATCHPROFILER_DEFAULTS[BATCH_ID] my_batch = RunBatch.BPBatch.select(batch_id) path = RunBatch.batch_data_file_path(my_batch) h = HDF5Dict(path, mode="r") if M_USER_PIPELINE in h.second_level_names(EXPERIMENT): feature = M_USER_PIPELINE else: feature = M_PIPELINE pipeline_text = h[EXPERIMENT, feature, 0][0].decode("unicode_escape") h.close() del h if isinstance(pipeline_text, unicode): pipeline_text = pipeline_text.encode("us-ascii") print "Content-Type: text/plain" print "Content-Length: %d" % len(pipeline_text) print "Content-Disposition: attachment; filename=\"%d.cppipe\"" % batch_id print sys.stdout.write(pipeline_text)
# import cgitb cgitb.enable() import RunBatch import sql_jobs import StyleSheet import cgi import os import os.path print "Content-Type: text/html" print form = cgi.FieldStorage() batch_id = int(form["batch_id"].value) my_batch = RunBatch.LoadBatch(batch_id) jobs_by_state = {} job_ids = [run["job_id"] for run in my_batch["runs"]] job_dictionary = RunBatch.GetJobStatus(job_ids) for run in my_batch["runs"]: stat = "Unknown" if os.path.isfile(RunBatch.RunDoneFilePath(my_batch, run)): stat = "Complete" elif run["job_id"] == None: pass else: job_status = job_dictionary[run["job_id"]] if job_status and job_status.has_key("STAT"): stat = job_status["STAT"]
""" # # View the text file that is produced by bsub during batch processing # import cgitb cgitb.enable() import RunBatch import cgi import os import os.path import stat if cgi.FieldStorage().has_key("run_id"): run_id = int(cgi.FieldStorage()["run_id"].value) my_batch, my_run = RunBatch.LoadRun(run_id) text_file_path = RunBatch.RunTextFilePath(my_batch, my_run) else: text_file_path = cgi.FieldStorage()["file_name"].value # # This is a temporary work-around because files get created # with the wrong permissions # if (os.stat(text_file_path)[0] & stat.S_IREAD) == 0: os.chmod(text_file_path, 0644) text_file = open(text_file_path, "r") print "Content-Type: text/plain" print print text_file.read() text_file.close()
import os import subprocess import sys job_id = BATCHPROFILER_VARIABLES[JOB_ID] task_id = BATCHPROFILER_VARIABLES[TASK_ID] batch_id = BATCHPROFILER_VARIABLES[BATCH_ID] if job_id is not None: job = RunBatch.BPJob.select_by_job_id(job_id) if job is None: if task_id is None: bputilities.kill_job(job_id) else: bputilities.kill_tasks(job_id, [task_id]) elif task_id is None: RunBatch.kill_job(job) else: task = RunBatch.BPJobTask.select_by_task_id(job, task_id) RunBatch.kill_task(task) print""" <html><head><title>Job %(job_id)d killed</title></head> <body>Job %(job_id)d killed </body> </html> """%(globals()) elif batch_id is not None: RunBatch.kill_batch(batch_id) url = "ViewBatch.py?batch_id=%d"%(batch_id) print "<html><head>" print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
import os.path import subprocess import sql_jobs # # TODO: Move the logic that collects the load statements into sql_jobs # # TODO: use yattag to build the HTML # batch_id = BATCHPROFILER_VARIABLES[BATCH_ID] sql_script = BATCHPROFILER_VARIABLES[SQL_SCRIPT] output_file = BATCHPROFILER_VARIABLES[OUTPUT_FILE] queue = BATCHPROFILER_VARIABLES[QUEUE] my_batch = RunBatch.BPBatch() my_batch.select(batch_id) re_load_line = re.compile( "'(.+?)[0-9]+_[0-9]+_(Image|Object).CSV'\sREPLACE\sINTO\sTABLE\s([A-Za-z0-9_]+)\s" ) re_ignore_line = re.compile("SHOW WARNINGS;") table_lines = [] image_prefix = None object_prefix = None sql_script_file = open(my_batch.data_dir + os.sep + sql_script, "r") in_table_defs = True try: for line in sql_script_file: match = re_load_line.search(line) if match:
cgitb.enable() import RunBatch import bputilities from bpformdata import * import cgi import subprocess import sys job_id = BATCHPROFILER_VARIABLES[JOB_ID] batch_id = BATCHPROFILER_VARIABLES[BATCH_ID] if job_id is not None: job = RunBatch.BPJob.select(job_id) if job is None: bputilities.kill_job(job_id) else: RunBatch.kill_job(job) print""" <html><head><title>Job %(job_id)d killed</title></head> <body>Job %(job_id)d killed </body> </html> """%(globals()) elif batch_id is not None: RunBatch.kill_batch(batch_id) url = "ViewBatch.py?batch_id=%d"%(batch_id) print "<html><head>" print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals()) print "</head>" print "<body>This page should be redirected to <a href='%(url)s'/>%(url)s</a></body>"%(globals()) print "</html>"
Please see the AUTHORS file for credits. Website: http://www.cellprofiler.org """ # # Kill all jobs in a batch # import cgitb cgitb.enable() import RunBatch import cgi import os form = cgi.FieldStorage() batch_id = int(form["batch_id"].value) my_batch = RunBatch.LoadBatch(batch_id) status_dir = "%(data_dir)s/status/" % (my_batch) txt_output_dir = "%(data_dir)s/txt_output/" % (my_batch) if os.path.exists(status_dir): os.chmod(status_dir, 0777) if os.path.exists(txt_output_dir): os.chmod(txt_output_dir, 0777) for run in my_batch["runs"]: for path in [ RunBatch.RunTextFilePath(my_batch, run), RunBatch.RunDoneFilePath(my_batch, run), RunBatch.RunOutFilePath(my_batch, run) ]: if os.path.exists(path): os.chmod(path, 0644)
def build_submit_batch(self): '''Build the webpage for handling a submitted batch Also do the batch submission ''' batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE]) if self.has_groups: first_last = np.hstack( [[True], self.group_numbers[1:] != self.group_numbers[:-1], [True]]) gn = self.group_numbers[first_last[:-1]] first = self.image_numbers[first_last[:-1]] last = self.image_numbers[first_last[1:]] else: first = self.image_numbers[::batch_size] last = self.image_numbers[(batch_size-1)::batch_size] if len(last) < len(first): last = np.hstack([last, self.image_numbers[-1]]) batch = RunBatch.BPBatch() runs = [(f, l) for f, l in zip(first, last)] # # Put it in the database # write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0 batch.create( email = BATCHPROFILER_DEFAULTS[EMAIL], data_dir = BATCHPROFILER_DEFAULTS[DATA_DIR], queue = BATCHPROFILER_DEFAULTS[QUEUE], batch_size = BATCHPROFILER_DEFAULTS[BATCH_SIZE], write_data = write_data, timeout = 60, cpcluster = self.cpcluster, project = BATCHPROFILER_DEFAULTS[PROJECT], memory_limit = BATCHPROFILER_DEFAULTS[MEMORY_LIMIT], priority = BATCHPROFILER_DEFAULTS[PRIORITY], runs = runs) RunBatch.run_all(batch.batch_id) vb_url = "ViewBatch.py?batch_id=%d" % batch.batch_id self.send_batch_submission_email(batch, vb_url) job_list = batch.select_jobs() with self.tag("head"): with self.tag("title"): self.text("Batch #%d" % batch.batch_id) with self.tag("style", type="text/css"): self.doc.asis(""" table { border-spacing: 0px; border-collapse: collapse; } td { text-align: left; vertical-align: baseline; padding: 0.1em 0.5em; border: 1px solid #666666; }""") with self.tag("body"): with self.tag("h1"): self.text("Results for batch #") with self.tag("a", href=vb_url): self.text(str(batch.batch_id)) with self.tag("table"): with self.tag("thead"): with self.tag("tr"): with self.tag("th"): self.text("First image set") with self.tag("th"): self.text("Last image set") with self.tag("th"): self.text("job #") for run, job, status in job_list: assert isinstance(run, RunBatch.BPRun) assert isinstance(job, RunBatch.BPJob) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id))
# print "Content-Type: text/html" print import cgitb cgitb.enable() import RunBatch import cgi import subprocess import sys form = cgi.FieldStorage() if form.has_key("job_id"): import subprocess job_id = int(form["job_id"].value) run = {"job_id":job_id} RunBatch.KillOne(run) print""" <html><head><title>Job %(job_id)d killed</title></head> <body>Job %(job_id)d killed </body> </html> """%(globals()) elif form.has_key("batch_id"): batch_id = int(form["batch_id"].value) my_batch = RunBatch.LoadBatch(batch_id) for run in my_batch["runs"]: RunBatch.KillOne(run) url = "ViewBatch.py?batch_id=%(batch_id)d"%(my_batch) print "<html><head>" print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
def build_submit_batch(self): '''Build the webpage for handling a submitted batch Also do the batch submission ''' batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE]) if self.has_groups: first_last = np.hstack( [[True], self.group_numbers[1:] != self.group_numbers[:-1], [True]]) gn = self.group_numbers[first_last[:-1]] first = self.image_numbers[first_last[:-1]] last = self.image_numbers[first_last[1:]] else: first = self.image_numbers[::batch_size] last = self.image_numbers[(batch_size - 1)::batch_size] if len(last) < len(first): last = np.hstack([last, self.image_numbers[-1]]) runs = [(f, l) for f, l in zip(first, last)] # # Put it in the database # write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0 with RunBatch.bpcursor() as cursor: batch = RunBatch.BPBatch.create( cursor, email=BATCHPROFILER_DEFAULTS[EMAIL], data_dir=BATCHPROFILER_DEFAULTS[DATA_DIR], queue=BATCHPROFILER_DEFAULTS[QUEUE], batch_size=BATCHPROFILER_DEFAULTS[BATCH_SIZE], write_data=write_data, timeout=60, cpcluster=self.cpcluster, project=BATCHPROFILER_DEFAULTS[PROJECT], memory_limit=BATCHPROFILER_DEFAULTS[MEMORY_LIMIT], priority=BATCHPROFILER_DEFAULTS[PRIORITY]) bpruns = [] for bstart, bend in runs: cmd = RunBatch.cellprofiler_command(batch, bstart, bend) run = RunBatch.BPRun.create(cursor, batch, bstart, bend, cmd) bpruns.append(run) RunBatch.run(batch, bpruns) vb_url = BATCHPROFILER_DEFAULTS[URL] + \ "/ViewBatch.py?batch_id=%d" % batch.batch_id self.send_batch_submission_email(batch, vb_url) task_list = batch.select_tasks() with self.tag("head"): with self.tag("title"): self.text("Batch #%d" % batch.batch_id) with self.tag("style", type="text/css"): self.doc.asis(""" table { border-spacing: 0px; border-collapse: collapse; } td { text-align: left; vertical-align: baseline; padding: 0.1em 0.5em; border: 1px solid #666666; }""") with self.tag("body"): with self.tag("h1"): self.text("Results for batch #") with self.tag("a", href=vb_url): self.text(str(batch.batch_id)) with self.tag("table"): with self.tag("thead"): with self.tag("tr"): with self.tag("th"): self.text("First image set") with self.tag("th"): self.text("Last image set") with self.tag("th"): self.text("job #") with self.tag("th"): self.text("task #") for task_status in task_list: assert isinstance(task_status, RunBatch.BPJobTaskStatus) task = task_status.job_task assert isinstance(task, RunBatch.BPJobTask) job = task.job assert isinstance(job, RunBatch.BPJob) bat = task.batch_array_task assert isinstance(bat, RunBatch.BPBatchArrayTask) task_id = bat.task_id run = bat.run with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): self.text(str(task_id))
""" print "Content-Type: text/html" print import cgitb cgitb.enable() import RunBatch import cgi import subprocess import sys form = cgi.FieldStorage() if form.has_key("job_id"): import subprocess job_id = int(form["job_id"].value) run = {"job_id":job_id} RunBatch.KillOne(run) print""" <html><head><title>Job %(job_id)d killed</title></head> <body>Job %(job_id)d killed </body> </html> """%(globals()) elif form.has_key("batch_id"): batch_id = int(form["batch_id"].value) RunBatch.KillBatch(batch_id) url = "ViewBatch.py?batch_id=%d"%(batch_id) print "<html><head>" print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals()) print "</head>" print "<body>This page should be redirected to <a href='%(url)s'/>%(url)s</a></body>"%(globals())
def build_job_table_body(self): with self.tag("tbody"): for task_status in self.tasks: assert isinstance(task_status, RunBatch.BPJobTaskStatus) task = task_status.job_task status = task_status.status assert isinstance(task, RunBatch.BPJobTask) run = task.batch_array_task.run assert isinstance(run, RunBatch.BPRun) job = task.job assert isinstance(job, RunBatch.BPJob) out_file = RunBatch.run_out_file(self.my_batch, run) out_path = RunBatch.run_out_file_path(self.my_batch, run) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): self.text(str(task.batch_array_task.task_id)) if status == RunBatch.JS_DONE: with self.tag("td"): cpu = task_status.created - job.created self.text("Complete (%.2f sec)" % (cpu.total_seconds())) else: with self.tag("td", style='color:red'): with self.tag("div"): self.text(status.lower().capitalize()) with self.tag("div"): with self.tag("form", action="ViewBatch.py", method="POST", target="ResubmitWindow"): self.doc.input(type="hidden", name=BATCH_ID) self.doc.input(type="hidden", name=RUN_ID, value=str(run.run_id)) self.doc.input(type="hidden", name=SUBMIT_RUN, value=RunBatch.JS_ONE) self.doc.stag("input", type='submit', name="Resubmit_Button", value=RESUBMIT) self.build_text_file_table_cell(task) if self.my_batch.wants_measurements_file: with self.tag("td"): if os.path.isfile(out_path): with self.tag( "a", href='ViewTextFile.py?run_id=%d&%s=%s' % (run.run_id, FILE_TYPE, FT_OUT_FILE), title=out_path): self.text(out_file) else: with self.tag( "span", title='Output file not available'): self.text(out_file) # # This cell contains the form that deletes things # with self.tag("td"): with self.tag("form", action='DeleteFile.py', method='POST'): self.doc.input(type="hidden", name=RUN_ID, value=str(run.run_id)) text_file = os.path.basename( RunBatch.batch_array_task_text_file_path( task.batch_array_task)) for action, filename in ( (A_DELETE_TEXT, text_file), (A_DELETE_OUTPUT, RunBatch.run_out_file(self.my_batch, run)), (A_DELETE_ALL, "all files for this run")): self.doc.stag( "input", type="submit", name=K_DELETE_ACTION, value=action, title='Delete file %s' % filename, onclick='return confirm("Do you really want' 'to delete %s?")' % filename)
import os import subprocess import sys job_id = BATCHPROFILER_VARIABLES[JOB_ID] task_id = BATCHPROFILER_VARIABLES[TASK_ID] batch_id = BATCHPROFILER_VARIABLES[BATCH_ID] if job_id is not None: job = RunBatch.BPJob.select_by_job_id(job_id) if job is None: if task_id is None: bputilities.kill_job(job_id) else: bputilities.kill_tasks(job_id, [task_id]) elif task_id is None: RunBatch.kill_job(job) else: task = RunBatch.BPJobTask.select_by_task_id(job, task_id) RunBatch.kill_task(task) print """ <html><head><title>Job %(job_id)d killed</title></head> <body>Job %(job_id)d killed </body> </html> """ % ( globals() ) elif batch_id is not None: RunBatch.kill_batch(batch_id) url = "ViewBatch.py?batch_id=%d" % (batch_id)
if form_data.has_key("batch_size"): batch_size = int(form_data["batch_size"].value) for i in image_numbers[::batch_size]: start = i end = min(start + batch_size - 1, max(image_numbers)) status_file_name = ("%s/status/Batch_%d_to_%d_DONE.mat" % (batch["data_dir"], start, end)) run = { "start": start, "end": end, "group": None, "status_file_name": status_file_name } batch["runs"].append(run) batch_id = RunBatch.CreateBatchRun(batch) email_text = [] email_text.append("<html>") email_text.append("<head><title>Batch # %d</title>" % (batch_id)) email_text.append("<style type='text/css'>") email_text.append(""" table { border-spacing: 0px; border-collapse: collapse; } td { text-align: left; vertical-align: baseline; padding: 0.1em 0.5em; border: 1px solid #666666;
def build_job_table_body(self): with self.tag("tbody"): for run, job, status in self.jobs: assert isinstance(run, RunBatch.BPRun) assert isinstance(job, RunBatch.BPJob) out_file = RunBatch.run_out_file(self.my_batch, run) out_path = RunBatch.run_out_file_path(self.my_batch, run) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id)) if status == RunBatch.JS_DONE: with self.tag("td"): cpu = RunBatch.GetCPUTime(self.my_batch, run) self.text("Complete (%.2f sec)"%(cpu)) else: with self.tag("td", style='color:red'): with self.tag("div"): self.text(status.lower().capitalize()) with self.tag("div"): with self.tag("form", action="ViewBatch.py", method="POST", target="ResubmitWindow"): self.doc.input( type="hidden", name=BATCH_ID) self.doc.input( type="hidden", name=RUN_ID, value=str(run.run_id)) self.doc.stag( "input", type='submit', name=SUBMIT_RUN, value=RESUBMIT) self.build_text_file_table_cell(run) if self.my_batch.wants_measurements_file: with self.tag("td"): if os.path.isfile(out_path): with self.tag( "a", href='ViewTextFile.py?run_id=%d&%s=%s' % (run.run_id, FILE_TYPE, FT_OUT_FILE), title=out_path): self.text(out_file) else: with self.tag("span", title='Output file not available'): self.text(out_file) # # This cell contains the form that deletes things # with self.tag("td"): with self.tag("form", action='DeleteFile.py', method='POST'): self.doc.input( type="hidden", name=RUN_ID, value = str(run.run_id)) for action, filename in ( (A_DELETE_TEXT, RunBatch.run_text_file(run)), (A_DELETE_OUTPUT, RunBatch.run_out_file(self.my_batch, run)), (A_DELETE_ALL, "all files for this run")): self.doc.stag( "input", type="submit", name=K_DELETE_ACTION, value=action, title='Delete file %s' % filename, onclick='return confirm("Do you really want' 'to delete %s?")' % filename)