def build_submit_run(self): title = "Batch # %d resubmitted" % (self.my_batch.batch_id) with self.tag("html"): with self.tag("head"): with self.tag("title"): self.text(title) with self.tag("style"): self.doc.asis(StyleSheet.BATCHPROFILER_STYLE) with self.tag("body"): with self.tag("h1"): self.text(title) with self.tag("div"): self.text("Your batch # %d has been resubmitted." % self.my_batch.batch_id) with self.tag("table", klass="run-table"): with self.tag("tr"): with self.tag("th"): self.text("First") with self.tag("th"): self.text("Last") with self.tag("th"): self.text("Job #") with self.tag("th"): self.text("Task #") kwds = {} run_id = BATCHPROFILER_VARIABLES[RUN_ID] submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN] if submit_run == RunBatch.JS_ALL: runs = self.my_batch.select_runs() elif submit_run == RunBatch.JS_ONE: runs = [RunBatch.BPRun.select(run_id)] else: if submit_run == RunBatch.JS_INCOMPLETE: statuses = RunBatch.INCOMPLETE_STATUSES else: statuses = [submit_run] tasks = self.my_batch.select_tasks_by_state(statuses) runs = [task.batch_array_task.run for task in tasks] tasks = RunBatch.run(self.my_batch, runs) for task in tasks: assert isinstance(task, RunBatch.BPJobTask) run = task.batch_array_task.run assert isinstance(run, RunBatch.BPRun) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(task.job.job_id)) with self.tag("td"): self.text(str(task.batch_array_task.task_id))
def build_submit_run(self): title = "Batch # %d resubmitted"%(self.my_batch.batch_id) with self.tag("html"): with self.tag("head"): with self.tag("title"): self.text(title) with self.tag("style"): self.doc.asis(StyleSheet.BATCHPROFILER_STYLE) with self.tag("body"): with self.tag("h1"): self.text(title) with self.tag("div"): self.text("Your batch # %d has been resubmitted." % self.my_batch.batch_id) with self.tag("table", klass="run-table"): with self.tag("tr"): with self.tag("th"): self.text("First") with self.tag("th"): self.text("Last") with self.tag("th"): self.text("Job #") with self.tag("th"): self.text("Task #") kwds = {} run_id = BATCHPROFILER_VARIABLES[RUN_ID] submit_run = BATCHPROFILER_VARIABLES[SUBMIT_RUN] if submit_run == RunBatch.JS_ALL: runs = self.my_batch.select_runs() elif submit_run == RunBatch.JS_ONE: runs = [RunBatch.BPRun.select(run_id)] else: if submit_run == RunBatch.JS_INCOMPLETE: statuses = RunBatch.INCOMPLETE_STATUSES else: statuses = [submit_run] tasks = self.my_batch.select_tasks_by_state(statuses) runs = [task.batch_array_task.run for task in tasks] tasks = RunBatch.run(self.my_batch, runs) for task in tasks: assert isinstance(task, RunBatch.BPJobTask) run = task.batch_array_task.run assert isinstance(run, RunBatch.BPRun) with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(task.job.job_id)) with self.tag("td"): self.text(str(task.batch_array_task.task_id))
def run_sql_file(batch_id, sql_filename): """Use the mysql command line to run the given SQL script batch_id - sql_file is associated with this batch sql_path - path and file of the sql script queue - run job on this queue email - who to email when done returns the RunBatch.BPJob """ cwd = os.path.dirname(__file__) batch = RunBatch.BPBatch.select(batch_id) run = RunBatch.BPSQLRun.select_by_sql_filename(batch, sql_filename) if run is None: sql_path = os.path.join(RunBatch.batch_script_directory(batch), sql_filename) cmd = "%s -b %d -i %s"%(os.path.join(cwd, "sql_jobs.py"), batch_id, sql_path) run = RunBatch.BPSQLRun.create(batch, sql_filename, cmd) return RunBatch.run(batch, [run], cwd=cwd)
def build_submit_batch(self): '''Build the webpage for handling a submitted batch Also do the batch submission ''' batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE]) if self.has_groups: first_last = np.hstack( [[True], self.group_numbers[1:] != self.group_numbers[:-1], [True]]) gn = self.group_numbers[first_last[:-1]] first = self.image_numbers[first_last[:-1]] last = self.image_numbers[first_last[1:]] else: first = self.image_numbers[::batch_size] last = self.image_numbers[(batch_size - 1)::batch_size] if len(last) < len(first): last = np.hstack([last, self.image_numbers[-1]]) runs = [(f, l) for f, l in zip(first, last)] # # Put it in the database # write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0 with RunBatch.bpcursor() as cursor: batch = RunBatch.BPBatch.create( cursor, email=BATCHPROFILER_DEFAULTS[EMAIL], data_dir=BATCHPROFILER_DEFAULTS[DATA_DIR], queue=BATCHPROFILER_DEFAULTS[QUEUE], batch_size=BATCHPROFILER_DEFAULTS[BATCH_SIZE], write_data=write_data, timeout=60, cpcluster=self.cpcluster, project=BATCHPROFILER_DEFAULTS[PROJECT], memory_limit=BATCHPROFILER_DEFAULTS[MEMORY_LIMIT], priority=BATCHPROFILER_DEFAULTS[PRIORITY]) bpruns = [] for bstart, bend in runs: cmd = RunBatch.cellprofiler_command(batch, bstart, bend) run = RunBatch.BPRun.create(cursor, batch, bstart, bend, cmd) bpruns.append(run) RunBatch.run(batch, bpruns) vb_url = BATCHPROFILER_DEFAULTS[URL] + \ "/ViewBatch.py?batch_id=%d" % batch.batch_id self.send_batch_submission_email(batch, vb_url) task_list = batch.select_tasks() with self.tag("head"): with self.tag("title"): self.text("Batch #%d" % batch.batch_id) with self.tag("style", type="text/css"): self.doc.asis(""" table { border-spacing: 0px; border-collapse: collapse; } td { text-align: left; vertical-align: baseline; padding: 0.1em 0.5em; border: 1px solid #666666; }""") with self.tag("body"): with self.tag("h1"): self.text("Results for batch #") with self.tag("a", href=vb_url): self.text(str(batch.batch_id)) with self.tag("table"): with self.tag("thead"): with self.tag("tr"): with self.tag("th"): self.text("First image set") with self.tag("th"): self.text("Last image set") with self.tag("th"): self.text("job #") with self.tag("th"): self.text("task #") for task_status in task_list: assert isinstance(task_status, RunBatch.BPJobTaskStatus) task = task_status.job_task assert isinstance(task, RunBatch.BPJobTask) job = task.job assert isinstance(job, RunBatch.BPJob) bat = task.batch_array_task assert isinstance(bat, RunBatch.BPBatchArrayTask) task_id = bat.task_id run = bat.run with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): self.text(str(task_id))
def build_submit_batch(self): '''Build the webpage for handling a submitted batch Also do the batch submission ''' batch_size = int(BATCHPROFILER_DEFAULTS[BATCH_SIZE]) if self.has_groups: first_last = np.hstack( [[True], self.group_numbers[1:] != self.group_numbers[:-1], [True]]) gn = self.group_numbers[first_last[:-1]] first = self.image_numbers[first_last[:-1]] last = self.image_numbers[first_last[1:]] else: first = self.image_numbers[::batch_size] last = self.image_numbers[(batch_size-1)::batch_size] if len(last) < len(first): last = np.hstack([last, self.image_numbers[-1]]) runs = [(f, l) for f, l in zip(first, last)] # # Put it in the database # write_data = 1 if BATCHPROFILER_VARIABLES[WRITE_DATA] is not None else 0 with RunBatch.bpcursor() as cursor: batch = RunBatch.BPBatch.create( cursor, email = BATCHPROFILER_DEFAULTS[EMAIL], data_dir = BATCHPROFILER_DEFAULTS[DATA_DIR], queue = BATCHPROFILER_DEFAULTS[QUEUE], batch_size = BATCHPROFILER_DEFAULTS[BATCH_SIZE], write_data = write_data, timeout = 60, cpcluster = self.cpcluster, project = BATCHPROFILER_DEFAULTS[PROJECT], memory_limit = BATCHPROFILER_DEFAULTS[MEMORY_LIMIT], priority = BATCHPROFILER_DEFAULTS[PRIORITY]) bpruns = [] for bstart, bend in runs: cmd = RunBatch.cellprofiler_command(batch, bstart, bend) run = RunBatch.BPRun.create(cursor, batch, bstart, bend, cmd) bpruns.append(run) RunBatch.run(batch, bpruns) vb_url = BATCHPROFILER_DEFAULTS[URL] + \ "/ViewBatch.py?batch_id=%d" % batch.batch_id self.send_batch_submission_email(batch, vb_url) task_list = batch.select_tasks() with self.tag("head"): with self.tag("title"): self.text("Batch #%d" % batch.batch_id) with self.tag("style", type="text/css"): self.doc.asis(""" table { border-spacing: 0px; border-collapse: collapse; } td { text-align: left; vertical-align: baseline; padding: 0.1em 0.5em; border: 1px solid #666666; }""") with self.tag("body"): with self.tag("h1"): self.text("Results for batch #") with self.tag("a", href=vb_url): self.text(str(batch.batch_id)) with self.tag("table"): with self.tag("thead"): with self.tag("tr"): with self.tag("th"): self.text("First image set") with self.tag("th"): self.text("Last image set") with self.tag("th"): self.text("job #") with self.tag("th"): self.text("task #") for task_status in task_list: assert isinstance(task_status, RunBatch.BPJobTaskStatus) task = task_status.job_task assert isinstance(task, RunBatch.BPJobTask) job = task.job assert isinstance(job, RunBatch.BPJob) bat = task.batch_array_task assert isinstance(bat, RunBatch.BPBatchArrayTask) task_id = bat.task_id run = bat.run with self.tag("tr"): with self.tag("td"): self.text(str(run.bstart)) with self.tag("td"): self.text(str(run.bend)) with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): self.text(str(task_id))