def build_database_scripts(self): sql_files = [] kill_db_action = 'confirm("Are you sure you want to kill the database upload?")' upload_action = 'confirm("Are you sure you want to upload to the database using %s?")' for filename in os.listdir(self.my_batch.data_dir): if filename.upper().endswith(".SQL"): sql_files.append(filename) if len(sql_files) == 0: return with self.tag("div", style='clear:both; padding-top:10px'): with self.tag("h2"): self.text("Database scripts") with self.tag("table", klass="run_table"): with self.tag("tr"): for caption in ("Script file", "Action", "Last job id", "Status", "Output"): with self.tag("th"): self.text(caption) for filename in sql_files: if filename.startswith('batch_'): continue run, task, task_status = sql_jobs.sql_file_job_and_status( self.my_batch.batch_id, RunBatch.batch_script_file(filename)) status = None if task_status is None else task_status.status with self.tag("tr"): with self.tag("td"): self.text(filename) with self.tag("td", style="text-align:left"): run_button = True output_file = filename[:-3] + 'out' output_path = os.path.join(self.my_batch.data_dir, output_file) if task is not None and task_status.status in ( RunBatch.JS_RUNNING, RunBatch.JS_SUBMITTED): # # A kill button for jobs that are killable with self.tag("div"): with self.tag("form", action="KillJobs.py", method="POST", target="KillJob"): self.doc.input(type='hidden', name=JOB_ID, value=str(job.job_id)) self.doc.stag("input", type='submit', value="Kill", name="Kill_db_button", onclick=kill_db_action) with self.tag("div"): with self.tag("form", action="UploadToDatabase.py", method="POST"): self.doc.input(type='hidden', name=SQL_SCRIPT, value=filename) self.doc.input(type='hidden', name=OUTPUT_FILE, value=output_path) self.doc.input(type='hidden', name=BATCH_ID) with self.tag("span", style='white-space:nowrap'): self.doc.stag("input", type="submit", value="Run", onclick=upload_action % filename) self.doc.asis(" Queue:") with self.doc.select(name=QUEUE): for queue_name in\ bputilities.get_queues(): with self.doc.option( value=queue_name): self.text(queue_name) if task is None: with self.tag("td", colspan="4"): self.text("not run") else: with self.tag("td"): self.text(str(task.job.job_id)) with self.tag("td"): if status != RunBatch.JS_DONE: self.text(status) else: run_time = \ task_status.created - task.job.created self.text("Complete(%.2f sec)" % run_time.total_seconds()) self.build_text_file_table_cell(task)
def build_database_scripts(self): sql_files = [] kill_db_action = 'confirm("Are you sure you want to kill the database upload?")' upload_action = 'confirm("Are you sure you want to upload to the database using %s?")' for filename in os.listdir(self.my_batch.data_dir): if filename.upper().endswith(".SQL"): sql_files.append(filename) if len(sql_files) == 0: return with self.tag("div", style='clear:both; padding-top:10px'): with self.tag("h2"): self.text("Database scripts") with self.tag("table", klass="run_table"): with self.tag("tr"): for caption in ("Script file", "Action", "Last job id", "Status", "Output"): with self.tag("th"): self.text(caption) for filename in sql_files: if filename.startswith('batch_'): continue run, job, status = sql_jobs.sql_file_job_and_status( self.my_batch.batch_id, RunBatch.batch_script_file(filename)) with self.tag("tr"): with self.tag("td"): self.text(filename) with self.tag("td", style="text-align:left"): run_button = True output_file = filename[:-3]+'out' output_path = os.path.join( self.my_batch.data_dir,output_file) if job is not None and status in ( RunBatch.JS_RUNNING, RunBatch.JS_SUBMITTED): # # A kill button for jobs that are killable with self.tag("div"): with self.tag("form", action="KillJobs.py", method="POST", target="KillJob"): self.doc.input(type='hidden', name=JOB_ID, value=str(job.job_id)) self.doc.stag( "input", type='submit', value="Kill", name="Kill_db_button", onclick=kill_db_action) with self.tag("div"): with self.tag("form", action = "UploadToDatabase.py", method = "POST"): self.doc.input( type='hidden', name=SQL_SCRIPT, value=filename) self.doc.input(type='hidden', name=OUTPUT_FILE, value = output_path) self.doc.input(type='hidden', name=BATCH_ID) with self.tag( "span", style='white-space:nowrap'): self.doc.stag( "input", type="submit", value="Run", onclick=upload_action % filename) self.doc.asis(" Queue:") with self.doc.select(name=QUEUE): for queue_name in\ bputilities.get_queues(): with self.doc.option( value=queue_name): self.text(queue_name) if job is None: with self.tag("td", colspan="4"): self.text("not run") else: with self.tag("td"): self.text(str(job.job_id)) with self.tag("td"): if status != RunBatch.JS_DONE: self.text(status) else: run_time = RunBatch.GetCPUTime( self.my_batch, run) self.text("Complete(%.2f sec)" % run_time) self.build_text_file_table_cell(run)
sql_script_file.close() re_file_name = re.compile("^(.+?)[0-9]+_[0-9]+_(Image|Object).CSV$") image_files = [] object_files = [] for file_name in os.listdir(my_batch.data_dir): match = re_file_name.search(file_name) if match: if (image_prefix and match.groups(1)[0] == image_prefix and match.groups(1)[1] == 'Image'): image_files.append(file_name) elif (object_prefix and match.groups(1)[0] == object_prefix and match.groups(1)[1] == 'Object'): object_files.append(file_name) batch_script_file = RunBatch.batch_script_file(sql_script) batch_script_dir = RunBatch.batch_script_directory(my_batch) if not os.path.isdir(batch_script_dir): os.makedirs(batch_script_dir) batch_script_path = RunBatch.batch_script_path(my_batch, sql_script) sql_script_file = open(batch_script_path, "w") try: sql_script_file.writelines(table_lines) for file_name in image_files: path_name = os.path.join(my_batch.data_dir, file_name) sql_script_file.write( """SELECT 'Loading %(file_name)s into %(image_table)s';""" % (globals())) sql_script_file.write( """LOAD DATA LOCAL INFILE '%(path_name)s' REPLACE INTO TABLE %(image_table)s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'; SHOW WARNINGS;
re_file_name = re.compile("^(.+?)[0-9]+_[0-9]+_(Image|Object).CSV$") image_files = [] object_files = [] for file_name in os.listdir(my_batch.data_dir): match = re_file_name.search(file_name) if match: if (image_prefix and match.groups(1)[0] == image_prefix and match.groups(1)[1] == 'Image'): image_files.append(file_name) elif (object_prefix and match.groups(1)[0] == object_prefix and match.groups(1)[1] == 'Object'): object_files.append(file_name) batch_script_file = RunBatch.batch_script_file(sql_script) batch_script_dir = RunBatch.batch_script_directory(my_batch) if not os.path.isdir(batch_script_dir): os.makedirs(batch_script_dir) batch_script_path = RunBatch.batch_script_path(my_batch, sql_script) sql_script_file = open(batch_script_path, "w") try: sql_script_file.writelines(table_lines) for file_name in image_files: path_name = os.path.join(my_batch.data_dir, file_name) sql_script_file.write("""SELECT 'Loading %(file_name)s into %(image_table)s';"""%(globals())) sql_script_file.write("""LOAD DATA LOCAL INFILE '%(path_name)s' REPLACE INTO TABLE %(image_table)s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'; SHOW WARNINGS; """%(globals())) for file_name in object_files: path_name = os.path.join(my_batch.data_dir, file_name)