def build_database_scripts(self):
sql_files = []
kill_db_action = 'confirm("Are you sure you want to kill the database upload?")'
upload_action = 'confirm("Are you sure you want to upload to the database using %s?")'
for filename in os.listdir(self.my_batch.data_dir):
if filename.upper().endswith(".SQL"):
sql_files.append(filename)
if len(sql_files) == 0:
return
with self.tag("div", style='clear:both; padding-top:10px'):
with self.tag("h2"):
self.text("Database scripts")
with self.tag("table", klass="run_table"):
with self.tag("tr"):
for caption in ("Script file", "Action", "Last job id",
"Status", "Output"):
with self.tag("th"):
self.text(caption)
for filename in sql_files:
if filename.startswith('batch_'):
continue
run, task, task_status = sql_jobs.sql_file_job_and_status(
self.my_batch.batch_id,
RunBatch.batch_script_file(filename))
status = None if task_status is None else task_status.status
with self.tag("tr"):
with self.tag("td"):
self.text(filename)
with self.tag("td", style="text-align:left"):
run_button = True
output_file = filename[:-3] + 'out'
output_path = os.path.join(self.my_batch.data_dir,
output_file)
if task is not None and task_status.status in (
RunBatch.JS_RUNNING,
RunBatch.JS_SUBMITTED):
#
# A kill button for jobs that are killable
with self.tag("div"):
with self.tag("form",
action="KillJobs.py",
method="POST",
target="KillJob"):
self.doc.input(type='hidden',
name=JOB_ID,
value=str(job.job_id))
self.doc.stag("input",
type='submit',
value="Kill",
name="Kill_db_button",
onclick=kill_db_action)
with self.tag("div"):
with self.tag("form",
action="UploadToDatabase.py",
method="POST"):
self.doc.input(type='hidden',
name=SQL_SCRIPT,
value=filename)
self.doc.input(type='hidden',
name=OUTPUT_FILE,
value=output_path)
self.doc.input(type='hidden',
name=BATCH_ID)
with self.tag("span",
style='white-space:nowrap'):
self.doc.stag("input",
type="submit",
value="Run",
onclick=upload_action %
filename)
self.doc.asis(" Queue:")
with self.doc.select(name=QUEUE):
for queue_name in\
bputilities.get_queues():
with self.doc.option(
value=queue_name):
self.text(queue_name)
if task is None:
with self.tag("td", colspan="4"):
self.text("not run")
else:
with self.tag("td"):
self.text(str(task.job.job_id))
with self.tag("td"):
if status != RunBatch.JS_DONE:
self.text(status)
else:
run_time = \
task_status.created - task.job.created
self.text("Complete(%.2f sec)" %
run_time.total_seconds())
self.build_text_file_table_cell(task)
def build_database_scripts(self):
sql_files = []
kill_db_action = 'confirm("Are you sure you want to kill the database upload?")'
upload_action = 'confirm("Are you sure you want to upload to the database using %s?")'
for filename in os.listdir(self.my_batch.data_dir):
if filename.upper().endswith(".SQL"):
sql_files.append(filename)
if len(sql_files) == 0:
return
with self.tag("div", style='clear:both; padding-top:10px'):
with self.tag("h2"):
self.text("Database scripts")
with self.tag("table", klass="run_table"):
with self.tag("tr"):
for caption in ("Script file", "Action", "Last job id",
"Status", "Output"):
with self.tag("th"):
self.text(caption)
for filename in sql_files:
if filename.startswith('batch_'):
continue
run, job, status = sql_jobs.sql_file_job_and_status(
self.my_batch.batch_id,
RunBatch.batch_script_file(filename))
with self.tag("tr"):
with self.tag("td"):
self.text(filename)
with self.tag("td", style="text-align:left"):
run_button = True
output_file = filename[:-3]+'out'
output_path = os.path.join(
self.my_batch.data_dir,output_file)
if job is not None and status in (
RunBatch.JS_RUNNING, RunBatch.JS_SUBMITTED):
#
# A kill button for jobs that are killable
with self.tag("div"):
with self.tag("form",
action="KillJobs.py",
method="POST",
target="KillJob"):
self.doc.input(type='hidden',
name=JOB_ID,
value=str(job.job_id))
self.doc.stag(
"input",
type='submit', value="Kill",
name="Kill_db_button",
onclick=kill_db_action)
with self.tag("div"):
with self.tag("form",
action = "UploadToDatabase.py",
method = "POST"):
self.doc.input(
type='hidden',
name=SQL_SCRIPT,
value=filename)
self.doc.input(type='hidden',
name=OUTPUT_FILE,
value = output_path)
self.doc.input(type='hidden',
name=BATCH_ID)
with self.tag(
"span", style='white-space:nowrap'):
self.doc.stag(
"input",
type="submit",
value="Run",
onclick=upload_action % filename)
self.doc.asis(" Queue:")
with self.doc.select(name=QUEUE):
for queue_name in\
bputilities.get_queues():
with self.doc.option(
value=queue_name):
self.text(queue_name)
if job is None:
with self.tag("td", colspan="4"):
self.text("not run")
else:
with self.tag("td"):
self.text(str(job.job_id))
with self.tag("td"):
if status != RunBatch.JS_DONE:
self.text(status)
else:
run_time = RunBatch.GetCPUTime(
self.my_batch, run)
self.text("Complete(%.2f sec)" % run_time)
self.build_text_file_table_cell(run)
sql_script_file.close()
re_file_name = re.compile("^(.+?)[0-9]+_[0-9]+_(Image|Object).CSV$")
image_files = []
object_files = []
for file_name in os.listdir(my_batch.data_dir):
match = re_file_name.search(file_name)
if match:
if (image_prefix and match.groups(1)[0] == image_prefix
and match.groups(1)[1] == 'Image'):
image_files.append(file_name)
elif (object_prefix and match.groups(1)[0] == object_prefix
and match.groups(1)[1] == 'Object'):
object_files.append(file_name)
batch_script_file = RunBatch.batch_script_file(sql_script)
batch_script_dir = RunBatch.batch_script_directory(my_batch)
if not os.path.isdir(batch_script_dir):
os.makedirs(batch_script_dir)
batch_script_path = RunBatch.batch_script_path(my_batch, sql_script)
sql_script_file = open(batch_script_path, "w")
try:
sql_script_file.writelines(table_lines)
for file_name in image_files:
path_name = os.path.join(my_batch.data_dir, file_name)
sql_script_file.write(
"""SELECT 'Loading %(file_name)s into %(image_table)s';""" %
(globals()))
sql_script_file.write(
"""LOAD DATA LOCAL INFILE '%(path_name)s' REPLACE INTO TABLE %(image_table)s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"';
SHOW WARNINGS;
re_file_name = re.compile("^(.+?)[0-9]+_[0-9]+_(Image|Object).CSV$")
image_files = []
object_files = []
for file_name in os.listdir(my_batch.data_dir):
match = re_file_name.search(file_name)
if match:
if (image_prefix and match.groups(1)[0] == image_prefix
and match.groups(1)[1] == 'Image'):
image_files.append(file_name)
elif (object_prefix and
match.groups(1)[0] == object_prefix and
match.groups(1)[1] == 'Object'):
object_files.append(file_name)
batch_script_file = RunBatch.batch_script_file(sql_script)
batch_script_dir = RunBatch.batch_script_directory(my_batch)
if not os.path.isdir(batch_script_dir):
os.makedirs(batch_script_dir)
batch_script_path = RunBatch.batch_script_path(my_batch, sql_script)
sql_script_file = open(batch_script_path, "w")
try:
sql_script_file.writelines(table_lines)
for file_name in image_files:
path_name = os.path.join(my_batch.data_dir, file_name)
sql_script_file.write("""SELECT 'Loading %(file_name)s into %(image_table)s';"""%(globals()))
sql_script_file.write("""LOAD DATA LOCAL INFILE '%(path_name)s' REPLACE INTO TABLE %(image_table)s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"';
SHOW WARNINGS;
"""%(globals()))
for file_name in object_files:
path_name = os.path.join(my_batch.data_dir, file_name)
