def submit(self, previous_job = None):
from xfel.command_line.cxi_mpi_submit import Script as submit_script
output_path = os.path.join(get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task), 'out')
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
identifier_string = self.get_identifier_string()
submit_phil_path = os.path.join(configs_dir, identifier_string + "_submit.phil")
target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil")
input_folder, expt_suffix, refl_suffix, _, _ = previous_job.get_output_files()
with open(target_phil_path, 'w') as f:
f.write("input.path=%s\n"%input_folder)
f.write("input.experiments_suffix=%s\n"%expt_suffix)
f.write("input.reflections_suffix=%s\n"%refl_suffix)
f.write("output.output_dir=%s\n"%output_path)
f.write("output.prefix=%s_%d\n"%(self.task.type, self.task.id))
f.write(self.task.parameters)
self.write_submit_phil(submit_phil_path, target_phil_path)
args = [submit_phil_path]
return submit_script().run(args)
def get_log_path(self):
from xfel.ui.db import get_run_path
import os
run_path = str(
get_run_path(self.app.params.output_folder, self.trial,
self.rungroup, self.run))
return os.path.join(run_path, "stdout", "log.out")
def get_output_files(self):
run_path = get_run_path(self.app.params.output_folder, self.trial,
self.rungroup, self.run, self.task)
return os.path.join(run_path,
'combine_experiments_t%03d' % self.trial.trial,
'intermediates',
"*reintegrated*"), '.expt', '.refl'
def submit(self, previous_job=None):
from xfel.command_line.striping import Script
from xfel.command_line.cxi_mpi_submit import get_submission_id
from libtbx import easy_run
configs_dir = os.path.join(settings_dir, "cfgs")
identifier_string = self.get_identifier_string()
target_phil_path = os.path.join(configs_dir,
identifier_string + "_params.phil")
with open(target_phil_path, 'w') as f:
f.write(self.task.parameters)
path = get_run_path(self.app.params.output_folder, self.trial,
self.rungroup, self.run, self.task)
os.mkdir(path)
arguments = """
mp.queue={}
mp.nproc={}
mp.method={}
{}
mp.use_mpi=False
striping.results_dir={}
striping.trial={}
striping.rungroup={}
striping.run={}
{}
striping.chunk_size=3000
striping.stripe=False
striping.dry_run=True
striping.output_folder={}
""".format(
self.app.params.mp.queue
if len(self.app.params.mp.queue) > 0 else None,
self.app.params.mp.nproc,
self.app.params.mp.method,
'\n'.join([
'mp.env_script={}'.format(p)
for p in self.app.params.mp.env_script
]),
self.app.params.output_folder,
self.trial.trial,
self.rungroup.id,
self.run.run,
target_phil_path,
path,
).split()
commands = Script(arguments).run()
submission_ids = []
for command in commands:
try:
result = easy_run.fully_buffered(command=command)
result.raise_if_errors()
except Exception as e:
if not "Warning: job being submitted without an AFS token." in str(
e):
raise e
submission_ids.append(
get_submission_id(result, self.app.params.mp.method))
return ",".join(submission_ids)
def delete(self, output_only=False):
job_folder = get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task)
if os.path.exists(job_folder):
print("Deleting job folder for job", self.id)
shutil.rmtree(job_folder)
else:
print("Cannot find job folder (%s)"%job_folder)
self.status = "DELETED"
def get_log_path(self):
run_path = get_run_path(self.app.params.output_folder, self.trial,
self.rungroup, self.run, self.task)
return os.path.join(
run_path, 'combine_experiments_t%03d' % self.trial.trial,
'intermediates', "combine_t%03d_rg%03d_chunk000.out" %
(self.trial.trial, self.rungroup.id)
) # XXX there can be multiple chunks or multiple clusters
def delete(self, output_only=False):
from xfel.ui.db import get_run_path
import os, shutil
if self.status not in finished_job_statuses:
print "Job is not finished (status = %s)" % self.status
return
if self.status == "DELETED":
return
job_folder = get_run_path(self.app.params.output_folder, self.trial,
self.rungroup, self.run)
if os.path.exists(job_folder):
print "Deleting job folder for job", self.id
shutil.rmtree(job_folder)
else:
print "Cannot find job folder (%s)" % job_folder
# Have to be careful to delete from the tables in the right order
tag = self.app.params.experiment_tag
def delete_and_commit(query):
cursor = self.app.execute_query(query, commit=True)
print "(%d)" % cursor.rowcount
print "Deleting cell_bin entries",
query = """DELETE cell_bin FROM `%s_cell_bin` cell_bin
JOIN `%s_crystal` crystal ON crystal.id = cell_bin.crystal_id
JOIN `%s_experiment` expr ON expr.crystal_id = crystal.id
JOIN `%s_imageset` imgset ON imgset.id = expr.imageset_id
JOIN `%s_imageset_event` ie_e ON ie_e.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d""" % (
tag, tag, tag, tag, tag, tag, self.run.id, self.trial.id,
self.rungroup.id)
delete_and_commit(query)
ids = {}
for item in "crystal", "beam", "detector":
print "Listing %s ids" % item,
query = """SELECT %s.id FROM `%s_%s` %s
JOIN `%s_experiment` expr ON expr.%s_id = %s.id
JOIN `%s_imageset` imgset ON imgset.id = expr.imageset_id
JOIN `%s_imageset_event` ie_e ON ie_e.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d""" % (
item, tag, item, item, tag, item, item, tag, tag, tag,
self.run.id, self.trial.id, self.rungroup.id)
cursor = self.app.execute_query(query)
item_ids = ["%d" % i[0] for i in cursor.fetchall()]
print "(%d)" % len(item_ids)
ids[item] = ",".join(item_ids)
if len(self.trial.isoforms) == 0:
print "Listing bin entries",
query = """SELECT bin.id FROM `%s_bin` bin
JOIN `%s_cell` cell ON bin.cell_id = cell.id
JOIN `%s_crystal` crystal ON crystal.cell_id = cell.id
JOIN `%s_experiment` expr ON expr.crystal_id = crystal.id
JOIN `%s_imageset` imgset ON imgset.id = expr.imageset_id
JOIN `%s_imageset_event` ie_e ON ie_e.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d
AND cell.trial_id is NULL""" % (
tag, tag, tag, tag, tag, tag, tag, self.run.id, self.trial.id,
self.rungroup.id)
cursor = self.app.execute_query(query)
item_ids = ["%d" % i[0] for i in cursor.fetchall()]
print "(%d)" % len(item_ids)
bin_ids = ",".join(item_ids)
print "Listing cell entries",
query = """SELECT cell.id FROM `%s_cell` cell
JOIN `%s_crystal` crystal ON crystal.cell_id = cell.id
JOIN `%s_experiment` expr ON expr.crystal_id = crystal.id
JOIN `%s_imageset` imgset ON imgset.id = expr.imageset_id
JOIN `%s_imageset_event` ie_e ON ie_e.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d
AND cell.trial_id IS NULL""" % (tag, tag, tag, tag, tag, tag,
self.run.id, self.trial.id,
self.rungroup.id)
cursor = self.app.execute_query(query)
item_ids = ["%d" % i[0] for i in cursor.fetchall()]
print "(%d)" % len(item_ids)
cell_ids = ",".join(item_ids)
print "Deleting experiment entries",
query = """DELETE expr FROM `%s_experiment` expr
JOIN `%s_imageset` imgset ON imgset.id = expr.imageset_id
JOIN `%s_imageset_event` ie_e ON ie_e.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d""" % (
tag, tag, tag, tag, self.run.id, self.trial.id, self.rungroup.id)
delete_and_commit(query)
for item in "crystal", "beam", "detector":
if len(ids[item]) > 0:
print "Deleting %s entries" % item,
query = """DELETE %s FROM `%s_%s` %s
WHERE %s.id IN (%s)""" % (item, tag, item, item, item,
ids[item])
delete_and_commit(query)
if len(self.trial.isoforms) == 0 and len(bin_ids) > 0:
print "Deleting bin entries",
query = """DELETE bin FROM `%s_bin` bin
WHERE bin.id IN (%s)""" % (tag, bin_ids)
delete_and_commit(query)
if len(self.trial.isoforms) == 0 and len(cell_ids) > 0:
print "Deleting cell entries",
query = """DELETE cell FROM `%s_cell` cell
WHERE cell.id IN (%s)""" % (tag, cell_ids)
delete_and_commit(query)
print "Listing imageset entries",
query = """SELECT imgset.id FROM `%s_imageset` imgset
JOIN `%s_imageset_event` ie_e ON ie_e.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d""" % (
tag, tag, tag, self.run.id, self.trial.id, self.rungroup.id)
cursor = self.app.execute_query(query)
item_ids = ["%d" % i[0] for i in cursor.fetchall()]
print "(%d)" % len(item_ids)
imageset_ids = ",".join(item_ids)
print "Deleting imageset_event entries",
query = """DELETE is_e FROM `%s_imageset_event` is_e
JOIN `%s_event` evt ON evt.id = is_e.event_id
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d""" % (
tag, tag, self.run.id, self.trial.id, self.rungroup.id)
delete_and_commit(query)
if len(imageset_ids) > 0:
print "Deleting imageset entries",
query = """DELETE imgset FROM `%s_imageset` imgset
WHERE imgset.id IN (%s)""" % (tag, imageset_ids)
delete_and_commit(query)
print "Deleting event entries",
query = """DELETE evt FROM `%s_event` evt
WHERE evt.run_id = %d AND evt.trial_id = %d AND evt.rungroup_id = %d""" % (
tag, self.run.id, self.trial.id, self.rungroup.id)
delete_and_commit(query)
self.status = "DELETED"
def get_log_path(self): run_path = get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run) return os.path.join(run_path, "stdout", "log.out")
def get_output_files(self): run_path = get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task) return os.path.join(run_path, 'out'), ".expt", ".refl", None, None
def submit(self, previous_job = None):
from xfel.command_line.striping import Script
from xfel.command_line.cxi_mpi_submit import get_submission_id
from libtbx import easy_run
configs_dir = os.path.join(settings_dir, "cfgs")
identifier_string = self.get_identifier_string()
target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil")
with open(target_phil_path, 'w') as f:
if self.task.parameters:
f.write(self.task.parameters)
path = get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task)
os.mkdir(path)
arguments = """
mp.queue={}
mp.nnodes={}
mp.nproc_per_node={}
mp.method={}
{}
{}
mp.use_mpi=False
mp.mpi_command={}
{}
striping.results_dir={}
striping.trial={}
striping.rungroup={}
striping.run={}
{}
striping.chunk_size=200
striping.stripe=False
striping.dry_run=True
striping.output_folder={}
reintegration.integration.lookup.mask={}
mp.local.include_mp_in_command=False
""".format(self.app.params.mp.queue if len(self.app.params.mp.queue) > 0 else None,
1,#self.app.params.mp.nproc,
self.app.params.mp.nproc_per_node,
self.app.params.mp.method,
'\n'.join(['mp.env_script={}'.format(p) for p in self.app.params.mp.env_script if p]),
'\n'.join(['mp.phenix_script={}'.format(p) for p in self.app.params.mp.phenix_script if p]),
self.app.params.mp.mpi_command,
"\n".join(["extra_options={}".format(opt) for opt in self.app.params.mp.extra_options]),
self.app.params.output_folder,
self.trial.trial,
self.rungroup.id,
self.run.run,
target_phil_path,
path,
self.rungroup.untrusted_pixel_mask_path,
).split()
try:
commands = Script(arguments).run()
except Exception as e:
if 'no DIALS integration results found' in str(e):
print("No DIALS integration results found")
self.status = "EXIT"
return
else: raise
submission_ids = []
if self.app.params.mp.method == 'local':
self.status = "RUNNING"
for command in commands:
try:
result = easy_run.fully_buffered(command=command)
result.raise_if_errors()
except Exception as e:
if not "Warning: job being submitted without an AFS token." in str(e):
raise e
submission_ids.append(get_submission_id(result, self.app.params.mp.method))
if self.app.params.mp.method == 'local':
self.status = "DONE"
else:
return ",".join(submission_ids)
def get_output_files(self): run_path = str(get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run)) return os.path.join(run_path, 'out'), '_integrated.expt', '_integrated.refl', None, None
def get_log_path(self): from xfel.ui.db import get_run_path import os run_path = str(get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run)) return os.path.join(run_path, "stdout", "log.out")