def submit_job(params, dbobj, trial_id, trial, rungroup_id, run, config_path):
assert os.path.exists(config_path)
submit_phil_path = os.path.join(params.config_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil"%(params.experiment, params.experiment_tag, run, trial, rungroup_id))
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/xpp/cfgs"), "submit.phil"))
phil = open(submit_phil_path, "w")
d = dict(dry_run = params.dry_run,
cfg = config_path,
experiment = params.experiment,
run_num = run,
trial = trial,
rungroup = rungroup_id,
output_dir = params.output_dir,
nproc = params.mp.nproc,
queue = params.mp.queue
)
for line in template.readlines():
phil.write(line.format(**d))
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
submit_script().run([submit_phil_path])
def submit_job(params, dbobj, trial_id, trial, rungroup_id, run, config_path):
assert os.path.exists(config_path)
submit_phil_path = os.path.join(
params.config_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" %
(params.experiment, params.experiment_tag, run, trial, rungroup_id))
template = open(
os.path.join(libtbx.env.find_in_repositories("xfel/xpp/cfgs"),
"submit.phil"))
phil = open(submit_phil_path, "w")
d = dict(dry_run=params.dry_run,
cfg=config_path,
experiment=params.experiment,
run_num=run,
trial=trial,
rungroup=rungroup_id,
output_dir=params.output_dir,
nproc=params.mp.nproc,
queue=params.mp.queue)
for line in template.readlines():
phil.write(line.format(**d))
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
submit_script().run([submit_phil_path])
def submit(self, previous_job = None):
from xfel.command_line.cxi_mpi_submit import Script as submit_script
output_path = os.path.join(get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task), 'out')
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
identifier_string = self.get_identifier_string()
submit_phil_path = os.path.join(configs_dir, identifier_string + "_submit.phil")
target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil")
input_folder, expt_suffix, refl_suffix, _, _ = previous_job.get_output_files()
with open(target_phil_path, 'w') as f:
f.write("input.path=%s\n"%input_folder)
f.write("input.experiments_suffix=%s\n"%expt_suffix)
f.write("input.reflections_suffix=%s\n"%refl_suffix)
f.write("output.output_dir=%s\n"%output_path)
f.write("output.prefix=%s_%d\n"%(self.task.type, self.task.id))
f.write(self.task.parameters)
self.write_submit_phil(submit_phil_path, target_phil_path)
args = [submit_phil_path]
return submit_script().run(args)
def submit_job(app, job):
import os, libtbx.load_env
from xfel.ui import settings_dir
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
target_phil_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
backend = ['labelit',
'dials'][['cxi.xtc_process', 'cctbx.xfel.xtc_process'
].index(app.params.dispatcher)]
phil_str = job.trial.target_phil_str
if job.rungroup.extra_phil_str is not None:
phil_str += "\n" + job.rungroup.extra_phil_str
if backend == 'dials':
from xfel.command_line.xtc_process import phil_scope
from iotbx.phil import parse
trial_params = phil_scope.fetch(parse(phil_str)).extract()
image_format = trial_params.format.file_format = job.rungroup.format
assert image_format in ['cbf', 'pickle']
if image_format == 'cbf':
if "rayonix" in job.rungroup.detector_address.lower():
mode = trial_params.format.cbf.mode = "rayonix"
elif "cspad" in job.rungroup.detector_address.lower():
mode = trial_params.format.cbf.mode = "cspad"
else:
assert False, "Couldn't figure out what kind of detector is specified by address %s" % job.rungroup.detector_address
else:
image_format = 'pickle'
if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or backend == 'labelit' or image_format == 'pickle':
config_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
else:
config_path = None
# Dictionary for formating the submit phil and, if used, the labelit cfg file
d = dict(
# Generally for the LABELIT backend or image pickles
address=job.rungroup.detector_address,
default_calib_dir=libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"),
dark_avg_path=job.rungroup.dark_avg_path,
dark_stddev_path=job.rungroup.dark_stddev_path,
untrusted_pixel_mask_path=job.rungroup.untrusted_pixel_mask_path,
detz_parameter=job.rungroup.detz_parameter,
gain_map_path=job.rungroup.gain_map_path,
gain_mask_level=job.rungroup.gain_mask_level,
beamx=job.rungroup.beamx,
beamy=job.rungroup.beamy,
energy=job.rungroup.energy,
binning=job.rungroup.binning,
two_theta_low=job.rungroup.two_theta_low,
two_theta_high=job.rungroup.two_theta_high,
# Generally for job submission
dry_run=app.params.dry_run,
dispatcher=app.params.dispatcher,
cfg=config_path,
experiment=app.params.experiment,
run_num=job.run.run,
output_dir=app.params.output_folder,
use_ffb=app.params.use_ffb,
# Generally for both
trial=job.trial.trial,
rungroup=job.rungroup.rungroup_id,
experiment_tag=app.params.experiment_tag,
calib_dir=job.rungroup.calib_dir,
nproc=app.params.mp.nproc,
queue=app.params.mp.queue,
target=target_phil_path,
host=app.params.db.host,
dbname=app.params.db.name,
user=app.params.db.user,
)
if app.params.db.password is not None and len(app.params.db.password) == 0:
d['password'] = None
else:
d['password'] = app.params.db.password
phil = open(target_phil_path, "w")
if backend == 'dials':
if trial_params.format.file_format == "cbf":
trial_params.input.address = job.rungroup.detector_address
trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter
trial_params.format.cbf.override_energy = job.rungroup.energy
trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path
if mode == 'cspad':
trial_params.format.cbf.cspad.gain_mask_value = job.rungroup.gain_mask_level
elif mode == 'rayonix':
trial_params.format.cbf.rayonix.bin_size = job.rungroup.binning
trial_params.format.cbf.rayonix.override_beam_x = job.rungroup.beamx
trial_params.format.cbf.rayonix.override_beam_y = job.rungroup.beamy
trial_params.dispatch.process_percent = job.trial.process_percent
if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None:
trial_params.radial_average.enable = True
trial_params.radial_average.show_plots = False
trial_params.radial_average.verbose = False
trial_params.radial_average.output_bins = False
trial_params.radial_average.two_theta_low = job.rungroup.two_theta_low
trial_params.radial_average.two_theta_high = job.rungroup.two_theta_high
working_phil = phil_scope.format(python_object=trial_params)
diff_phil = phil_scope.fetch_diff(source=working_phil)
phil.write(diff_phil.as_str())
elif backend == 'labelit':
phil.write(phil_str)
else:
assert False
phil.close()
if config_path is not None:
if backend == 'dials':
d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will
# will be used during dials processing directly
config_str = "[psana]\n"
if job.rungroup.calib_dir is not None:
config_str += "calib-dir=%s\n" % job.rungroup.calib_dir
modules = []
if job.rungroup.config_str is not None:
for line in job.rungroup.config_str.split("\n"):
if line.startswith('['):
modules.append(line.lstrip('[').rstrip(']'))
if backend == 'labelit':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(
['my_ana_pkg.mod_hitfind:index', 'my_ana_pkg.mod_dump:index'])
elif image_format == 'pickle':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_image_dict'])
if app.params.dump_shots:
modules.insert(0, 'my_ana_pkg.mod_dump:shot')
if len(modules) > 0:
config_str += "modules = %s\n" % (" ".join(modules))
if job.rungroup.config_str is not None:
config_str += job.rungroup.config_str + "\n"
if backend == 'labelit' or image_format == 'pickle':
d['address'] = d['address'].replace('.', '-').replace(
':', '|') # old style address
if backend == 'labelit':
template = open(
os.path.join(
libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"index_all.cfg"))
elif image_format == 'pickle':
template = open(
os.path.join(
libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"image_dict.cfg"))
for line in template.readlines():
config_str += line.format(**d)
template.close()
d['address'] = job.rungroup.detector_address
cfg = open(config_path, 'w')
cfg.write(config_str)
cfg.close()
if backend == 'dials':
d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path
submit_phil_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
template = open(
os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"submit.phil"))
phil = open(submit_phil_path, "w")
if backend == 'labelit':
d['target'] = None # any target phil will be in mod_hitfind
for line in template.readlines():
phil.write(line.format(**d))
d['target'] = target_phil_path
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
return submit_script().run([submit_phil_path])
def submit(self, previous_job = None):
import libtbx.load_env
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
identifier_string = self.get_identifier_string()
target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil")
dispatcher = self.app.params.dispatcher
phil_str = self.trial.target_phil_str
if phil_str is None: phil_str = ""
if self.rungroup.extra_phil_str is not None:
phil_str += "\n" + self.rungroup.extra_phil_str
from xfel.ui import load_phil_scope_from_dispatcher
if dispatcher == "cxi.xtc_process":
image_format = 'pickle'
else:
orig_phil_scope = load_phil_scope_from_dispatcher(dispatcher)
if os.path.isfile(dispatcher):
dispatcher = 'libtbx.python ' + dispatcher
from iotbx.phil import parse
if self.rungroup.two_theta_low is not None or self.rungroup.two_theta_high is not None:
override_str = """
radial_average {
enable = True
show_plots = False
verbose = False
output_bins = False
mask = %s
}
"""%(self.rungroup.untrusted_pixel_mask_path)
phil_scope = orig_phil_scope.fetch(parse(override_str))
else:
phil_scope = orig_phil_scope
trial_params = phil_scope.fetch(parse(phil_str)).extract()
image_format = self.rungroup.format
mode = "other"
if self.app.params.facility.name == 'lcls':
if "rayonix" in self.rungroup.detector_address.lower():
mode = "rayonix"
elif "cspad" in self.rungroup.detector_address.lower():
mode = "cspad"
elif "jungfrau" in self.rungroup.detector_address.lower():
mode = "jungfrau"
if hasattr(trial_params, 'format'):
trial_params.format.file_format = image_format
trial_params.format.cbf.mode = mode
if self.rungroup.calib_dir is not None or self.rungroup.config_str is not None or dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
config_path = os.path.join(configs_dir, identifier_string + ".cfg")
else:
config_path = None
if hasattr(trial_params.dispatch, 'process_percent'):
trial_params.dispatch.process_percent = self.trial.process_percent
# Dictionary for formating the submit phil and, if used, the labelit cfg file
d = dict(
# Generally for the LABELIT backend or image pickles
address = self.rungroup.detector_address,
default_calib_dir = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"),
dark_avg_path = self.rungroup.dark_avg_path,
dark_stddev_path = self.rungroup.dark_stddev_path,
untrusted_pixel_mask_path = self.rungroup.untrusted_pixel_mask_path,
detz_parameter = self.rungroup.detz_parameter,
gain_map_path = self.rungroup.gain_map_path,
gain_mask_level = self.rungroup.gain_mask_level,
beamx = self.rungroup.beamx,
beamy = self.rungroup.beamy,
energy = self.rungroup.energy,
binning = self.rungroup.binning,
two_theta_low = self.rungroup.two_theta_low,
two_theta_high = self.rungroup.two_theta_high,
# Generally for job submission
dry_run = self.app.params.dry_run,
dispatcher = dispatcher,
cfg = config_path,
experiment = self.app.params.facility.lcls.experiment, # LCLS specific parameter
run_num = self.run.run,
output_dir = self.app.params.output_folder,
use_ffb = self.app.params.facility.lcls.use_ffb, # LCLS specific parameter
# Generally for both
trial = self.trial.trial,
rungroup = self.rungroup.rungroup_id,
experiment_tag = self.app.params.experiment_tag,
calib_dir = self.rungroup.calib_dir,
nproc = self.app.params.mp.nproc,
nnodes = self.app.params.mp.nnodes_index or self.app.params.mp.nnodes,
nproc_per_node = self.app.params.mp.nproc_per_node,
queue = self.app.params.mp.queue or None,
env_script = self.app.params.mp.env_script[0] if self.app.params.mp.env_script is not None and len(self.app.params.mp.env_script) > 0 and len(self.app.params.mp.env_script[0]) > 0 else None,
phenix_script = self.app.params.mp.phenix_script[0] if self.app.params.mp.phenix_script is not None and len(self.app.params.mp.phenix_script) > 0 and len(self.app.params.mp.phenix_script[0]) > 0 else None,
method = self.app.params.mp.method,
wall_time = self.app.params.mp.wall_time,
htcondor_executable_path = self.app.params.mp.htcondor.executable_path,
nersc_shifter_image = self.app.params.mp.shifter.shifter_image,
sbatch_script_template = self.app.params.mp.shifter.sbatch_script_template,
srun_script_template = self.app.params.mp.shifter.srun_script_template,
nersc_partition = self.app.params.mp.shifter.partition,
nersc_jobname = self.app.params.mp.shifter.jobname,
nersc_project = self.app.params.mp.shifter.project,
nersc_constraint = self.app.params.mp.shifter.constraint,
nersc_reservation = self.app.params.mp.shifter.reservation,
nersc_staging = self.app.params.mp.shifter.staging,
target = target_phil_path,
host = self.app.params.db.host,
dbname = self.app.params.db.name,
user = self.app.params.db.user,
port = self.app.params.db.port,
# always use mpi for 'lcls'
use_mpi = self.app.params.mp.method != 'local' or (self.app.params.mp.method == 'local' and self.app.params.facility.name == 'lcls'),
mpi_command = self.app.params.mp.mpi_command,
extra_options = "\n".join(["extra_options = %s"%opt for opt in self.app.params.mp.extra_options]),
)
if self.app.params.mp.method == 'sge':
d['use_mpi'] = False
if self.app.params.db.password is not None and len(self.app.params.db.password) == 0:
d['password'] = None
else:
d['password'] = self.app.params.db.password
phil = open(target_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
phil.write(phil_str)
else:
extra_scope = None
if hasattr(trial_params, 'format'):
if image_format == "cbf":
trial_params.input.address = self.rungroup.detector_address
trial_params.format.cbf.detz_offset = self.rungroup.detz_parameter
trial_params.format.cbf.override_energy = self.rungroup.energy
trial_params.format.cbf.invalid_pixel_mask = self.rungroup.untrusted_pixel_mask_path
if mode == 'cspad':
trial_params.format.cbf.cspad.gain_mask_value = self.rungroup.gain_mask_level
elif mode == 'rayonix':
trial_params.format.cbf.rayonix.bin_size = self.rungroup.binning
trial_params.format.cbf.rayonix.override_beam_x = self.rungroup.beamx
trial_params.format.cbf.rayonix.override_beam_y = self.rungroup.beamy
if trial_params.input.known_orientations_folder is not None:
trial_params.input.known_orientations_folder = trial_params.input.known_orientations_folder.format(run=self.run.run)
else:
if trial_params.spotfinder.lookup.mask is None:
trial_params.spotfinder.lookup.mask = self.rungroup.untrusted_pixel_mask_path
if trial_params.integration.lookup.mask is None:
trial_params.integration.lookup.mask = self.rungroup.untrusted_pixel_mask_path
if self.app.params.facility.name == 'lcls':
locator_path = os.path.join(configs_dir, identifier_string + ".loc")
locator = open(locator_path, 'w')
locator.write("experiment=%s\n"%self.app.params.facility.lcls.experiment) # LCLS specific parameter
locator.write("run=%s\n"%self.run.run)
locator.write("detector_address=%s\n"%self.rungroup.detector_address)
if self.rungroup.wavelength_offset:
locator.write("wavelength_offset=%s\n"%self.rungroup.wavelength_offset)
if self.rungroup.spectrum_eV_per_pixel:
locator.write("spectrum_eV_per_pixel=%s\n"%self.rungroup.spectrum_eV_per_pixel)
if self.rungroup.spectrum_eV_offset:
locator.write("spectrum_eV_offset=%s\n"%self.rungroup.spectrum_eV_offset)
if self.app.params.facility.lcls.use_ffb:
locator.write("use_ffb=True\n")
if mode == 'rayonix':
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.rungroup.binning)
extra_scope = parse("geometry { detector { panel { origin = (%f, %f, %f) } } }"%(-self.rungroup.beamx * pixel_size,
self.rungroup.beamy * pixel_size,
-self.rungroup.detz_parameter))
locator.write("rayonix.bin_size=%s\n"%self.rungroup.binning)
elif mode == 'cspad':
locator.write("cspad.detz_offset=%s\n"%self.rungroup.detz_parameter)
locator.close()
d['locator'] = locator_path
else:
d['locator'] = None
if self.rungroup.two_theta_low is not None or self.rungroup.two_theta_high is not None:
try:
trial_params.radial_average.two_theta_low = self.rungroup.two_theta_low
trial_params.radial_average.two_theta_high = self.rungroup.two_theta_high
except AttributeError:
pass # not all dispatchers support radial averaging
working_phil = phil_scope.format(python_object=trial_params)
if extra_scope:
working_phil = working_phil.fetch(extra_scope)
diff_phil = orig_phil_scope.fetch_diff(source=working_phil)
phil.write(diff_phil.as_str())
phil.close()
if config_path is not None:
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will
# will be used during dials processing directly
config_str = "[psana]\n"
if self.rungroup.calib_dir is not None:
config_str += "calib-dir=%s\n"%self.rungroup.calib_dir
modules = []
if self.rungroup.config_str is not None:
for line in self.rungroup.config_str.split("\n"):
if line.startswith('['):
modules.append(line.lstrip('[').rstrip(']'))
if dispatcher == 'cxi.xtc_process':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_hitfind:index','my_ana_pkg.mod_dump:index'])
elif image_format == 'pickle':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_image_dict'])
if self.app.params.facility.lcls.dump_shots:
modules.insert(0, 'my_ana_pkg.mod_dump:shot')
if len(modules) > 0:
config_str += "modules = %s\n"%(" ".join(modules))
if self.rungroup.config_str is not None:
config_str += self.rungroup.config_str + "\n"
if dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
d['address'] = d['address'].replace('.','-').replace(':','|') # old style address
if dispatcher == 'cxi.xtc_process':
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg"))
elif image_format == 'pickle':
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg"))
for line in template.readlines():
config_str += line.format(**d)
template.close()
d['address'] = self.rungroup.detector_address
cfg = open(config_path, 'w')
cfg.write(config_str)
cfg.close()
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = self.rungroup.untrusted_pixel_mask_path
submit_phil_path = os.path.join(configs_dir, identifier_string + "_submit.phil")
submit_root = libtbx.env.find_in_repositories("xfel/ui/db/cfgs")
if dispatcher in ['cxi.xtc_process', 'cctbx.xfel.xtc_process']:
template = open(os.path.join(submit_root, "submit_xtc_process.phil"))
else:
test_root = os.path.join(submit_root, "submit_" + dispatcher + ".phil")
if os.path.exists(test_root):
template = open(test_root)
else:
if hasattr(trial_params, 'format'):
template = open(os.path.join(submit_root, "submit_xtc_process.phil"))
else:
template = open(os.path.join(submit_root, "submit_xfel_process.phil"))
phil = open(submit_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
d['target'] = None # any target phil will be in mod_hitfind
for line in template.readlines():
phil.write(line.format(**d))
d['target'] = target_phil_path
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
args = [submit_phil_path]
if self.app.params.facility.name not in ['lcls']:
args.append(self.run.path)
return submit_script().run(args)
def submit_job(app, job):
import os, libtbx.load_env
from xfel.ui import settings_dir
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
target_phil_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
dispatcher = app.params.dispatcher
phil_str = job.trial.target_phil_str
if job.rungroup.extra_phil_str is not None:
phil_str += "\n" + job.rungroup.extra_phil_str
from xfel.ui import known_dials_dispatchers
if dispatcher in known_dials_dispatchers:
import importlib
orig_phil_scope = importlib.import_module(
known_dials_dispatchers[dispatcher]).phil_scope
from iotbx.phil import parse
if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None:
override_str = """
radial_average {
enable = True
show_plots = False
verbose = False
output_bins = False
}
"""
phil_scope = orig_phil_scope.fetch(parse(override_str))
else:
phil_scope = orig_phil_scope
trial_params = phil_scope.fetch(parse(phil_str)).extract()
image_format = job.rungroup.format
assert image_format in ['cbf', 'pickle']
if image_format == 'cbf':
if "rayonix" in job.rungroup.detector_address.lower():
mode = "rayonix"
elif "cspad" in job.rungroup.detector_address.lower():
mode = "cspad"
elif "jungfrau" in job.rungroup.detector_address.lower():
mode = "jungfrau"
else:
assert False, "Couldn't figure out what kind of detector is specified by address %s" % job.rungroup.detector_address
if dispatcher == 'cctbx.xfel.xtc_process':
trial_params.format.file_format = image_format
trial_params.format.cbf.mode = mode
elif dispatcher == "cxi.xtc_process":
image_format = 'pickle'
else:
raise RuntimeError("Unknown dispatcher: %s" % dispatcher)
if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
config_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
else:
config_path = None
# Dictionary for formating the submit phil and, if used, the labelit cfg file
d = dict(
# Generally for the LABELIT backend or image pickles
address=job.rungroup.detector_address,
default_calib_dir=libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"),
dark_avg_path=job.rungroup.dark_avg_path,
dark_stddev_path=job.rungroup.dark_stddev_path,
untrusted_pixel_mask_path=job.rungroup.untrusted_pixel_mask_path,
detz_parameter=job.rungroup.detz_parameter,
gain_map_path=job.rungroup.gain_map_path,
gain_mask_level=job.rungroup.gain_mask_level,
beamx=job.rungroup.beamx,
beamy=job.rungroup.beamy,
energy=job.rungroup.energy,
binning=job.rungroup.binning,
two_theta_low=job.rungroup.two_theta_low,
two_theta_high=job.rungroup.two_theta_high,
# Generally for job submission
dry_run=app.params.dry_run,
dispatcher=app.params.dispatcher,
cfg=config_path,
experiment=app.params.experiment,
run_num=job.run.run,
output_dir=app.params.output_folder,
use_ffb=app.params.use_ffb,
# Generally for both
trial=job.trial.trial,
rungroup=job.rungroup.rungroup_id,
experiment_tag=app.params.experiment_tag,
calib_dir=job.rungroup.calib_dir,
nproc=app.params.mp.nproc,
queue=app.params.mp.queue,
target=target_phil_path,
host=app.params.db.host,
dbname=app.params.db.name,
user=app.params.db.user,
)
if app.params.db.password is not None and len(app.params.db.password) == 0:
d['password'] = None
else:
d['password'] = app.params.db.password
phil = open(target_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
phil.write(phil_str)
elif dispatcher in known_dials_dispatchers:
extra_scope = None
if dispatcher == 'cctbx.xfel.xtc_process':
if image_format == "cbf":
trial_params.input.address = job.rungroup.detector_address
trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter
trial_params.format.cbf.override_energy = job.rungroup.energy
trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path
if mode == 'cspad':
trial_params.format.cbf.cspad.gain_mask_value = job.rungroup.gain_mask_level
elif mode == 'rayonix':
trial_params.format.cbf.rayonix.bin_size = job.rungroup.binning
trial_params.format.cbf.rayonix.override_beam_x = job.rungroup.beamx
trial_params.format.cbf.rayonix.override_beam_y = job.rungroup.beamy
trial_params.dispatch.process_percent = job.trial.process_percent
if trial_params.input.known_orientations_folder is not None:
trial_params.input.known_orientations_folder = trial_params.input.known_orientations_folder.format(
run=job.run.run)
else:
trial_params.spotfinder.lookup.mask = job.rungroup.untrusted_pixel_mask_path
trial_params.integration.lookup.mask = job.rungroup.untrusted_pixel_mask_path
locator_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d.loc" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
locator = open(locator_path, 'w')
locator.write("experiment=%s\n" % app.params.experiment)
locator.write("run=%d\n" % job.run.run)
locator.write("detector_address=%s\n" %
job.rungroup.detector_address)
if image_format == "cbf":
if mode == 'rayonix':
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
job.rungroup.binning)
fast, slow = rayonix_tbx.get_rayonix_detector_dimensions(
job.rungroup.binning)
extra_scope = parse(
"geometry { detector { panel { origin = (%f, %f, %f) } } }"
%
(-job.rungroup.beamx * pixel_size, job.rungroup.beamy *
pixel_size, -job.rungroup.detz_parameter))
locator.write("rayonix.bin_size=%s\n" %
job.rungroup.binning)
locator.close()
d['locator'] = locator_path
if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None:
try:
trial_params.radial_average.two_theta_low = job.rungroup.two_theta_low
trial_params.radial_average.two_theta_high = job.rungroup.two_theta_high
except AttributeError:
pass # not all dispatchers support radial averaging
working_phil = phil_scope.format(python_object=trial_params)
if extra_scope:
working_phil = working_phil.fetch(extra_scope)
diff_phil = orig_phil_scope.fetch_diff(source=working_phil)
phil.write(diff_phil.as_str())
phil.close()
if config_path is not None:
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will
# will be used during dials processing directly
config_str = "[psana]\n"
if job.rungroup.calib_dir is not None:
config_str += "calib-dir=%s\n" % job.rungroup.calib_dir
modules = []
if job.rungroup.config_str is not None:
for line in job.rungroup.config_str.split("\n"):
if line.startswith('['):
modules.append(line.lstrip('[').rstrip(']'))
if dispatcher == 'cxi.xtc_process':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(
['my_ana_pkg.mod_hitfind:index', 'my_ana_pkg.mod_dump:index'])
elif image_format == 'pickle':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_image_dict'])
if app.params.dump_shots:
modules.insert(0, 'my_ana_pkg.mod_dump:shot')
if len(modules) > 0:
config_str += "modules = %s\n" % (" ".join(modules))
if job.rungroup.config_str is not None:
config_str += job.rungroup.config_str + "\n"
if dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
d['address'] = d['address'].replace('.', '-').replace(
':', '|') # old style address
if dispatcher == 'cxi.xtc_process':
template = open(
os.path.join(
libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"index_all.cfg"))
elif image_format == 'pickle':
template = open(
os.path.join(
libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"image_dict.cfg"))
for line in template.readlines():
config_str += line.format(**d)
template.close()
d['address'] = job.rungroup.detector_address
cfg = open(config_path, 'w')
cfg.write(config_str)
cfg.close()
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path
submit_phil_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
submit_root = libtbx.env.find_in_repositories("xfel/ui/db/cfgs")
if dispatcher in ['cxi.xtc_process', 'cctbx.xfel.xtc_process']:
template = open(os.path.join(submit_root, "submit_xtc_process.phil"))
elif dispatcher == 'cctbx.xfel.process':
template = open(os.path.join(submit_root, "submit_xfel_process.phil"))
else:
test_root = os.path.join(submit_root, "submit_" + dispatcher + ".phil")
if os.path.exists(test_root):
template = open(test_root)
else:
template = open(os.path.joins(submit_root, "submit.phil"))
phil = open(submit_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
d['target'] = None # any target phil will be in mod_hitfind
for line in template.readlines():
phil.write(line.format(**d))
d['target'] = target_phil_path
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
return submit_script().run([submit_phil_path])
def submit_job(app, job):
import os, libtbx.load_env
from xfel.ui import settings_dir
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
target_phil_path = os.path.join(configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil"%
(app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id))
backend = ['labelit', 'dials'][['cxi.xtc_process', 'cctbx.xfel.xtc_process'].index(app.params.dispatcher)]
phil_str = job.trial.target_phil_str
if job.rungroup.extra_phil_str is not None:
phil_str += "\n" + job.rungroup.extra_phil_str
if backend == 'dials':
from xfel.command_line.xtc_process import phil_scope
from iotbx.phil import parse
trial_params = phil_scope.fetch(parse(phil_str)).extract()
image_format = trial_params.format.file_format
assert image_format in ['cbf', 'pickle']
else:
image_format = 'pickle'
if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or backend == 'labelit' or image_format == 'pickle':
config_path = os.path.join(configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg"%
(app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id))
else:
config_path = None
# Dictionary for formating the submit phil and, if used, the labelit cfg file
d = dict(
# Generally for the LABELIT backend or image pickles
address = job.rungroup.detector_address,
default_calib_dir = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"),
dark_avg_path = job.rungroup.dark_avg_path,
dark_stddev_path = job.rungroup.dark_stddev_path,
untrusted_pixel_mask_path = job.rungroup.untrusted_pixel_mask_path,
detz_parameter = job.rungroup.detz_parameter,
gain_map_path = job.rungroup.gain_map_path,
gain_mask_level = job.rungroup.gain_mask_level,
beamx = job.rungroup.beamx,
beamy = job.rungroup.beamy,
energy = job.rungroup.energy,
binning = job.rungroup.binning,
two_theta_low = 12.5, # FIXME
two_theta_high = 22.8, # FIXME
# Generally for job submission
dry_run = app.params.dry_run,
dispatcher = app.params.dispatcher,
cfg = config_path,
experiment = app.params.experiment,
run_num = job.run.run,
output_dir = app.params.output_folder,
use_ffb = app.params.use_ffb,
# Generally for both
trial = job.trial.trial,
rungroup = job.rungroup.rungroup_id,
experiment_tag = app.params.experiment_tag,
calib_dir = job.rungroup.calib_dir,
nproc = app.params.mp.nproc,
queue = app.params.mp.queue,
target = target_phil_path,
host = app.params.db.host,
dbname = app.params.db.name,
user = app.params.db.user,
)
if app.params.db.password is not None and len(app.params.db.password) == 0:
d['password'] = None
else:
d['password'] = app.params.db.password
phil = open(target_phil_path, "w")
if backend == 'dials':
if trial_params.format.file_format == "cbf":
trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter
trial_params.format.cbf.override_energy = job.rungroup.energy
trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path
trial_params.format.cbf.gain_mask_value = job.rungroup.gain_mask_level
trial_params.dispatch.process_percent = job.trial.process_percent
working_phil = phil_scope.format(python_object=trial_params)
diff_phil = phil_scope.fetch_diff(source=working_phil)
phil.write(diff_phil.as_str())
elif backend == 'labelit':
phil.write(phil_str)
else:
assert False
phil.close()
if config_path is not None:
if backend == 'dials':
d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will
# will be used during dials processing directly
config_str = "[psana]\n"
if job.rungroup.calib_dir is not None:
config_str += "calib-dir=%s\n"%job.rungroup.calib_dir
modules = []
if job.rungroup.config_str is not None:
for line in job.rungroup.config_str.split("\n"):
if line.startswith('['):
modules.append(line.lstrip('[').rstrip(']'))
if backend == 'labelit':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_hitfind:index','my_ana_pkg.mod_dump:index'])
elif image_format == 'pickle':
modules.extend(['my_ana_pkg.mod_image_dict'])
if app.params.dump_shots:
modules.insert(0, 'my_ana_pkg.mod_dump:shot')
if len(modules) > 0:
config_str += "modules = %s\n"%(" ".join(modules))
if job.rungroup.config_str is not None:
config_str += job.rungroup.config_str + "\n"
if backend == 'labelit' or image_format == 'pickle':
d['address'] = d['address'].replace('.','-').replace(':','|') # old style address
if backend == 'labelit':
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg"))
elif image_format == 'pickle':
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg"))
for line in template.readlines():
config_str += line.format(**d)
template.close()
d['address'] = job.rungroup.detector_address
cfg = open(config_path, 'w')
cfg.write(config_str)
cfg.close()
if backend == 'dials':
d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path
submit_phil_path = os.path.join(configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil"%
(app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id))
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "submit.phil"))
phil = open(submit_phil_path, "w")
if backend == 'labelit':
d['target'] = None # any target phil will be in mod_hitfind
for line in template.readlines():
phil.write(line.format(**d))
d['target'] = target_phil_path
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
return submit_script().run([submit_phil_path])
