def submit_job(params, dbobj, trial_id, trial, rungroup_id, run, config_path): assert os.path.exists(config_path) submit_phil_path = os.path.join(params.config_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil"%(params.experiment, params.experiment_tag, run, trial, rungroup_id)) template = open(os.path.join(libtbx.env.find_in_repositories("xfel/xpp/cfgs"), "submit.phil")) phil = open(submit_phil_path, "w") d = dict(dry_run = params.dry_run, cfg = config_path, experiment = params.experiment, run_num = run, trial = trial, rungroup = rungroup_id, output_dir = params.output_dir, nproc = params.mp.nproc, queue = params.mp.queue ) for line in template.readlines(): phil.write(line.format(**d)) template.close() phil.close() from xfel.command_line.cxi_mpi_submit import Script as submit_script submit_script().run([submit_phil_path])
def submit_job(params, dbobj, trial_id, trial, rungroup_id, run, config_path): assert os.path.exists(config_path) submit_phil_path = os.path.join( params.config_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" % (params.experiment, params.experiment_tag, run, trial, rungroup_id)) template = open( os.path.join(libtbx.env.find_in_repositories("xfel/xpp/cfgs"), "submit.phil")) phil = open(submit_phil_path, "w") d = dict(dry_run=params.dry_run, cfg=config_path, experiment=params.experiment, run_num=run, trial=trial, rungroup=rungroup_id, output_dir=params.output_dir, nproc=params.mp.nproc, queue=params.mp.queue) for line in template.readlines(): phil.write(line.format(**d)) template.close() phil.close() from xfel.command_line.cxi_mpi_submit import Script as submit_script submit_script().run([submit_phil_path])
def submit(self, previous_job = None): from xfel.command_line.cxi_mpi_submit import Script as submit_script output_path = os.path.join(get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task), 'out') configs_dir = os.path.join(settings_dir, "cfgs") if not os.path.exists(configs_dir): os.makedirs(configs_dir) identifier_string = self.get_identifier_string() submit_phil_path = os.path.join(configs_dir, identifier_string + "_submit.phil") target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil") input_folder, expt_suffix, refl_suffix, _, _ = previous_job.get_output_files() with open(target_phil_path, 'w') as f: f.write("input.path=%s\n"%input_folder) f.write("input.experiments_suffix=%s\n"%expt_suffix) f.write("input.reflections_suffix=%s\n"%refl_suffix) f.write("output.output_dir=%s\n"%output_path) f.write("output.prefix=%s_%d\n"%(self.task.type, self.task.id)) f.write(self.task.parameters) self.write_submit_phil(submit_phil_path, target_phil_path) args = [submit_phil_path] return submit_script().run(args)
def submit_job(app, job): import os, libtbx.load_env from xfel.ui import settings_dir configs_dir = os.path.join(settings_dir, "cfgs") if not os.path.exists(configs_dir): os.makedirs(configs_dir) target_phil_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) backend = ['labelit', 'dials'][['cxi.xtc_process', 'cctbx.xfel.xtc_process' ].index(app.params.dispatcher)] phil_str = job.trial.target_phil_str if job.rungroup.extra_phil_str is not None: phil_str += "\n" + job.rungroup.extra_phil_str if backend == 'dials': from xfel.command_line.xtc_process import phil_scope from iotbx.phil import parse trial_params = phil_scope.fetch(parse(phil_str)).extract() image_format = trial_params.format.file_format = job.rungroup.format assert image_format in ['cbf', 'pickle'] if image_format == 'cbf': if "rayonix" in job.rungroup.detector_address.lower(): mode = trial_params.format.cbf.mode = "rayonix" elif "cspad" in job.rungroup.detector_address.lower(): mode = trial_params.format.cbf.mode = "cspad" else: assert False, "Couldn't figure out what kind of detector is specified by address %s" % job.rungroup.detector_address else: image_format = 'pickle' if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or backend == 'labelit' or image_format == 'pickle': config_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) else: config_path = None # Dictionary for formating the submit phil and, if used, the labelit cfg file d = dict( # Generally for the LABELIT backend or image pickles address=job.rungroup.detector_address, default_calib_dir=libtbx.env.find_in_repositories( "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"), dark_avg_path=job.rungroup.dark_avg_path, dark_stddev_path=job.rungroup.dark_stddev_path, untrusted_pixel_mask_path=job.rungroup.untrusted_pixel_mask_path, detz_parameter=job.rungroup.detz_parameter, gain_map_path=job.rungroup.gain_map_path, gain_mask_level=job.rungroup.gain_mask_level, beamx=job.rungroup.beamx, beamy=job.rungroup.beamy, energy=job.rungroup.energy, binning=job.rungroup.binning, two_theta_low=job.rungroup.two_theta_low, two_theta_high=job.rungroup.two_theta_high, # Generally for job submission dry_run=app.params.dry_run, dispatcher=app.params.dispatcher, cfg=config_path, experiment=app.params.experiment, run_num=job.run.run, output_dir=app.params.output_folder, use_ffb=app.params.use_ffb, # Generally for both trial=job.trial.trial, rungroup=job.rungroup.rungroup_id, experiment_tag=app.params.experiment_tag, calib_dir=job.rungroup.calib_dir, nproc=app.params.mp.nproc, queue=app.params.mp.queue, target=target_phil_path, host=app.params.db.host, dbname=app.params.db.name, user=app.params.db.user, ) if app.params.db.password is not None and len(app.params.db.password) == 0: d['password'] = None else: d['password'] = app.params.db.password phil = open(target_phil_path, "w") if backend == 'dials': if trial_params.format.file_format == "cbf": trial_params.input.address = job.rungroup.detector_address trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter trial_params.format.cbf.override_energy = job.rungroup.energy trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path if mode == 'cspad': trial_params.format.cbf.cspad.gain_mask_value = job.rungroup.gain_mask_level elif mode == 'rayonix': trial_params.format.cbf.rayonix.bin_size = job.rungroup.binning trial_params.format.cbf.rayonix.override_beam_x = job.rungroup.beamx trial_params.format.cbf.rayonix.override_beam_y = job.rungroup.beamy trial_params.dispatch.process_percent = job.trial.process_percent if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None: trial_params.radial_average.enable = True trial_params.radial_average.show_plots = False trial_params.radial_average.verbose = False trial_params.radial_average.output_bins = False trial_params.radial_average.two_theta_low = job.rungroup.two_theta_low trial_params.radial_average.two_theta_high = job.rungroup.two_theta_high working_phil = phil_scope.format(python_object=trial_params) diff_phil = phil_scope.fetch_diff(source=working_phil) phil.write(diff_phil.as_str()) elif backend == 'labelit': phil.write(phil_str) else: assert False phil.close() if config_path is not None: if backend == 'dials': d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will # will be used during dials processing directly config_str = "[psana]\n" if job.rungroup.calib_dir is not None: config_str += "calib-dir=%s\n" % job.rungroup.calib_dir modules = [] if job.rungroup.config_str is not None: for line in job.rungroup.config_str.split("\n"): if line.startswith('['): modules.append(line.lstrip('[').rstrip(']')) if backend == 'labelit': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend( ['my_ana_pkg.mod_hitfind:index', 'my_ana_pkg.mod_dump:index']) elif image_format == 'pickle': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend(['my_ana_pkg.mod_image_dict']) if app.params.dump_shots: modules.insert(0, 'my_ana_pkg.mod_dump:shot') if len(modules) > 0: config_str += "modules = %s\n" % (" ".join(modules)) if job.rungroup.config_str is not None: config_str += job.rungroup.config_str + "\n" if backend == 'labelit' or image_format == 'pickle': d['address'] = d['address'].replace('.', '-').replace( ':', '|') # old style address if backend == 'labelit': template = open( os.path.join( libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg")) elif image_format == 'pickle': template = open( os.path.join( libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg")) for line in template.readlines(): config_str += line.format(**d) template.close() d['address'] = job.rungroup.detector_address cfg = open(config_path, 'w') cfg.write(config_str) cfg.close() if backend == 'dials': d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path submit_phil_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) template = open( os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "submit.phil")) phil = open(submit_phil_path, "w") if backend == 'labelit': d['target'] = None # any target phil will be in mod_hitfind for line in template.readlines(): phil.write(line.format(**d)) d['target'] = target_phil_path template.close() phil.close() from xfel.command_line.cxi_mpi_submit import Script as submit_script return submit_script().run([submit_phil_path])
def submit(self, previous_job = None): import libtbx.load_env configs_dir = os.path.join(settings_dir, "cfgs") if not os.path.exists(configs_dir): os.makedirs(configs_dir) identifier_string = self.get_identifier_string() target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil") dispatcher = self.app.params.dispatcher phil_str = self.trial.target_phil_str if phil_str is None: phil_str = "" if self.rungroup.extra_phil_str is not None: phil_str += "\n" + self.rungroup.extra_phil_str from xfel.ui import load_phil_scope_from_dispatcher if dispatcher == "cxi.xtc_process": image_format = 'pickle' else: orig_phil_scope = load_phil_scope_from_dispatcher(dispatcher) if os.path.isfile(dispatcher): dispatcher = 'libtbx.python ' + dispatcher from iotbx.phil import parse if self.rungroup.two_theta_low is not None or self.rungroup.two_theta_high is not None: override_str = """ radial_average { enable = True show_plots = False verbose = False output_bins = False mask = %s } """%(self.rungroup.untrusted_pixel_mask_path) phil_scope = orig_phil_scope.fetch(parse(override_str)) else: phil_scope = orig_phil_scope trial_params = phil_scope.fetch(parse(phil_str)).extract() image_format = self.rungroup.format mode = "other" if self.app.params.facility.name == 'lcls': if "rayonix" in self.rungroup.detector_address.lower(): mode = "rayonix" elif "cspad" in self.rungroup.detector_address.lower(): mode = "cspad" elif "jungfrau" in self.rungroup.detector_address.lower(): mode = "jungfrau" if hasattr(trial_params, 'format'): trial_params.format.file_format = image_format trial_params.format.cbf.mode = mode if self.rungroup.calib_dir is not None or self.rungroup.config_str is not None or dispatcher == 'cxi.xtc_process' or image_format == 'pickle': config_path = os.path.join(configs_dir, identifier_string + ".cfg") else: config_path = None if hasattr(trial_params.dispatch, 'process_percent'): trial_params.dispatch.process_percent = self.trial.process_percent # Dictionary for formating the submit phil and, if used, the labelit cfg file d = dict( # Generally for the LABELIT backend or image pickles address = self.rungroup.detector_address, default_calib_dir = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"), dark_avg_path = self.rungroup.dark_avg_path, dark_stddev_path = self.rungroup.dark_stddev_path, untrusted_pixel_mask_path = self.rungroup.untrusted_pixel_mask_path, detz_parameter = self.rungroup.detz_parameter, gain_map_path = self.rungroup.gain_map_path, gain_mask_level = self.rungroup.gain_mask_level, beamx = self.rungroup.beamx, beamy = self.rungroup.beamy, energy = self.rungroup.energy, binning = self.rungroup.binning, two_theta_low = self.rungroup.two_theta_low, two_theta_high = self.rungroup.two_theta_high, # Generally for job submission dry_run = self.app.params.dry_run, dispatcher = dispatcher, cfg = config_path, experiment = self.app.params.facility.lcls.experiment, # LCLS specific parameter run_num = self.run.run, output_dir = self.app.params.output_folder, use_ffb = self.app.params.facility.lcls.use_ffb, # LCLS specific parameter # Generally for both trial = self.trial.trial, rungroup = self.rungroup.rungroup_id, experiment_tag = self.app.params.experiment_tag, calib_dir = self.rungroup.calib_dir, nproc = self.app.params.mp.nproc, nnodes = self.app.params.mp.nnodes_index or self.app.params.mp.nnodes, nproc_per_node = self.app.params.mp.nproc_per_node, queue = self.app.params.mp.queue or None, env_script = self.app.params.mp.env_script[0] if self.app.params.mp.env_script is not None and len(self.app.params.mp.env_script) > 0 and len(self.app.params.mp.env_script[0]) > 0 else None, phenix_script = self.app.params.mp.phenix_script[0] if self.app.params.mp.phenix_script is not None and len(self.app.params.mp.phenix_script) > 0 and len(self.app.params.mp.phenix_script[0]) > 0 else None, method = self.app.params.mp.method, wall_time = self.app.params.mp.wall_time, htcondor_executable_path = self.app.params.mp.htcondor.executable_path, nersc_shifter_image = self.app.params.mp.shifter.shifter_image, sbatch_script_template = self.app.params.mp.shifter.sbatch_script_template, srun_script_template = self.app.params.mp.shifter.srun_script_template, nersc_partition = self.app.params.mp.shifter.partition, nersc_jobname = self.app.params.mp.shifter.jobname, nersc_project = self.app.params.mp.shifter.project, nersc_constraint = self.app.params.mp.shifter.constraint, nersc_reservation = self.app.params.mp.shifter.reservation, nersc_staging = self.app.params.mp.shifter.staging, target = target_phil_path, host = self.app.params.db.host, dbname = self.app.params.db.name, user = self.app.params.db.user, port = self.app.params.db.port, # always use mpi for 'lcls' use_mpi = self.app.params.mp.method != 'local' or (self.app.params.mp.method == 'local' and self.app.params.facility.name == 'lcls'), mpi_command = self.app.params.mp.mpi_command, extra_options = "\n".join(["extra_options = %s"%opt for opt in self.app.params.mp.extra_options]), ) if self.app.params.mp.method == 'sge': d['use_mpi'] = False if self.app.params.db.password is not None and len(self.app.params.db.password) == 0: d['password'] = None else: d['password'] = self.app.params.db.password phil = open(target_phil_path, "w") if dispatcher == 'cxi.xtc_process': phil.write(phil_str) else: extra_scope = None if hasattr(trial_params, 'format'): if image_format == "cbf": trial_params.input.address = self.rungroup.detector_address trial_params.format.cbf.detz_offset = self.rungroup.detz_parameter trial_params.format.cbf.override_energy = self.rungroup.energy trial_params.format.cbf.invalid_pixel_mask = self.rungroup.untrusted_pixel_mask_path if mode == 'cspad': trial_params.format.cbf.cspad.gain_mask_value = self.rungroup.gain_mask_level elif mode == 'rayonix': trial_params.format.cbf.rayonix.bin_size = self.rungroup.binning trial_params.format.cbf.rayonix.override_beam_x = self.rungroup.beamx trial_params.format.cbf.rayonix.override_beam_y = self.rungroup.beamy if trial_params.input.known_orientations_folder is not None: trial_params.input.known_orientations_folder = trial_params.input.known_orientations_folder.format(run=self.run.run) else: if trial_params.spotfinder.lookup.mask is None: trial_params.spotfinder.lookup.mask = self.rungroup.untrusted_pixel_mask_path if trial_params.integration.lookup.mask is None: trial_params.integration.lookup.mask = self.rungroup.untrusted_pixel_mask_path if self.app.params.facility.name == 'lcls': locator_path = os.path.join(configs_dir, identifier_string + ".loc") locator = open(locator_path, 'w') locator.write("experiment=%s\n"%self.app.params.facility.lcls.experiment) # LCLS specific parameter locator.write("run=%s\n"%self.run.run) locator.write("detector_address=%s\n"%self.rungroup.detector_address) if self.rungroup.wavelength_offset: locator.write("wavelength_offset=%s\n"%self.rungroup.wavelength_offset) if self.rungroup.spectrum_eV_per_pixel: locator.write("spectrum_eV_per_pixel=%s\n"%self.rungroup.spectrum_eV_per_pixel) if self.rungroup.spectrum_eV_offset: locator.write("spectrum_eV_offset=%s\n"%self.rungroup.spectrum_eV_offset) if self.app.params.facility.lcls.use_ffb: locator.write("use_ffb=True\n") if mode == 'rayonix': from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.rungroup.binning) extra_scope = parse("geometry { detector { panel { origin = (%f, %f, %f) } } }"%(-self.rungroup.beamx * pixel_size, self.rungroup.beamy * pixel_size, -self.rungroup.detz_parameter)) locator.write("rayonix.bin_size=%s\n"%self.rungroup.binning) elif mode == 'cspad': locator.write("cspad.detz_offset=%s\n"%self.rungroup.detz_parameter) locator.close() d['locator'] = locator_path else: d['locator'] = None if self.rungroup.two_theta_low is not None or self.rungroup.two_theta_high is not None: try: trial_params.radial_average.two_theta_low = self.rungroup.two_theta_low trial_params.radial_average.two_theta_high = self.rungroup.two_theta_high except AttributeError: pass # not all dispatchers support radial averaging working_phil = phil_scope.format(python_object=trial_params) if extra_scope: working_phil = working_phil.fetch(extra_scope) diff_phil = orig_phil_scope.fetch_diff(source=working_phil) phil.write(diff_phil.as_str()) phil.close() if config_path is not None: if dispatcher != 'cxi.xtc_process': d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will # will be used during dials processing directly config_str = "[psana]\n" if self.rungroup.calib_dir is not None: config_str += "calib-dir=%s\n"%self.rungroup.calib_dir modules = [] if self.rungroup.config_str is not None: for line in self.rungroup.config_str.split("\n"): if line.startswith('['): modules.append(line.lstrip('[').rstrip(']')) if dispatcher == 'cxi.xtc_process': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend(['my_ana_pkg.mod_hitfind:index','my_ana_pkg.mod_dump:index']) elif image_format == 'pickle': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend(['my_ana_pkg.mod_image_dict']) if self.app.params.facility.lcls.dump_shots: modules.insert(0, 'my_ana_pkg.mod_dump:shot') if len(modules) > 0: config_str += "modules = %s\n"%(" ".join(modules)) if self.rungroup.config_str is not None: config_str += self.rungroup.config_str + "\n" if dispatcher == 'cxi.xtc_process' or image_format == 'pickle': d['address'] = d['address'].replace('.','-').replace(':','|') # old style address if dispatcher == 'cxi.xtc_process': template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg")) elif image_format == 'pickle': template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg")) for line in template.readlines(): config_str += line.format(**d) template.close() d['address'] = self.rungroup.detector_address cfg = open(config_path, 'w') cfg.write(config_str) cfg.close() if dispatcher != 'cxi.xtc_process': d['untrusted_pixel_mask_path'] = self.rungroup.untrusted_pixel_mask_path submit_phil_path = os.path.join(configs_dir, identifier_string + "_submit.phil") submit_root = libtbx.env.find_in_repositories("xfel/ui/db/cfgs") if dispatcher in ['cxi.xtc_process', 'cctbx.xfel.xtc_process']: template = open(os.path.join(submit_root, "submit_xtc_process.phil")) else: test_root = os.path.join(submit_root, "submit_" + dispatcher + ".phil") if os.path.exists(test_root): template = open(test_root) else: if hasattr(trial_params, 'format'): template = open(os.path.join(submit_root, "submit_xtc_process.phil")) else: template = open(os.path.join(submit_root, "submit_xfel_process.phil")) phil = open(submit_phil_path, "w") if dispatcher == 'cxi.xtc_process': d['target'] = None # any target phil will be in mod_hitfind for line in template.readlines(): phil.write(line.format(**d)) d['target'] = target_phil_path template.close() phil.close() from xfel.command_line.cxi_mpi_submit import Script as submit_script args = [submit_phil_path] if self.app.params.facility.name not in ['lcls']: args.append(self.run.path) return submit_script().run(args)
def submit_job(app, job): import os, libtbx.load_env from xfel.ui import settings_dir configs_dir = os.path.join(settings_dir, "cfgs") if not os.path.exists(configs_dir): os.makedirs(configs_dir) target_phil_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) dispatcher = app.params.dispatcher phil_str = job.trial.target_phil_str if job.rungroup.extra_phil_str is not None: phil_str += "\n" + job.rungroup.extra_phil_str from xfel.ui import known_dials_dispatchers if dispatcher in known_dials_dispatchers: import importlib orig_phil_scope = importlib.import_module( known_dials_dispatchers[dispatcher]).phil_scope from iotbx.phil import parse if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None: override_str = """ radial_average { enable = True show_plots = False verbose = False output_bins = False } """ phil_scope = orig_phil_scope.fetch(parse(override_str)) else: phil_scope = orig_phil_scope trial_params = phil_scope.fetch(parse(phil_str)).extract() image_format = job.rungroup.format assert image_format in ['cbf', 'pickle'] if image_format == 'cbf': if "rayonix" in job.rungroup.detector_address.lower(): mode = "rayonix" elif "cspad" in job.rungroup.detector_address.lower(): mode = "cspad" elif "jungfrau" in job.rungroup.detector_address.lower(): mode = "jungfrau" else: assert False, "Couldn't figure out what kind of detector is specified by address %s" % job.rungroup.detector_address if dispatcher == 'cctbx.xfel.xtc_process': trial_params.format.file_format = image_format trial_params.format.cbf.mode = mode elif dispatcher == "cxi.xtc_process": image_format = 'pickle' else: raise RuntimeError("Unknown dispatcher: %s" % dispatcher) if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or dispatcher == 'cxi.xtc_process' or image_format == 'pickle': config_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) else: config_path = None # Dictionary for formating the submit phil and, if used, the labelit cfg file d = dict( # Generally for the LABELIT backend or image pickles address=job.rungroup.detector_address, default_calib_dir=libtbx.env.find_in_repositories( "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"), dark_avg_path=job.rungroup.dark_avg_path, dark_stddev_path=job.rungroup.dark_stddev_path, untrusted_pixel_mask_path=job.rungroup.untrusted_pixel_mask_path, detz_parameter=job.rungroup.detz_parameter, gain_map_path=job.rungroup.gain_map_path, gain_mask_level=job.rungroup.gain_mask_level, beamx=job.rungroup.beamx, beamy=job.rungroup.beamy, energy=job.rungroup.energy, binning=job.rungroup.binning, two_theta_low=job.rungroup.two_theta_low, two_theta_high=job.rungroup.two_theta_high, # Generally for job submission dry_run=app.params.dry_run, dispatcher=app.params.dispatcher, cfg=config_path, experiment=app.params.experiment, run_num=job.run.run, output_dir=app.params.output_folder, use_ffb=app.params.use_ffb, # Generally for both trial=job.trial.trial, rungroup=job.rungroup.rungroup_id, experiment_tag=app.params.experiment_tag, calib_dir=job.rungroup.calib_dir, nproc=app.params.mp.nproc, queue=app.params.mp.queue, target=target_phil_path, host=app.params.db.host, dbname=app.params.db.name, user=app.params.db.user, ) if app.params.db.password is not None and len(app.params.db.password) == 0: d['password'] = None else: d['password'] = app.params.db.password phil = open(target_phil_path, "w") if dispatcher == 'cxi.xtc_process': phil.write(phil_str) elif dispatcher in known_dials_dispatchers: extra_scope = None if dispatcher == 'cctbx.xfel.xtc_process': if image_format == "cbf": trial_params.input.address = job.rungroup.detector_address trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter trial_params.format.cbf.override_energy = job.rungroup.energy trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path if mode == 'cspad': trial_params.format.cbf.cspad.gain_mask_value = job.rungroup.gain_mask_level elif mode == 'rayonix': trial_params.format.cbf.rayonix.bin_size = job.rungroup.binning trial_params.format.cbf.rayonix.override_beam_x = job.rungroup.beamx trial_params.format.cbf.rayonix.override_beam_y = job.rungroup.beamy trial_params.dispatch.process_percent = job.trial.process_percent if trial_params.input.known_orientations_folder is not None: trial_params.input.known_orientations_folder = trial_params.input.known_orientations_folder.format( run=job.run.run) else: trial_params.spotfinder.lookup.mask = job.rungroup.untrusted_pixel_mask_path trial_params.integration.lookup.mask = job.rungroup.untrusted_pixel_mask_path locator_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d.loc" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) locator = open(locator_path, 'w') locator.write("experiment=%s\n" % app.params.experiment) locator.write("run=%d\n" % job.run.run) locator.write("detector_address=%s\n" % job.rungroup.detector_address) if image_format == "cbf": if mode == 'rayonix': from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size( job.rungroup.binning) fast, slow = rayonix_tbx.get_rayonix_detector_dimensions( job.rungroup.binning) extra_scope = parse( "geometry { detector { panel { origin = (%f, %f, %f) } } }" % (-job.rungroup.beamx * pixel_size, job.rungroup.beamy * pixel_size, -job.rungroup.detz_parameter)) locator.write("rayonix.bin_size=%s\n" % job.rungroup.binning) locator.close() d['locator'] = locator_path if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None: try: trial_params.radial_average.two_theta_low = job.rungroup.two_theta_low trial_params.radial_average.two_theta_high = job.rungroup.two_theta_high except AttributeError: pass # not all dispatchers support radial averaging working_phil = phil_scope.format(python_object=trial_params) if extra_scope: working_phil = working_phil.fetch(extra_scope) diff_phil = orig_phil_scope.fetch_diff(source=working_phil) phil.write(diff_phil.as_str()) phil.close() if config_path is not None: if dispatcher != 'cxi.xtc_process': d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will # will be used during dials processing directly config_str = "[psana]\n" if job.rungroup.calib_dir is not None: config_str += "calib-dir=%s\n" % job.rungroup.calib_dir modules = [] if job.rungroup.config_str is not None: for line in job.rungroup.config_str.split("\n"): if line.startswith('['): modules.append(line.lstrip('[').rstrip(']')) if dispatcher == 'cxi.xtc_process': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend( ['my_ana_pkg.mod_hitfind:index', 'my_ana_pkg.mod_dump:index']) elif image_format == 'pickle': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend(['my_ana_pkg.mod_image_dict']) if app.params.dump_shots: modules.insert(0, 'my_ana_pkg.mod_dump:shot') if len(modules) > 0: config_str += "modules = %s\n" % (" ".join(modules)) if job.rungroup.config_str is not None: config_str += job.rungroup.config_str + "\n" if dispatcher == 'cxi.xtc_process' or image_format == 'pickle': d['address'] = d['address'].replace('.', '-').replace( ':', '|') # old style address if dispatcher == 'cxi.xtc_process': template = open( os.path.join( libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg")) elif image_format == 'pickle': template = open( os.path.join( libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg")) for line in template.readlines(): config_str += line.format(**d) template.close() d['address'] = job.rungroup.detector_address cfg = open(config_path, 'w') cfg.write(config_str) cfg.close() if dispatcher != 'cxi.xtc_process': d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path submit_phil_path = os.path.join( configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" % (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) submit_root = libtbx.env.find_in_repositories("xfel/ui/db/cfgs") if dispatcher in ['cxi.xtc_process', 'cctbx.xfel.xtc_process']: template = open(os.path.join(submit_root, "submit_xtc_process.phil")) elif dispatcher == 'cctbx.xfel.process': template = open(os.path.join(submit_root, "submit_xfel_process.phil")) else: test_root = os.path.join(submit_root, "submit_" + dispatcher + ".phil") if os.path.exists(test_root): template = open(test_root) else: template = open(os.path.joins(submit_root, "submit.phil")) phil = open(submit_phil_path, "w") if dispatcher == 'cxi.xtc_process': d['target'] = None # any target phil will be in mod_hitfind for line in template.readlines(): phil.write(line.format(**d)) d['target'] = target_phil_path template.close() phil.close() from xfel.command_line.cxi_mpi_submit import Script as submit_script return submit_script().run([submit_phil_path])
def submit_job(app, job): import os, libtbx.load_env from xfel.ui import settings_dir configs_dir = os.path.join(settings_dir, "cfgs") if not os.path.exists(configs_dir): os.makedirs(configs_dir) target_phil_path = os.path.join(configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil"% (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) backend = ['labelit', 'dials'][['cxi.xtc_process', 'cctbx.xfel.xtc_process'].index(app.params.dispatcher)] phil_str = job.trial.target_phil_str if job.rungroup.extra_phil_str is not None: phil_str += "\n" + job.rungroup.extra_phil_str if backend == 'dials': from xfel.command_line.xtc_process import phil_scope from iotbx.phil import parse trial_params = phil_scope.fetch(parse(phil_str)).extract() image_format = trial_params.format.file_format assert image_format in ['cbf', 'pickle'] else: image_format = 'pickle' if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or backend == 'labelit' or image_format == 'pickle': config_path = os.path.join(configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg"% (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) else: config_path = None # Dictionary for formating the submit phil and, if used, the labelit cfg file d = dict( # Generally for the LABELIT backend or image pickles address = job.rungroup.detector_address, default_calib_dir = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"), dark_avg_path = job.rungroup.dark_avg_path, dark_stddev_path = job.rungroup.dark_stddev_path, untrusted_pixel_mask_path = job.rungroup.untrusted_pixel_mask_path, detz_parameter = job.rungroup.detz_parameter, gain_map_path = job.rungroup.gain_map_path, gain_mask_level = job.rungroup.gain_mask_level, beamx = job.rungroup.beamx, beamy = job.rungroup.beamy, energy = job.rungroup.energy, binning = job.rungroup.binning, two_theta_low = 12.5, # FIXME two_theta_high = 22.8, # FIXME # Generally for job submission dry_run = app.params.dry_run, dispatcher = app.params.dispatcher, cfg = config_path, experiment = app.params.experiment, run_num = job.run.run, output_dir = app.params.output_folder, use_ffb = app.params.use_ffb, # Generally for both trial = job.trial.trial, rungroup = job.rungroup.rungroup_id, experiment_tag = app.params.experiment_tag, calib_dir = job.rungroup.calib_dir, nproc = app.params.mp.nproc, queue = app.params.mp.queue, target = target_phil_path, host = app.params.db.host, dbname = app.params.db.name, user = app.params.db.user, ) if app.params.db.password is not None and len(app.params.db.password) == 0: d['password'] = None else: d['password'] = app.params.db.password phil = open(target_phil_path, "w") if backend == 'dials': if trial_params.format.file_format == "cbf": trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter trial_params.format.cbf.override_energy = job.rungroup.energy trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path trial_params.format.cbf.gain_mask_value = job.rungroup.gain_mask_level trial_params.dispatch.process_percent = job.trial.process_percent working_phil = phil_scope.format(python_object=trial_params) diff_phil = phil_scope.fetch_diff(source=working_phil) phil.write(diff_phil.as_str()) elif backend == 'labelit': phil.write(phil_str) else: assert False phil.close() if config_path is not None: if backend == 'dials': d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will # will be used during dials processing directly config_str = "[psana]\n" if job.rungroup.calib_dir is not None: config_str += "calib-dir=%s\n"%job.rungroup.calib_dir modules = [] if job.rungroup.config_str is not None: for line in job.rungroup.config_str.split("\n"): if line.startswith('['): modules.append(line.lstrip('[').rstrip(']')) if backend == 'labelit': modules.insert(0, 'my_ana_pkg.mod_radial_average') modules.extend(['my_ana_pkg.mod_hitfind:index','my_ana_pkg.mod_dump:index']) elif image_format == 'pickle': modules.extend(['my_ana_pkg.mod_image_dict']) if app.params.dump_shots: modules.insert(0, 'my_ana_pkg.mod_dump:shot') if len(modules) > 0: config_str += "modules = %s\n"%(" ".join(modules)) if job.rungroup.config_str is not None: config_str += job.rungroup.config_str + "\n" if backend == 'labelit' or image_format == 'pickle': d['address'] = d['address'].replace('.','-').replace(':','|') # old style address if backend == 'labelit': template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg")) elif image_format == 'pickle': template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg")) for line in template.readlines(): config_str += line.format(**d) template.close() d['address'] = job.rungroup.detector_address cfg = open(config_path, 'w') cfg.write(config_str) cfg.close() if backend == 'dials': d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path submit_phil_path = os.path.join(configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil"% (app.params.experiment, app.params.experiment_tag, job.run.run, job.trial.trial, job.rungroup.id)) template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "submit.phil")) phil = open(submit_phil_path, "w") if backend == 'labelit': d['target'] = None # any target phil will be in mod_hitfind for line in template.readlines(): phil.write(line.format(**d)) d['target'] = target_phil_path template.close() phil.close() from xfel.command_line.cxi_mpi_submit import Script as submit_script return submit_script().run([submit_phil_path])