def setup(self): """Set everything up...""" # check arguments are all ascii Debug.write("Start parsing command line: " + str(sys.argv)) for token in sys.argv: try: token.encode("ascii") except UnicodeDecodeError: raise RuntimeError("non-ascii characters in input") self._argv = copy.deepcopy(sys.argv) replacements = { "-2d": "pipeline=2d", "-2di": "pipeline=2di", "-3d": "pipeline=3d", "-3di": "pipeline=3di", "-3dii": "pipeline=3dii", "-3dd": "pipeline=3dd", "-dials": "pipeline=dials", "-quick": "dials.fast_mode=true", "-failover": "failover=true", "-small_molecule": "small_molecule=true", } for k, v in replacements.iteritems(): if k in self._argv: print( "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % (k, v)) self._argv[self._argv.index(k)] = v if "-atom" in self._argv: idx = self._argv.index("-atom") element = self._argv[idx + 1] self._argv[idx:idx + 2] = ["atom=%s" % element] print( "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % (element, element)) # first of all try to interpret arguments as phil parameters/files from xia2.Handlers.Phil import master_phil from libtbx.phil import command_line cmd_line = command_line.argument_interpreter(master_phil=master_phil) working_phil, self._argv = cmd_line.process_and_fetch( args=self._argv, custom_processor="collect_remaining") PhilIndex.merge_phil(working_phil) try: params = PhilIndex.get_python_object() except RuntimeError as e: raise Sorry(e) # sanity check / interpret Auto in input from libtbx import Auto if params.xia2.settings.input.atom is None: if params.xia2.settings.input.anomalous is Auto: PhilIndex.update("xia2.settings.input.anomalous=false") else: if params.xia2.settings.input.anomalous is False: raise Sorry( "Setting anomalous=false and atom type inconsistent") params.xia2.settings.input.anomalous = True PhilIndex.update("xia2.settings.input.anomalous=true") if params.xia2.settings.resolution.keep_all_reflections is Auto: if (params.xia2.settings.small_molecule is True and params.xia2.settings.resolution.d_min is None and params.xia2.settings.resolution.d_max is None): PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=true") else: PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=false") if params.xia2.settings.small_molecule is True: Debug.write("Small molecule selected") if params.xia2.settings.symmetry.chirality is None: PhilIndex.update("xia2.settings.symmetry.chirality=nonchiral") params = PhilIndex.get_python_object() # pipeline options self._read_pipeline() for (parameter, value) in ( ("project", params.xia2.settings.project), ("crystal", params.xia2.settings.crystal), ): validate_project_crystal_name(parameter, value) Debug.write("Project: %s" % params.xia2.settings.project) Debug.write("Crystal: %s" % params.xia2.settings.crystal) # FIXME add some consistency checks in here e.g. that there are # images assigned, there is a lattice assigned if cell constants # are given and so on params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing from xia2.Handlers.Environment import get_number_cpus if mp_params.mode == "parallel": if mp_params.type == "qsub": if which("qsub") is None: raise Sorry("qsub not available") if mp_params.njob is Auto: mp_params.njob = get_number_cpus() if mp_params.nproc is Auto: mp_params.nproc = 1 elif mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() elif mp_params.mode == "serial": if mp_params.type == "qsub": if which("qsub") is None: raise Sorry("qsub not available") if mp_params.njob is Auto: mp_params.njob = 1 if mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() PhilIndex.update("xia2.settings.multiprocessing.njob=%d" % mp_params.njob) PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" % mp_params.nproc) params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing if mp_params.nproc > 1 and os.name == "nt": raise Sorry("nproc > 1 is not supported on Windows.") # #191 if params.xia2.settings.indexer is not None: add_preference("indexer", params.xia2.settings.indexer) if params.xia2.settings.refiner is not None: add_preference("refiner", params.xia2.settings.refiner) if params.xia2.settings.integrater is not None: add_preference("integrater", params.xia2.settings.integrater) if params.xia2.settings.scaler is not None: add_preference("scaler", params.xia2.settings.scaler) if params.xia2.settings.multi_sweep_indexing is Auto: if (params.xia2.settings.small_molecule is True and "dials" == params.xia2.settings.indexer): PhilIndex.update("xia2.settings.multi_sweep_indexing=True") else: PhilIndex.update("xia2.settings.multi_sweep_indexing=False") if (params.xia2.settings.multi_sweep_indexing is True and params.xia2.settings.multiprocessing.mode == "parallel"): Chatter.write( "Multi sweep indexing disabled:\nMSI is not available for parallel processing." ) PhilIndex.update("xia2.settings.multi_sweep_indexing=False") input_json = params.xia2.settings.input.json if input_json is not None and len(input_json): for json_file in input_json: assert os.path.isfile(json_file) load_experiments(json_file) reference_geometry = params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: reference_geometries = "\n".join([ "xia2.settings.input.reference_geometry=%s" % os.path.abspath(g) for g in params.xia2.settings.input.reference_geometry ]) Debug.write(reference_geometries) PhilIndex.update(reference_geometries) Debug.write("xia2.settings.trust_beam_centre=true") PhilIndex.update("xia2.settings.trust_beam_centre=true") params = PhilIndex.get_python_object() params = PhilIndex.get_python_object() if params.xia2.settings.input.xinfo is not None: xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() self.set_xinfo(xinfo_file) # issue #55 if not set ATOM in xinfo but anomalous=true or atom= set # on commandline, set here, should be idempotent if params.xia2.settings.input.anomalous is True: crystals = self._xinfo.get_crystals() for xname in crystals: xtal = crystals[xname] Debug.write("Setting anomalous for crystal %s" % xname) xtal.set_anomalous(True) else: xinfo_file = "%s/automatic.xinfo" % os.path.abspath(os.curdir) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() if params.dials.find_spots.phil_file is not None: PhilIndex.update( "dials.find_spots.phil_file=%s" % os.path.abspath(params.dials.find_spots.phil_file)) if params.dials.index.phil_file is not None: PhilIndex.update("dials.index.phil_file=%s" % os.path.abspath(params.dials.index.phil_file)) if params.dials.refine.phil_file is not None: PhilIndex.update("dials.refine.phil_file=%s" % os.path.abspath(params.dials.refine.phil_file)) if params.dials.integrate.phil_file is not None: PhilIndex.update("dials.integrate.phil_file=%s" % os.path.abspath(params.dials.integrate.phil_file)) if params.xds.index.xparm is not None: Flags.set_xparm(params.xds.index.xparm) if params.xds.index.xparm_ub is not None: Flags.set_xparm_ub(params.xds.index.xparm_ub) if params.xia2.settings.scale.freer_file is not None: freer_file = os.path.abspath(params.xia2.settings.scale.freer_file) if not os.path.exists(freer_file): raise RuntimeError("%s does not exist" % freer_file) from xia2.Modules.FindFreeFlag import FindFreeFlag column = FindFreeFlag(freer_file) Debug.write("FreeR_flag column in %s found: %s" % (freer_file, column)) PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file) if params.xia2.settings.scale.reference_reflection_file is not None: reference_reflection_file = os.path.abspath( params.xia2.settings.scale.reference_reflection_file) if not os.path.exists(reference_reflection_file): raise RuntimeError("%s does not exist" % reference_reflection_file) PhilIndex.update( "xia2.settings.scale.reference_reflection_file=%s" % reference_reflection_file) params = PhilIndex.get_python_object() datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image) for dataset in datasets: start_end = None # here we only care about ':' which are later than C:\ if ":" in dataset[3:]: tokens = dataset.split(":") # cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100 if len(tokens[0]) == 1: tokens = ["%s:%s" % (tokens[0], tokens[1])] + tokens[2:] if len(tokens) != 3: raise RuntimeError("/path/to/image_0001.cbf:start:end") dataset = tokens[0] start_end = int(tokens[1]), int(tokens[2]) from xia2.Applications.xia2setup import is_hd5f_name if os.path.exists(os.path.abspath(dataset)): dataset = os.path.abspath(dataset) else: directories = [os.getcwd()] + self._argv[1:] found = False for d in directories: if os.path.exists(os.path.join(d, dataset)): dataset = os.path.join(d, dataset) found = True break if not found: raise Sorry("Could not find %s in %s" % (dataset, " ".join(directories))) if is_hd5f_name(dataset): self._hdf5_master_files.append(dataset) if start_end: Debug.write("Image range: %d %d" % start_end) if dataset not in self._default_start_end: self._default_start_end[dataset] = [] self._default_start_end[dataset].append(start_end) else: Debug.write("No image range specified") else: template, directory = image2template_directory( os.path.abspath(dataset)) self._default_template.append(os.path.join( directory, template)) self._default_directory.append(directory) Debug.write("Interpreted from image %s:" % dataset) Debug.write("Template %s" % template) Debug.write("Directory %s" % directory) if start_end: Debug.write("Image range: %d %d" % start_end) key = os.path.join(directory, template) if key not in self._default_start_end: self._default_start_end[key] = [] self._default_start_end[key].append(start_end) else: Debug.write("No image range specified") # finally, check that all arguments were read and raise an exception # if any of them were nonsense. with open("xia2-working.phil", "wb") as f: f.write(PhilIndex.working_phil.as_str()) f.write( os.linesep ) # temporarily required for https://github.com/dials/dials/issues/522 with open("xia2-diff.phil", "wb") as f: f.write(PhilIndex.get_diff().as_str()) f.write( os.linesep ) # temporarily required for https://github.com/dials/dials/issues/522 Debug.write("\nDifference PHIL:") Debug.write(PhilIndex.get_diff().as_str(), strip=False) Debug.write("Working PHIL:") Debug.write(PhilIndex.working_phil.as_str(), strip=False) nonsense = "Unknown command-line options:" was_nonsense = False for j, argv in enumerate(self._argv): if j == 0: continue if argv[0] != "-" and "=" not in argv: continue if j not in self._understood: nonsense += " %s" % argv was_nonsense = True if was_nonsense: raise RuntimeError(nonsense)
def __init__(self): self._quick = False self._interactive = False self._ice = False self._egg = False self._uniform_sd = True self._chef = False self._mask = None self._reversephi = False self._no_lattice_test = False self._migrate_data = False self._trust_timestaps = False # XDS specific things - to help with handling tricky data sets self._xparm = None self._xparm_beam_vector = None self._xparm_rotation_axis = None self._xparm_origin = None self._xparm_a = None self._xparm_b = None self._xparm_c = None try: self._parallel = get_number_cpus() except: self._parallel = 0 self._xparallel = 0 self._start_end = None self._batch_scale = False # File from which to copy the FreeR_flag column self._freer_file = None # or alternatively the fraction, or total number of free # reflections self._free_fraction = None self._free_total = None # reference reflection file self._reference_reflection_file = None # these are development parameters for the XDS implementation self._z_min = 0.0 self._remove = True self._zero_dose = False self._relax = True # and these for the Mosflm / Aimless and perhaps XDS implementation self._scale_model = False self._scale_model_decay = False self._scale_model_modulation = False self._scale_model_absorption = False self._scale_model_partiality = False self._rmerge_target = 'low' # options to support the -spacegroup flag - the spacegroup is # set from this, the lattice and pointgroup derived from such self._spacegroup = None self._pointgroup = None self._lattice = None # aimless secondary correction self._aimless_secondary = 6 # resolution limit flags self._resolution_low = None self._resolution_high = None # and these are general rejection criteria self._rejection_threshold = 1.5 self._isigma = 1.0 self._misigma = 2.0 self._completeness = 0.0 self._rmerge = 0.0 self._cc_half = 0.0 self._microcrystal = False self._failover = False self._blend = False # are we working with small molecule data? self._small_molecule = False # flags relating to fixing specific bugs self._fixed_628 = False # ISPyB things self._ispyb_xml_out = None # header cache / json things self._hdr_in = None self._hdr_out = None # pickle output self._pickle = None # serialization of indexer/integrater state to/from json self._serialize_state = False return
def __init__(self, experiments, reflections, params): self._data_manager = DataManager(experiments, reflections) self._params = params if self._params.nproc is Auto: self._params.nproc = get_number_cpus() PhilIndex.params.xia2.settings.multiprocessing.nproc = self._params.nproc if self._params.identifiers is not None: self._data_manager.select(self._params.identifiers) if self._params.dose is not None: self._data_manager.filter_dose(*self._params.dose) if params.remove_profile_fitting_failures: reflections = self._data_manager.reflections profile_fitted_mask = reflections.get_flags( reflections.flags.integrated_prf ) keep_expts = [] for i, expt in enumerate(self._data_manager.experiments): if ( reflections.select(profile_fitted_mask) .select_on_experiment_identifiers([expt.identifier]) .size() ): keep_expts.append(expt.identifier) if len(keep_expts): logger.info( "Selecting %i experiments with profile-fitted reflections" % len(keep_expts) ) self._data_manager.select(keep_expts) reflections = self._data_manager.reflections used_in_refinement_mask = reflections.get_flags( reflections.flags.used_in_refinement ) keep_expts = [] for i, expt in enumerate(self._data_manager.experiments): if ( reflections.select(used_in_refinement_mask) .select_on_experiment_identifiers([expt.identifier]) .size() ): keep_expts.append(expt.identifier) else: logger.info( "Removing experiment %s (no refined reflections remaining)" % expt.identifier ) if len(keep_expts): logger.info( "Selecting %i experiments with refined reflections" % len(keep_expts) ) self._data_manager.select(keep_expts) experiments = self._data_manager.experiments reflections = self._data_manager.reflections self.unit_cell_clustering(plot_name="cluster_unit_cell_p1.png") if self._params.symmetry.resolve_indexing_ambiguity: self.cosym() self.decide_space_group() self._individual_report_dicts = OrderedDict() self._comparison_graphs = OrderedDict() self._scaled = Scale(self._data_manager, self._params) self._record_individual_report(self._scaled.report(), "All data") self._data_manager.export_experiments("multiplex.expt") self._data_manager.export_reflections("multiplex.refl") scaled_unmerged_mtz = py.path.local(self._scaled.scaled_unmerged_mtz) scaled_unmerged_mtz.copy(py.path.local("scaled_unmerged.mtz")) scaled_mtz = py.path.local(self._scaled.scaled_mtz) scaled_mtz.copy(py.path.local("scaled.mtz")) self._mca = self.multi_crystal_analysis() self.cluster_analysis() # cluster_name = "cos_angle_cluster_%i" % self._cos_angle_clusters[-1].cluster_id # self._individual_report_dicts[cluster_name] = self._dict_from_report(self._scaled.report(), cluster_name) min_completeness = self._params.min_completeness min_multiplicity = self._params.min_multiplicity max_clusters = self._params.max_clusters if ( (max_clusters is not None and max_clusters > 1) or min_completeness is not None or min_multiplicity is not None ): self._data_manager_original = self._data_manager cwd = os.path.abspath(os.getcwd()) n_processed = 0 for cluster in reversed(self._cos_angle_clusters): if max_clusters is not None and n_processed == max_clusters: break if ( min_completeness is not None and cluster.completeness < min_completeness ): continue if ( min_multiplicity is not None and cluster.multiplicity < min_multiplicity ): continue if len(cluster.labels) == len(self._data_manager_original.experiments): continue n_processed += 1 logger.info("Scaling cos angle cluster %i:" % cluster.cluster_id) logger.info(cluster) cluster_dir = "cos_angle_cluster_%i" % cluster.cluster_id if not os.path.exists(cluster_dir): os.mkdir(cluster_dir) os.chdir(cluster_dir) data_manager = copy.deepcopy(self._data_manager_original) data_manager.select(cluster.labels) scaled = Scale(data_manager, self._params) self._record_individual_report( scaled.report(), cluster_dir.replace("_", " ") ) os.chdir(cwd) self.report()
def __init__(self, experiments, reflections, params): self._data_manager = DataManager(experiments, reflections) self._params = params if all([params.symmetry.laue_group, params.symmetry.space_group]): raise ValueError("Can not specify both laue_group and space_group") if self._params.nproc is Auto: self._params.nproc = get_number_cpus() PhilIndex.params.xia2.settings.multiprocessing.nproc = self._params.nproc if self._params.identifiers is not None: self._data_manager.select(self._params.identifiers) if self._params.dose is not None: self._data_manager.filter_dose(*self._params.dose) if params.remove_profile_fitting_failures: reflections = self._data_manager.reflections profile_fitted_mask = reflections.get_flags( reflections.flags.integrated_prf) keep_expts = [] for i, expt in enumerate(self._data_manager.experiments): if (reflections.select( profile_fitted_mask).select_on_experiment_identifiers( [expt.identifier]).size()): keep_expts.append(expt.identifier) if len(keep_expts): logger.info( "Selecting %i experiments with profile-fitted reflections" % len(keep_expts)) self._data_manager.select(keep_expts) reflections = self._data_manager.reflections used_in_refinement_mask = reflections.get_flags( reflections.flags.used_in_refinement) keep_expts = [] for i, expt in enumerate(self._data_manager.experiments): if (reflections.select( used_in_refinement_mask).select_on_experiment_identifiers( [expt.identifier]).size()): keep_expts.append(expt.identifier) else: logger.info( "Removing experiment %s (no refined reflections remaining)" % expt.identifier) if len(keep_expts): logger.info("Selecting %i experiments with refined reflections" % len(keep_expts)) self._data_manager.select(keep_expts) self.unit_cell_clustering(plot_name="cluster_unit_cell_p1.png") if self._params.symmetry.resolve_indexing_ambiguity: self.cosym() self._individual_report_dicts = OrderedDict() self._comparison_graphs = OrderedDict() self._scaled = Scale(self._data_manager, self._params) self._experiments_filename = self._scaled._experiments_filename self._reflections_filename = self._scaled._reflections_filename self.decide_space_group() self._data_manager.export_unmerged_mtz("scaled_unmerged.mtz", d_min=self._scaled.d_min) self._data_manager.export_merged_mtz("scaled.mtz", d_min=self._scaled.d_min) self._data_manager.export_experiments("scaled.expt") self._data_manager.export_reflections("scaled.refl", d_min=self._scaled.d_min) self._record_individual_report(self._data_manager, self._scaled.report(), "All data") self._mca = self.multi_crystal_analysis() self.cluster_analysis() min_completeness = self._params.min_completeness min_multiplicity = self._params.min_multiplicity max_clusters = self._params.max_clusters if self._params.cluster_method == "cos_angle": clusters = self._cos_angle_clusters elif self._params.cluster_method == "correlation": clusters = self._cc_clusters else: raise ValueError("Invalid cluster method: %s" % self._params.cluster_method) if max_clusters or min_completeness is not None or min_multiplicity is not None: self._data_manager_original = self._data_manager cwd = os.path.abspath(os.getcwd()) n_processed = 0 for cluster in reversed(clusters): if max_clusters is not None and n_processed == max_clusters: break if (min_completeness is not None and cluster.completeness < min_completeness): continue if (min_multiplicity is not None and cluster.multiplicity < min_multiplicity): continue if len(cluster.labels) == len( self._data_manager_original.experiments): continue n_processed += 1 logger.info("Scaling cluster %i:" % cluster.cluster_id) logger.info(cluster) cluster_dir = "cluster_%i" % cluster.cluster_id if not os.path.exists(cluster_dir): os.mkdir(cluster_dir) os.chdir(cluster_dir) data_manager = copy.deepcopy(self._data_manager_original) cluster_identifiers = [ self._data_manager.ids_to_identifiers_map[l] for l in cluster.labels ] data_manager.select(cluster_identifiers) scaled = Scale(data_manager, self._params) data_manager.export_unmerged_mtz("scaled_unmerged.mtz", d_min=scaled.d_min) data_manager.export_merged_mtz("scaled.mtz", d_min=scaled.d_min) data_manager.export_experiments("scaled.expt") data_manager.export_reflections("scaled.refl", d_min=scaled.d_min) self._record_individual_report(data_manager, scaled.report(), cluster_dir.replace("_", " ")) os.chdir(cwd) if self._params.filtering.method: # Final round of scaling, this time filtering out any bad datasets data_manager = copy.deepcopy(self._data_manager) params = copy.deepcopy(self._params) params.unit_cell.refine = [] params.resolution.d_min = self._params.resolution.d_min scaled = Scale(data_manager, params, filtering=True) self.scale_and_filter_results = scaled.scale_and_filter_results logger.info("Scale and filtering:\n%s", self.scale_and_filter_results) data_manager.export_unmerged_mtz("filtered_unmerged.mtz", d_min=scaled.d_min) data_manager.export_merged_mtz("filtered.mtz", d_min=scaled.d_min) self._record_individual_report(data_manager, scaled.report(), "Filtered") data_manager.export_experiments("filtered.expt") data_manager.export_reflections("filtered.refl", d_min=scaled.d_min) else: self.scale_and_filter_results = None self.report()
def setup(self): '''Set everything up...''' # check arguments are all ascii Debug.write('Start parsing command line: ' + str(sys.argv)) for token in sys.argv: try: token.encode('ascii') except UnicodeDecodeError: raise RuntimeError('non-ascii characters in input') self._argv = copy.deepcopy(sys.argv) replacements = { '-2d': 'pipeline=2d', '-2di': 'pipeline=2di', '-3d': 'pipeline=3d', '-3di': 'pipeline=3di', '-3dii': 'pipeline=3dii', '-3dd': 'pipeline=3dd', '-dials': 'pipeline=dials', '-quick': 'dials.fast_mode=true', '-failover': 'failover=true', '-small_molecule': 'small_molecule=true' } for k, v in replacements.iteritems(): if k in self._argv: print "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % ( k, v) self._argv[self._argv.index(k)] = v if '-atom' in self._argv: idx = self._argv.index('-atom') element = self._argv[idx + 1] self._argv[idx:idx + 2] = ['atom=%s' % element] print "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % ( element, element) # first of all try to interpret arguments as phil parameters/files from xia2.Handlers.Phil import master_phil from libtbx.phil import command_line cmd_line = command_line.argument_interpreter(master_phil=master_phil) working_phil, self._argv = cmd_line.process_and_fetch( args=self._argv, custom_processor="collect_remaining") PhilIndex.merge_phil(working_phil) try: params = PhilIndex.get_python_object() except RuntimeError as e: raise Sorry(e) # sanity check / interpret Auto in input from libtbx import Auto if params.xia2.settings.input.atom is None: if params.xia2.settings.input.anomalous is Auto: PhilIndex.update("xia2.settings.input.anomalous=false") else: if params.xia2.settings.input.anomalous == False: raise Sorry( 'Setting anomalous=false and atom type inconsistent') params.xia2.settings.input.anomalous = True PhilIndex.update("xia2.settings.input.anomalous=true") if params.xia2.settings.resolution.keep_all_reflections is Auto: if params.xia2.settings.small_molecule == True and \ params.xia2.settings.resolution.d_min is None and \ params.xia2.settings.resolution.d_max is None: PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=true") else: PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=false") if params.xia2.settings.small_molecule == True: Debug.write('Small molecule selected') if params.ccp4.pointless.chirality is None: PhilIndex.update("ccp4.pointless.chirality=nonchiral") params = PhilIndex.get_python_object() # pipeline options self._read_pipeline() Debug.write('Project: %s' % params.xia2.settings.project) Debug.write('Crystal: %s' % params.xia2.settings.crystal) # FIXME add some consistency checks in here e.g. that there are # images assigned, there is a lattice assigned if cell constants # are given and so on params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing from xia2.Handlers.Environment import get_number_cpus if mp_params.mode == 'parallel': if mp_params.type == 'qsub': if which('qsub') is None: raise Sorry('qsub not available') if mp_params.njob is Auto: mp_params.njob = get_number_cpus() if mp_params.nproc is Auto: mp_params.nproc = 1 elif mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() elif mp_params.mode == 'serial': if mp_params.type == 'qsub': if which('qsub') is None: raise Sorry('qsub not available') if mp_params.njob is Auto: mp_params.njob = 1 if mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() PhilIndex.update("xia2.settings.multiprocessing.njob=%d" % mp_params.njob) PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" % mp_params.nproc) params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing if params.xia2.settings.indexer is not None: add_preference("indexer", params.xia2.settings.indexer) if params.xia2.settings.refiner is not None: add_preference("refiner", params.xia2.settings.refiner) if params.xia2.settings.integrater is not None: add_preference("integrater", params.xia2.settings.integrater) if params.xia2.settings.scaler is not None: add_preference("scaler", params.xia2.settings.scaler) if params.xia2.settings.multi_sweep_indexing is Auto: if params.xia2.settings.small_molecule == True and 'dials' == params.xia2.settings.indexer: PhilIndex.update("xia2.settings.multi_sweep_indexing=True") else: PhilIndex.update("xia2.settings.multi_sweep_indexing=False") if params.xia2.settings.multi_sweep_indexing == True and \ params.xia2.settings.multiprocessing.mode == 'parallel': Chatter.write( 'Multi sweep indexing disabled:\nMSI is not available for parallel processing.' ) PhilIndex.update("xia2.settings.multi_sweep_indexing=False") input_json = params.xia2.settings.input.json if (input_json is not None and len(input_json)): for json_file in input_json: assert os.path.isfile(json_file) load_datablock(json_file) reference_geometry = params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: reference_geometries = "\n".join([ "xia2.settings.input.reference_geometry=%s" % os.path.abspath(g) for g in params.xia2.settings.input.reference_geometry ]) Debug.write(reference_geometries) PhilIndex.update(reference_geometries) Debug.write("xia2.settings.trust_beam_centre=true") PhilIndex.update("xia2.settings.trust_beam_centre=true") params = PhilIndex.get_python_object() params = PhilIndex.get_python_object() if params.xia2.settings.input.xinfo is not None: xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() self.set_xinfo(xinfo_file) # issue #55 if not set ATOM in xinfo but anomalous=true or atom= set # on commandline, set here, should be idempotent if params.xia2.settings.input.anomalous is True: crystals = self._xinfo.get_crystals() for xname in crystals: xtal = crystals[xname] Debug.write("Setting anomalous for crystal %s" % xname) xtal.set_anomalous(True) else: xinfo_file = '%s/automatic.xinfo' % os.path.abspath(os.curdir) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() if params.dials.find_spots.phil_file is not None: PhilIndex.update( "dials.find_spots.phil_file=%s" % os.path.abspath(params.dials.find_spots.phil_file)) if params.dials.index.phil_file is not None: PhilIndex.update("dials.index.phil_file=%s" % os.path.abspath(params.dials.index.phil_file)) if params.dials.refine.phil_file is not None: PhilIndex.update("dials.refine.phil_file=%s" % os.path.abspath(params.dials.refine.phil_file)) if params.dials.integrate.phil_file is not None: PhilIndex.update("dials.integrate.phil_file=%s" % os.path.abspath(params.dials.integrate.phil_file)) if params.xds.index.xparm is not None: Flags.set_xparm(params.xds.index.xparm) if params.xds.index.xparm_ub is not None: Flags.set_xparm_ub(params.xds.index.xparm_ub) if params.xia2.settings.scale.freer_file is not None: freer_file = os.path.abspath(params.xia2.settings.scale.freer_file) if not os.path.exists(freer_file): raise RuntimeError('%s does not exist' % freer_file) from xia2.Modules.FindFreeFlag import FindFreeFlag column = FindFreeFlag(freer_file) Debug.write('FreeR_flag column in %s found: %s' % \ (freer_file, column)) PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file) if params.xia2.settings.scale.reference_reflection_file is not None: reference_reflection_file = os.path.abspath( params.xia2.settings.scale.reference_reflection_file) if not os.path.exists(reference_reflection_file): raise RuntimeError('%s does not exist' % reference_reflection_file) PhilIndex.update( "xia2.settings.scale.reference_reflection_file=%s" % reference_reflection_file) params = PhilIndex.get_python_object() datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image) for dataset in datasets: start_end = None if ':' in dataset: tokens = dataset.split(':') # cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100 if len(tokens[0]) == 1: tokens = ['%s:%s' % (tokens[0], tokens[1])] + tokens[2:] if len(tokens) != 3: raise RuntimeError('/path/to/image_0001.cbf:start:end') dataset = tokens[0] start_end = int(tokens[1]), int(tokens[2]) from xia2.Applications.xia2setup import is_hd5f_name if os.path.exists(os.path.abspath(dataset)): dataset = os.path.abspath(dataset) else: directories = [os.getcwd()] + self._argv[1:] found = False for d in directories: if os.path.exists(os.path.join(d, dataset)): dataset = os.path.join(d, dataset) found = True break if not found: raise Sorry('Cound not find %s in %s' % \ (dataset, ' '.join(directories))) if is_hd5f_name(dataset): self._hdf5_master_files.append(dataset) if start_end: Debug.write('Image range: %d %d' % start_end) if not dataset in self._default_start_end: self._default_start_end[dataset] = [] self._default_start_end[dataset].append(start_end) else: Debug.write('No image range specified') else: template, directory = image2template_directory( os.path.abspath(dataset)) self._default_template.append(os.path.join( directory, template)) self._default_directory.append(directory) Debug.write('Interpreted from image %s:' % dataset) Debug.write('Template %s' % template) Debug.write('Directory %s' % directory) if start_end: Debug.write('Image range: %d %d' % start_end) key = os.path.join(directory, template) if not key in self._default_start_end: self._default_start_end[key] = [] self._default_start_end[key].append(start_end) else: Debug.write('No image range specified') # finally, check that all arguments were read and raise an exception # if any of them were nonsense. with open('xia2-working.phil', 'wb') as f: print >> f, PhilIndex.working_phil.as_str() with open('xia2-diff.phil', 'wb') as f: print >> f, PhilIndex.get_diff().as_str() Debug.write('\nDifference PHIL:') Debug.write(PhilIndex.get_diff().as_str(), strip=False) Debug.write('Working PHIL:') Debug.write(PhilIndex.working_phil.as_str(), strip=False) nonsense = 'Unknown command-line options:' was_nonsense = False for j, argv in enumerate(self._argv): if j == 0: continue if argv[0] != '-' and '=' not in argv: continue if not j in self._understood: nonsense += ' %s' % argv was_nonsense = True if was_nonsense: raise RuntimeError(nonsense)