def setup(self): """Set everything up...""" # check arguments are all ascii Debug.write("Start parsing command line: " + str(sys.argv)) for token in sys.argv: try: token.encode("ascii") except UnicodeDecodeError: raise RuntimeError("non-ascii characters in input") self._argv = copy.deepcopy(sys.argv) replacements = { "-2d": "pipeline=2d", "-2di": "pipeline=2di", "-3d": "pipeline=3d", "-3di": "pipeline=3di", "-3dii": "pipeline=3dii", "-3dd": "pipeline=3dd", "-dials": "pipeline=dials", "-quick": "dials.fast_mode=true", "-failover": "failover=true", "-small_molecule": "small_molecule=true", } for k, v in replacements.iteritems(): if k in self._argv: print( "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % (k, v)) self._argv[self._argv.index(k)] = v if "-atom" in self._argv: idx = self._argv.index("-atom") element = self._argv[idx + 1] self._argv[idx:idx + 2] = ["atom=%s" % element] print( "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % (element, element)) # first of all try to interpret arguments as phil parameters/files from xia2.Handlers.Phil import master_phil from libtbx.phil import command_line cmd_line = command_line.argument_interpreter(master_phil=master_phil) working_phil, self._argv = cmd_line.process_and_fetch( args=self._argv, custom_processor="collect_remaining") PhilIndex.merge_phil(working_phil) try: params = PhilIndex.get_python_object() except RuntimeError as e: raise Sorry(e) # sanity check / interpret Auto in input from libtbx import Auto if params.xia2.settings.input.atom is None: if params.xia2.settings.input.anomalous is Auto: PhilIndex.update("xia2.settings.input.anomalous=false") else: if params.xia2.settings.input.anomalous is False: raise Sorry( "Setting anomalous=false and atom type inconsistent") params.xia2.settings.input.anomalous = True PhilIndex.update("xia2.settings.input.anomalous=true") if params.xia2.settings.resolution.keep_all_reflections is Auto: if (params.xia2.settings.small_molecule is True and params.xia2.settings.resolution.d_min is None and params.xia2.settings.resolution.d_max is None): PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=true") else: PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=false") if params.xia2.settings.small_molecule is True: Debug.write("Small molecule selected") if params.xia2.settings.symmetry.chirality is None: PhilIndex.update("xia2.settings.symmetry.chirality=nonchiral") params = PhilIndex.get_python_object() # pipeline options self._read_pipeline() for (parameter, value) in ( ("project", params.xia2.settings.project), ("crystal", params.xia2.settings.crystal), ): validate_project_crystal_name(parameter, value) Debug.write("Project: %s" % params.xia2.settings.project) Debug.write("Crystal: %s" % params.xia2.settings.crystal) # FIXME add some consistency checks in here e.g. that there are # images assigned, there is a lattice assigned if cell constants # are given and so on params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing from xia2.Handlers.Environment import get_number_cpus if mp_params.mode == "parallel": if mp_params.type == "qsub": if which("qsub") is None: raise Sorry("qsub not available") if mp_params.njob is Auto: mp_params.njob = get_number_cpus() if mp_params.nproc is Auto: mp_params.nproc = 1 elif mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() elif mp_params.mode == "serial": if mp_params.type == "qsub": if which("qsub") is None: raise Sorry("qsub not available") if mp_params.njob is Auto: mp_params.njob = 1 if mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() PhilIndex.update("xia2.settings.multiprocessing.njob=%d" % mp_params.njob) PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" % mp_params.nproc) params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing if mp_params.nproc > 1 and os.name == "nt": raise Sorry("nproc > 1 is not supported on Windows.") # #191 if params.xia2.settings.indexer is not None: add_preference("indexer", params.xia2.settings.indexer) if params.xia2.settings.refiner is not None: add_preference("refiner", params.xia2.settings.refiner) if params.xia2.settings.integrater is not None: add_preference("integrater", params.xia2.settings.integrater) if params.xia2.settings.scaler is not None: add_preference("scaler", params.xia2.settings.scaler) if params.xia2.settings.multi_sweep_indexing is Auto: if (params.xia2.settings.small_molecule is True and "dials" == params.xia2.settings.indexer): PhilIndex.update("xia2.settings.multi_sweep_indexing=True") else: PhilIndex.update("xia2.settings.multi_sweep_indexing=False") if (params.xia2.settings.multi_sweep_indexing is True and params.xia2.settings.multiprocessing.mode == "parallel"): Chatter.write( "Multi sweep indexing disabled:\nMSI is not available for parallel processing." ) PhilIndex.update("xia2.settings.multi_sweep_indexing=False") input_json = params.xia2.settings.input.json if input_json is not None and len(input_json): for json_file in input_json: assert os.path.isfile(json_file) load_experiments(json_file) reference_geometry = params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: reference_geometries = "\n".join([ "xia2.settings.input.reference_geometry=%s" % os.path.abspath(g) for g in params.xia2.settings.input.reference_geometry ]) Debug.write(reference_geometries) PhilIndex.update(reference_geometries) Debug.write("xia2.settings.trust_beam_centre=true") PhilIndex.update("xia2.settings.trust_beam_centre=true") params = PhilIndex.get_python_object() params = PhilIndex.get_python_object() if params.xia2.settings.input.xinfo is not None: xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() self.set_xinfo(xinfo_file) # issue #55 if not set ATOM in xinfo but anomalous=true or atom= set # on commandline, set here, should be idempotent if params.xia2.settings.input.anomalous is True: crystals = self._xinfo.get_crystals() for xname in crystals: xtal = crystals[xname] Debug.write("Setting anomalous for crystal %s" % xname) xtal.set_anomalous(True) else: xinfo_file = "%s/automatic.xinfo" % os.path.abspath(os.curdir) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() if params.dials.find_spots.phil_file is not None: PhilIndex.update( "dials.find_spots.phil_file=%s" % os.path.abspath(params.dials.find_spots.phil_file)) if params.dials.index.phil_file is not None: PhilIndex.update("dials.index.phil_file=%s" % os.path.abspath(params.dials.index.phil_file)) if params.dials.refine.phil_file is not None: PhilIndex.update("dials.refine.phil_file=%s" % os.path.abspath(params.dials.refine.phil_file)) if params.dials.integrate.phil_file is not None: PhilIndex.update("dials.integrate.phil_file=%s" % os.path.abspath(params.dials.integrate.phil_file)) if params.xds.index.xparm is not None: Flags.set_xparm(params.xds.index.xparm) if params.xds.index.xparm_ub is not None: Flags.set_xparm_ub(params.xds.index.xparm_ub) if params.xia2.settings.scale.freer_file is not None: freer_file = os.path.abspath(params.xia2.settings.scale.freer_file) if not os.path.exists(freer_file): raise RuntimeError("%s does not exist" % freer_file) from xia2.Modules.FindFreeFlag import FindFreeFlag column = FindFreeFlag(freer_file) Debug.write("FreeR_flag column in %s found: %s" % (freer_file, column)) PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file) if params.xia2.settings.scale.reference_reflection_file is not None: reference_reflection_file = os.path.abspath( params.xia2.settings.scale.reference_reflection_file) if not os.path.exists(reference_reflection_file): raise RuntimeError("%s does not exist" % reference_reflection_file) PhilIndex.update( "xia2.settings.scale.reference_reflection_file=%s" % reference_reflection_file) params = PhilIndex.get_python_object() datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image) for dataset in datasets: start_end = None # here we only care about ':' which are later than C:\ if ":" in dataset[3:]: tokens = dataset.split(":") # cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100 if len(tokens[0]) == 1: tokens = ["%s:%s" % (tokens[0], tokens[1])] + tokens[2:] if len(tokens) != 3: raise RuntimeError("/path/to/image_0001.cbf:start:end") dataset = tokens[0] start_end = int(tokens[1]), int(tokens[2]) from xia2.Applications.xia2setup import is_hd5f_name if os.path.exists(os.path.abspath(dataset)): dataset = os.path.abspath(dataset) else: directories = [os.getcwd()] + self._argv[1:] found = False for d in directories: if os.path.exists(os.path.join(d, dataset)): dataset = os.path.join(d, dataset) found = True break if not found: raise Sorry("Could not find %s in %s" % (dataset, " ".join(directories))) if is_hd5f_name(dataset): self._hdf5_master_files.append(dataset) if start_end: Debug.write("Image range: %d %d" % start_end) if dataset not in self._default_start_end: self._default_start_end[dataset] = [] self._default_start_end[dataset].append(start_end) else: Debug.write("No image range specified") else: template, directory = image2template_directory( os.path.abspath(dataset)) self._default_template.append(os.path.join( directory, template)) self._default_directory.append(directory) Debug.write("Interpreted from image %s:" % dataset) Debug.write("Template %s" % template) Debug.write("Directory %s" % directory) if start_end: Debug.write("Image range: %d %d" % start_end) key = os.path.join(directory, template) if key not in self._default_start_end: self._default_start_end[key] = [] self._default_start_end[key].append(start_end) else: Debug.write("No image range specified") # finally, check that all arguments were read and raise an exception # if any of them were nonsense. with open("xia2-working.phil", "wb") as f: f.write(PhilIndex.working_phil.as_str()) f.write( os.linesep ) # temporarily required for https://github.com/dials/dials/issues/522 with open("xia2-diff.phil", "wb") as f: f.write(PhilIndex.get_diff().as_str()) f.write( os.linesep ) # temporarily required for https://github.com/dials/dials/issues/522 Debug.write("\nDifference PHIL:") Debug.write(PhilIndex.get_diff().as_str(), strip=False) Debug.write("Working PHIL:") Debug.write(PhilIndex.working_phil.as_str(), strip=False) nonsense = "Unknown command-line options:" was_nonsense = False for j, argv in enumerate(self._argv): if j == 0: continue if argv[0] != "-" and "=" not in argv: continue if j not in self._understood: nonsense += " %s" % argv was_nonsense = True if was_nonsense: raise RuntimeError(nonsense)
def multi_crystal_analysis(stop_after=None): '''Actually process something...''' assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): cwd = os.path.abspath(os.curdir) working_directory = Environment.generate_directory( [crystal.get_name(), 'analysis']) os.chdir(working_directory) from xia2.Wrappers.CCP4.Blend import Blend from xia2.lib.bits import auto_logfiler hand_blender = Blend() hand_blender.set_working_directory(working_directory) auto_logfiler(hand_blender) Citations.cite('blend') scaler = crystal._get_scaler() #epoch_to_si = {} epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} try: epochs = scaler._sweep_information.keys() for epoch in epochs: si = scaler._sweep_information[epoch] epoch_to_batches[epoch] = si['batches'] epoch_to_integrated_intensities[epoch] = si['corrected_intensities'] epoch_to_sweep_name[epoch] = si['sname'] except AttributeError, e: epochs = scaler._sweep_handler.get_epochs() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) epoch_to_batches[epoch] = si.get_batches() epoch_to_integrated_intensities[epoch] = si.get_reflections() epoch_to_sweep_name[epoch] = si.get_sweep_name() unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0] from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(unmerged_mtz) intensities = None batches = None assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma from xia2.Handlers.Environment import which Rscript_binary = which('Rscript', debug=False) if Rscript_binary is None: Chatter.write('Skipping BLEND analysis: Rscript not available') else: for epoch in epochs: hand_blender.add_hklin(epoch_to_integrated_intensities[epoch], label=epoch_to_sweep_name[epoch]) hand_blender.analysis() Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file()) analysis = hand_blender.get_analysis() summary = hand_blender.get_summary() clusters = hand_blender.get_clusters() linkage_matrix = hand_blender.get_linkage_matrix() ddict = hand_blender.plot_dendrogram() rows = [] headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV'] completeness = flex.double() for i, cluster in clusters.iteritems(): sel_cluster = flex.bool(batches.size(), False) for j in cluster['dataset_ids']: batch_start, batch_end = epoch_to_batches[epochs[j-1]] sel_cluster |= ( (batches.data() >= batch_start) & (batches.data() <= batch_end)) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) dataset_ids = cluster['dataset_ids'] rows.append( ['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids), '%.1f' %flex.mean(multiplicities.data().as_double()), '%.2f' %completeness[-1], '%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print print 'Unit cell clustering summary:' print tabulate(rows, headers, tablefmt='rst') print blend_html = tabulate(rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">')
def multi_crystal_analysis(stop_after=None): '''Actually process something...''' assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): cwd = os.path.abspath(os.curdir) working_directory = Environment.generate_directory( [crystal.get_name(), 'analysis']) os.chdir(working_directory) scaler = crystal._get_scaler() #epoch_to_si = {} epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} epoch_to_experiments_filename = {} epoch_to_experiments = {} sweep_name_to_epoch = {} epoch_to_first_image = {} from dxtbx.serialize import load try: epochs = scaler._sweep_information.keys() for epoch in epochs: si = scaler._sweep_information[epoch] epoch_to_batches[epoch] = si['batches'] epoch_to_integrated_intensities[epoch] = si[ 'corrected_intensities'] epoch_to_sweep_name[epoch] = si['sname'] sweep_name_to_epoch[si['name']] = epoch intgr = si['integrater'] epoch_to_experiments_filename[ epoch] = intgr.get_integrated_experiments() epoch_to_experiments[epoch] = load.experiment_list( intgr.get_integrated_experiments()) except AttributeError: epochs = scaler._sweep_handler.get_epochs() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) epoch_to_batches[epoch] = si.get_batches() epoch_to_integrated_intensities[epoch] = si.get_reflections() epoch_to_sweep_name[epoch] = si.get_sweep_name() sweep_name_to_epoch[si.get_sweep_name()] = epoch intgr = si.get_integrater() epoch_to_experiments_filename[ epoch] = intgr.get_integrated_experiments() epoch_to_experiments[epoch] = load.experiment_list( intgr.get_integrated_experiments()) from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection sp_json_files = {} for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): sp = StereographicProjection() auto_logfiler(sp) sp.set_working_directory(working_directory) for experiments in epoch_to_experiments_filename.values(): sp.add_experiments(experiments) sp.set_hkl(hkl) sp.run() sp_json_files[hkl] = sp.get_json_filename() unmerged_mtz = scaler.get_scaled_reflections( 'mtz_unmerged').values()[0] from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(unmerged_mtz) from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity mult_json_files = {} for axis in ('h', 'k', 'l'): pm = PlotMultiplicity() auto_logfiler(pm) pm.set_working_directory(working_directory) pm.set_mtz_filename(unmerged_mtz) pm.set_slice_axis(axis) pm.set_show_missing(True) pm.run() mult_json_files[axis] = pm.get_json_filename() intensities = None batches = None assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma from xia2.Wrappers.CCP4.Blend import Blend hand_blender = Blend() hand_blender.set_working_directory(working_directory) auto_logfiler(hand_blender) Citations.cite('blend') from xia2.Handlers.Environment import which Rscript_binary = which('Rscript', debug=False) if Rscript_binary is None: Chatter.write('Skipping BLEND analysis: Rscript not available') else: for epoch in epochs: hand_blender.add_hklin(epoch_to_integrated_intensities[epoch], label=epoch_to_sweep_name[epoch]) hand_blender.analysis() Chatter.write("Dendrogram saved to: %s" % hand_blender.get_dendrogram_file()) analysis = hand_blender.get_analysis() summary = hand_blender.get_summary() clusters = hand_blender.get_clusters() ddict = hand_blender.plot_dendrogram() phil_files_dir = 'phil_files' if not os.path.exists(phil_files_dir): os.makedirs(phil_files_dir) rows = [] headers = [ 'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV', 'Average unit cell' ] completeness = flex.double() average_unit_cell_params = [] for i, cluster in clusters.iteritems(): print i sel_cluster = flex.bool(batches.size(), False) cluster_uc_params = [flex.double() for k in range(6)] for j in cluster['dataset_ids']: epoch = epochs[j - 1] batch_start, batch_end = epoch_to_batches[epoch] sel_cluster |= ((batches.data() >= batch_start) & (batches.data() <= batch_end)) expts = epoch_to_experiments.get(epoch) assert expts is not None, (epoch) assert len(expts) == 1, len(expts) expt = expts[0] uc_params = expt.crystal.get_unit_cell().parameters() for k in range(6): cluster_uc_params[k].append(uc_params[k]) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) average_unit_cell_params.append( tuple(flex.mean(p) for p in cluster_uc_params)) dataset_ids = cluster['dataset_ids'] assert min(dataset_ids) > 0 with open( os.path.join(phil_files_dir, 'blend_cluster_%i_images.phil' % i), 'wb') as f: sweep_names = [ hand_blender._labels[dataset_id - 1] for dataset_id in dataset_ids ] for sweep_name in sweep_names: expts = epoch_to_experiments.get( sweep_name_to_epoch.get(sweep_name)) assert expts is not None, ( sweep_name, sweep_name_to_epoch.get(sweep_name)) assert len(expts) == 1, len(expts) expt = expts[0] print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path( 0) rows.append([ '%i' % i, ' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids), '%.1f' % flex.mean(multiplicities.data().as_double()), '%.2f' % completeness[-1], '%.2f' % cluster['lcv'], '%.2f' % cluster['alcv'], '%g %g %g %g %g %g' % average_unit_cell_params[-1] ]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print print 'Unit cell clustering summary:' print tabulate(rows, headers, tablefmt='rst') print blend_html = tabulate(rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">') # XXX what about multiple wavelengths? with open('batches.phil', 'wb') as f: try: for epoch, si in scaler._sweep_information.iteritems(): print >> f, "batch {" print >> f, " id=%s" % si['sname'] print >> f, " range=%i,%i" % tuple(si['batches']) print >> f, "}" except AttributeError: for epoch in scaler._sweep_handler.get_epochs(): si = scaler._sweep_handler.get_sweep_information(epoch) print >> f, "batch {" print >> f, " id=%s" % si.get_sweep_name() print >> f, " range=%i,%i" % tuple(si.get_batches()) print >> f, "}" from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis mca = MultiCrystalAnalysis() auto_logfiler(mca, extra="MultiCrystalAnalysis") mca.add_command_line_args([ scaler.get_scaled_reflections(format="sca_unmerged").values()[0], "unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()), "batches.phil" ]) mca.set_working_directory(working_directory) mca.run() intensity_clusters = mca.get_clusters() rows = [] headers = [ 'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height', 'Average unit cell' ] completeness = flex.double() average_unit_cell_params = [] for i, cluster in intensity_clusters.iteritems(): sel_cluster = flex.bool(batches.size(), False) cluster_uc_params = [flex.double() for k in range(6)] for j in cluster['datasets']: epoch = epochs[j - 1] batch_start, batch_end = epoch_to_batches[epoch] sel_cluster |= ((batches.data() >= batch_start) & (batches.data() <= batch_end)) expts = epoch_to_experiments.get(epoch) assert expts is not None, (epoch) assert len(expts) == 1, len(expts) expt = expts[0] uc_params = expt.crystal.get_unit_cell().parameters() for k in range(6): cluster_uc_params[k].append(uc_params[k]) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) average_unit_cell_params.append( tuple(flex.mean(p) for p in cluster_uc_params)) dataset_ids = cluster['datasets'] rows.append([ '%i' % int(i), ' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids), '%.1f' % flex.mean(multiplicities.data().as_double()), '%.2f' % completeness[-1], '%.2f' % cluster['height'], '%g %g %g %g %g %g' % average_unit_cell_params[-1] ]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print 'Intensity clustering summary:' print tabulate(rows, headers, tablefmt='rst') print intensity_clustering_html = tabulate( rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">') import json json_data = {} if ddict is not None: from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict) else: json_data['blend_dendrogram'] = {'data': [], 'layout': {}} json_data['intensity_clustering'] = mca.get_dict() del json_data['intensity_clustering']['clusters'] for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): with open(sp_json_files[hkl], 'rb') as f: d = json.load(f) d['layout'][ 'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl json_data['stereographic_projection_%s%s%s' % hkl] = d for axis in ('h', 'k', 'l'): with open(mult_json_files[axis], 'rb') as f: json_data['multiplicity_%s' % axis] = json.load(f) json_str = json.dumps(json_data, indent=2) javascript = ['var graphs = %s' % (json_str)] javascript.append( 'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);' ) javascript.append( 'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);' ) for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): javascript.append( 'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);' % ({ 'hkl': "%s%s%s" % hkl })) for axis in ('h', 'k', 'l'): javascript.append( 'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);' % ({ 'axis': axis })) html_header = ''' <head> <!-- Plotly.js --> <script src="https://cdn.plot.ly/plotly-latest.min.js"></script> <meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8"> <link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script> <script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> <style type="text/css"> body { /*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/ margin: 8px; min-width: 240px; margin-left: 5%; margin-right: 5%; } .plot { float: left; width: 1200px; height: 800px; margin-bottom: 20px; } .square_plot { float: left; width: 800px; height: 800px; margin-bottom: 20px; } </style> </head> ''' html_body = ''' <body> <div class="page-header"> <h1>Multi-crystal analysis report</h1> </div> <div class="panel-group"> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity"> <h4 class="panel-title"> <a>Multiplicity plots</a> </h4> </div> <div id="collapse_multiplicity" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection"> <h4 class="panel-title"> <a>Stereographic projections</a> </h4> </div> <div id="collapse_stereographic_projection" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_cell"> <h4 class="panel-title"> <a>Unit cell clustering</a> </h4> </div> <div id="collapse_cell" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div> <div class="table-responsive" style="width: 800px"> %(blend_html)s </div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_intensity"> <h4 class="panel-title"> <a>Intensity clustering</a> </h4> </div> <div id="collapse_intensity" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div> <div class="table-responsive" style="width: 800px"> %(intensity_clustering_html)s </div> </div> </div> </div> </div> <script> %(script)s </script> </body> ''' % { 'script': '\n'.join(javascript), 'blend_html': blend_html, 'intensity_clustering_html': intensity_clustering_html } html = '\n'.join([html_header, html_body]) print "Writing html report to: %s" % 'multi-crystal-report.html' with open('multi-crystal-report.html', 'wb') as f: print >> f, html.encode('ascii', 'xmlcharrefreplace') write_citations() Environment.cleanup() return
def setup(self): '''Set everything up...''' # check arguments are all ascii Debug.write('Start parsing command line: ' + str(sys.argv)) for token in sys.argv: try: token.encode('ascii') except UnicodeDecodeError: raise RuntimeError('non-ascii characters in input') self._argv = copy.deepcopy(sys.argv) replacements = { '-2d': 'pipeline=2d', '-2di': 'pipeline=2di', '-3d': 'pipeline=3d', '-3di': 'pipeline=3di', '-3dii': 'pipeline=3dii', '-3dd': 'pipeline=3dd', '-dials': 'pipeline=dials', '-quick': 'dials.fast_mode=true', '-failover': 'failover=true', '-small_molecule': 'small_molecule=true' } for k, v in replacements.iteritems(): if k in self._argv: print "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % ( k, v) self._argv[self._argv.index(k)] = v if '-atom' in self._argv: idx = self._argv.index('-atom') element = self._argv[idx + 1] self._argv[idx:idx + 2] = ['atom=%s' % element] print "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % ( element, element) # first of all try to interpret arguments as phil parameters/files from xia2.Handlers.Phil import master_phil from libtbx.phil import command_line cmd_line = command_line.argument_interpreter(master_phil=master_phil) working_phil, self._argv = cmd_line.process_and_fetch( args=self._argv, custom_processor="collect_remaining") PhilIndex.merge_phil(working_phil) try: params = PhilIndex.get_python_object() except RuntimeError as e: raise Sorry(e) # sanity check / interpret Auto in input from libtbx import Auto if params.xia2.settings.input.atom is None: if params.xia2.settings.input.anomalous is Auto: PhilIndex.update("xia2.settings.input.anomalous=false") else: if params.xia2.settings.input.anomalous == False: raise Sorry( 'Setting anomalous=false and atom type inconsistent') params.xia2.settings.input.anomalous = True PhilIndex.update("xia2.settings.input.anomalous=true") if params.xia2.settings.resolution.keep_all_reflections is Auto: if params.xia2.settings.small_molecule == True and \ params.xia2.settings.resolution.d_min is None and \ params.xia2.settings.resolution.d_max is None: PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=true") else: PhilIndex.update( "xia2.settings.resolution.keep_all_reflections=false") if params.xia2.settings.small_molecule == True: Debug.write('Small molecule selected') if params.ccp4.pointless.chirality is None: PhilIndex.update("ccp4.pointless.chirality=nonchiral") params = PhilIndex.get_python_object() # pipeline options self._read_pipeline() Debug.write('Project: %s' % params.xia2.settings.project) Debug.write('Crystal: %s' % params.xia2.settings.crystal) # FIXME add some consistency checks in here e.g. that there are # images assigned, there is a lattice assigned if cell constants # are given and so on params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing from xia2.Handlers.Environment import get_number_cpus if mp_params.mode == 'parallel': if mp_params.type == 'qsub': if which('qsub') is None: raise Sorry('qsub not available') if mp_params.njob is Auto: mp_params.njob = get_number_cpus() if mp_params.nproc is Auto: mp_params.nproc = 1 elif mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() elif mp_params.mode == 'serial': if mp_params.type == 'qsub': if which('qsub') is None: raise Sorry('qsub not available') if mp_params.njob is Auto: mp_params.njob = 1 if mp_params.nproc is Auto: mp_params.nproc = get_number_cpus() PhilIndex.update("xia2.settings.multiprocessing.njob=%d" % mp_params.njob) PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" % mp_params.nproc) params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing if params.xia2.settings.indexer is not None: add_preference("indexer", params.xia2.settings.indexer) if params.xia2.settings.refiner is not None: add_preference("refiner", params.xia2.settings.refiner) if params.xia2.settings.integrater is not None: add_preference("integrater", params.xia2.settings.integrater) if params.xia2.settings.scaler is not None: add_preference("scaler", params.xia2.settings.scaler) if params.xia2.settings.multi_sweep_indexing is Auto: if params.xia2.settings.small_molecule == True and 'dials' == params.xia2.settings.indexer: PhilIndex.update("xia2.settings.multi_sweep_indexing=True") else: PhilIndex.update("xia2.settings.multi_sweep_indexing=False") if params.xia2.settings.multi_sweep_indexing == True and \ params.xia2.settings.multiprocessing.mode == 'parallel': Chatter.write( 'Multi sweep indexing disabled:\nMSI is not available for parallel processing.' ) PhilIndex.update("xia2.settings.multi_sweep_indexing=False") input_json = params.xia2.settings.input.json if (input_json is not None and len(input_json)): for json_file in input_json: assert os.path.isfile(json_file) load_datablock(json_file) reference_geometry = params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: reference_geometries = "\n".join([ "xia2.settings.input.reference_geometry=%s" % os.path.abspath(g) for g in params.xia2.settings.input.reference_geometry ]) Debug.write(reference_geometries) PhilIndex.update(reference_geometries) Debug.write("xia2.settings.trust_beam_centre=true") PhilIndex.update("xia2.settings.trust_beam_centre=true") params = PhilIndex.get_python_object() params = PhilIndex.get_python_object() if params.xia2.settings.input.xinfo is not None: xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() self.set_xinfo(xinfo_file) # issue #55 if not set ATOM in xinfo but anomalous=true or atom= set # on commandline, set here, should be idempotent if params.xia2.settings.input.anomalous is True: crystals = self._xinfo.get_crystals() for xname in crystals: xtal = crystals[xname] Debug.write("Setting anomalous for crystal %s" % xname) xtal.set_anomalous(True) else: xinfo_file = '%s/automatic.xinfo' % os.path.abspath(os.curdir) PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file) params = PhilIndex.get_python_object() if params.dials.find_spots.phil_file is not None: PhilIndex.update( "dials.find_spots.phil_file=%s" % os.path.abspath(params.dials.find_spots.phil_file)) if params.dials.index.phil_file is not None: PhilIndex.update("dials.index.phil_file=%s" % os.path.abspath(params.dials.index.phil_file)) if params.dials.refine.phil_file is not None: PhilIndex.update("dials.refine.phil_file=%s" % os.path.abspath(params.dials.refine.phil_file)) if params.dials.integrate.phil_file is not None: PhilIndex.update("dials.integrate.phil_file=%s" % os.path.abspath(params.dials.integrate.phil_file)) if params.xds.index.xparm is not None: Flags.set_xparm(params.xds.index.xparm) if params.xds.index.xparm_ub is not None: Flags.set_xparm_ub(params.xds.index.xparm_ub) if params.xia2.settings.scale.freer_file is not None: freer_file = os.path.abspath(params.xia2.settings.scale.freer_file) if not os.path.exists(freer_file): raise RuntimeError('%s does not exist' % freer_file) from xia2.Modules.FindFreeFlag import FindFreeFlag column = FindFreeFlag(freer_file) Debug.write('FreeR_flag column in %s found: %s' % \ (freer_file, column)) PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file) if params.xia2.settings.scale.reference_reflection_file is not None: reference_reflection_file = os.path.abspath( params.xia2.settings.scale.reference_reflection_file) if not os.path.exists(reference_reflection_file): raise RuntimeError('%s does not exist' % reference_reflection_file) PhilIndex.update( "xia2.settings.scale.reference_reflection_file=%s" % reference_reflection_file) params = PhilIndex.get_python_object() datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image) for dataset in datasets: start_end = None if ':' in dataset: tokens = dataset.split(':') # cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100 if len(tokens[0]) == 1: tokens = ['%s:%s' % (tokens[0], tokens[1])] + tokens[2:] if len(tokens) != 3: raise RuntimeError('/path/to/image_0001.cbf:start:end') dataset = tokens[0] start_end = int(tokens[1]), int(tokens[2]) from xia2.Applications.xia2setup import is_hd5f_name if os.path.exists(os.path.abspath(dataset)): dataset = os.path.abspath(dataset) else: directories = [os.getcwd()] + self._argv[1:] found = False for d in directories: if os.path.exists(os.path.join(d, dataset)): dataset = os.path.join(d, dataset) found = True break if not found: raise Sorry('Cound not find %s in %s' % \ (dataset, ' '.join(directories))) if is_hd5f_name(dataset): self._hdf5_master_files.append(dataset) if start_end: Debug.write('Image range: %d %d' % start_end) if not dataset in self._default_start_end: self._default_start_end[dataset] = [] self._default_start_end[dataset].append(start_end) else: Debug.write('No image range specified') else: template, directory = image2template_directory( os.path.abspath(dataset)) self._default_template.append(os.path.join( directory, template)) self._default_directory.append(directory) Debug.write('Interpreted from image %s:' % dataset) Debug.write('Template %s' % template) Debug.write('Directory %s' % directory) if start_end: Debug.write('Image range: %d %d' % start_end) key = os.path.join(directory, template) if not key in self._default_start_end: self._default_start_end[key] = [] self._default_start_end[key].append(start_end) else: Debug.write('No image range specified') # finally, check that all arguments were read and raise an exception # if any of them were nonsense. with open('xia2-working.phil', 'wb') as f: print >> f, PhilIndex.working_phil.as_str() with open('xia2-diff.phil', 'wb') as f: print >> f, PhilIndex.get_diff().as_str() Debug.write('\nDifference PHIL:') Debug.write(PhilIndex.get_diff().as_str(), strip=False) Debug.write('Working PHIL:') Debug.write(PhilIndex.working_phil.as_str(), strip=False) nonsense = 'Unknown command-line options:' was_nonsense = False for j, argv in enumerate(self._argv): if j == 0: continue if argv[0] != '-' and '=' not in argv: continue if not j in self._understood: nonsense += ' %s' % argv was_nonsense = True if was_nonsense: raise RuntimeError(nonsense)