def test_blend_wrapper(xia2_regression_build, tmpdir): if not cmd_exists('blend'): pytest.skip('blend not available') tmp_dir = tmpdir.strpath os.chdir(tmp_dir) blend_tutorial_dir = os.path.join(xia2_regression_build, 'blend_tutorial') tar_gz = os.path.join(blend_tutorial_dir, 'data02.tgz') import tarfile tar = tarfile.open(tar_gz, "r:gz") for tarinfo in tar: if tarinfo.isreg(): # extract the file print("Extracting", tarinfo.name) tar.extract(tarinfo, path=tmp_dir) tar.close() g = sorted( glob.glob(os.path.join(tmp_dir, 'data', 'lysozyme', 'dataset_*.mtz'))) from xia2.Wrappers.CCP4.Blend import Blend b = Blend() for f in g: b.add_hklin(f) b.analysis() #print("".join(b.get_all_output())) analysis = b.get_analysis() summary = b.get_summary() clusters = b.get_clusters() linkage_matrix = b.get_linkage_matrix() assert analysis.keys() == range(1, 29) assert analysis[1] == { 'start_image': 1, 'radiation_damage_cutoff': 50, 'd_min': 1.739, 'final_image': 50, 'input_file': os.path.join(tmp_dir, 'data', 'lysozyme', 'dataset_001.mtz') }, analysis[1] assert summary.keys() == range(1, 29) assert summary[1] == { 'volume': 238834.27, 'distance': 308.73, 'd_max': 24.854, 'cell': (78.595, 78.595, 38.664, 90.0, 90.0, 90.0), 'd_min': 1.692, 'mosaicity': 0.0, 'wavelength': 0.9173 } assert clusters.keys() == range(1, 28) if 'furthest_datasets' in clusters[1]: del clusters[1]['furthest_datasets'] del clusters[27]['furthest_datasets'] assert clusters[1] == { 'lcv': 0.05, 'alcv': 0.06, 'n_datasets': 2, 'dataset_ids': [13, 14], 'height': 0.001 } assert clusters[27] == { 'lcv': 13.98, 'alcv': 15.44, 'n_datasets': 28, 'dataset_ids': [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28 ], 'height': 17.335 } import numpy assert approx_equal( list(linkage_matrix.flat), list( numpy.array([[1.2000e+01, 1.3000e+01, 1.0000e-03, 2.0000e+00], [1.0000e+00, 1.1000e+01, 6.0000e-03, 2.0000e+00], [2.2000e+01, 2.6000e+01, 7.0000e-03, 2.0000e+00], [0.0000e+00, 4.0000e+00, 9.0000e-03, 2.0000e+00], [3.0000e+01, 2.5000e+01, 1.3000e-02, 3.0000e+00], [3.1000e+01, 6.0000e+00, 1.6000e-02, 3.0000e+00], [2.9000e+01, 2.0000e+00, 2.2000e-02, 3.0000e+00], [1.6000e+01, 1.9000e+01, 2.4000e-02, 2.0000e+00], [2.8000e+01, 2.1000e+01, 2.4000e-02, 3.0000e+00], [5.0000e+00, 9.0000e+00, 2.6000e-02, 2.0000e+00], [1.4000e+01, 1.7000e+01, 3.1000e-02, 2.0000e+00], [3.2000e+01, 2.7000e+01, 3.2000e-02, 4.0000e+00], [3.3000e+01, 1.0000e+01, 3.6000e-02, 4.0000e+00], [3.7000e+01, 7.0000e+00, 6.9000e-02, 3.0000e+00], [1.5000e+01, 2.4000e+01, 7.5000e-02, 2.0000e+00], [3.4000e+01, 3.0000e+00, 8.4000e-02, 4.0000e+00], [3.5000e+01, 3.6000e+01, 9.3000e-02, 5.0000e+00], [4.0000e+01, 8.0000e+00, 1.5700e-01, 5.0000e+00], [4.2000e+01, 2.3000e+01, 1.7500e-01, 3.0000e+00], [3.8000e+01, 3.9000e+01, 2.0200e-01, 6.0000e+00], [4.3000e+01, 4.5000e+01, 2.8100e-01, 9.0000e+00], [4.6000e+01, 4.7000e+01, 4.7600e-01, 9.0000e+00], [4.1000e+01, 4.8000e+01, 7.2000e-01, 1.2000e+01], [4.4000e+01, 4.9000e+01, 9.9300e-01, 1.4000e+01], [1.8000e+01, 2.0000e+01, 4.3150e+00, 2.0000e+00], [5.0000e+01, 5.1000e+01, 7.5490e+00, 2.6000e+01], [5.2000e+01, 5.3000e+01, 1.7335e+01, 2.8000e+01]]).flat)) assert os.path.exists(b.get_dendrogram_file())
def exercise_blend_wrapper(): if not cmd_exists("blend"): print "Skipping exercise_blend_wrapper(): blend not available" exit(0) if (not libtbx.env.has_module("xia2_regression")) or (xia2_regression is None): print "Skipping exercise_blend_wrapper(): xia2_regression not present." exit(0) cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) blend_tutorial_dir = os.path.join(xia2_regression, "blend_tutorial") tar_gz = os.path.join(blend_tutorial_dir, "data02.tgz") import tarfile tar = tarfile.open(tar_gz, "r:gz") for tarinfo in tar: if tarinfo.isreg(): # extract the file print "Extracting", tarinfo.name tar.extract(tarinfo, path=tmp_dir) tar.close() g = sorted(glob.glob(os.path.join(tmp_dir, "data", "lysozyme", "dataset_*.mtz"))) from xia2.Wrappers.CCP4.Blend import Blend b = Blend() for f in g: b.add_hklin(f) b.analysis() # print "".join(b.get_all_output()) analysis = b.get_analysis() summary = b.get_summary() clusters = b.get_clusters() linkage_matrix = b.get_linkage_matrix() assert analysis.keys() == range(1, 29) assert analysis[1] == { "start_image": 1, "radiation_damage_cutoff": 50, "d_min": 1.739, "final_image": 50, "input_file": os.path.join(tmp_dir, "data", "lysozyme", "dataset_001.mtz"), }, analysis[1] assert summary.keys() == range(1, 29) assert summary[1] == { "volume": 238834.27, "distance": 308.73, "d_max": 24.854, "cell": (78.595, 78.595, 38.664, 90.0, 90.0, 90.0), "d_min": 1.692, "mosaicity": 0.0, "wavelength": 0.9173, } assert clusters.keys() == range(1, 28) if "furthest_datasets" in clusters[1]: del clusters[1]["furthest_datasets"] del clusters[27]["furthest_datasets"] assert clusters[1] == {"lcv": 0.05, "alcv": 0.06, "n_datasets": 2, "dataset_ids": [13, 14], "height": 0.001} assert clusters[27] == { "lcv": 13.98, "alcv": 15.44, "n_datasets": 28, "dataset_ids": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, ], "height": 17.335, } from libtbx.test_utils import approx_equal import numpy assert approx_equal( list(linkage_matrix.flat), list( numpy.array( [ [1.2000e01, 1.3000e01, 1.0000e-03, 2.0000e00], [1.0000e00, 1.1000e01, 6.0000e-03, 2.0000e00], [2.2000e01, 2.6000e01, 7.0000e-03, 2.0000e00], [0.0000e00, 4.0000e00, 9.0000e-03, 2.0000e00], [3.0000e01, 2.5000e01, 1.3000e-02, 3.0000e00], [3.1000e01, 6.0000e00, 1.6000e-02, 3.0000e00], [2.9000e01, 2.0000e00, 2.2000e-02, 3.0000e00], [1.6000e01, 1.9000e01, 2.4000e-02, 2.0000e00], [2.8000e01, 2.1000e01, 2.4000e-02, 3.0000e00], [5.0000e00, 9.0000e00, 2.6000e-02, 2.0000e00], [1.4000e01, 1.7000e01, 3.1000e-02, 2.0000e00], [3.2000e01, 2.7000e01, 3.2000e-02, 4.0000e00], [3.3000e01, 1.0000e01, 3.6000e-02, 4.0000e00], [3.7000e01, 7.0000e00, 6.9000e-02, 3.0000e00], [1.5000e01, 2.4000e01, 7.5000e-02, 2.0000e00], [3.4000e01, 3.0000e00, 8.4000e-02, 4.0000e00], [3.5000e01, 3.6000e01, 9.3000e-02, 5.0000e00], [4.0000e01, 8.0000e00, 1.5700e-01, 5.0000e00], [4.2000e01, 2.3000e01, 1.7500e-01, 3.0000e00], [3.8000e01, 3.9000e01, 2.0200e-01, 6.0000e00], [4.3000e01, 4.5000e01, 2.8100e-01, 9.0000e00], [4.6000e01, 4.7000e01, 4.7600e-01, 9.0000e00], [4.1000e01, 4.8000e01, 7.2000e-01, 1.2000e01], [4.4000e01, 4.9000e01, 9.9300e-01, 1.4000e01], [1.8000e01, 2.0000e01, 4.3150e00, 2.0000e00], [5.0000e01, 5.1000e01, 7.5490e00, 2.6000e01], [5.2000e01, 5.3000e01, 1.7335e01, 2.8000e01], ] ).flat ), ) assert os.path.exists(b.get_dendrogram_file())
def multi_crystal_analysis(stop_after=None): '''Actually process something...''' assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): cwd = os.path.abspath(os.curdir) working_directory = Environment.generate_directory( [crystal.get_name(), 'analysis']) os.chdir(working_directory) scaler = crystal._get_scaler() #epoch_to_si = {} epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} epoch_to_experiments_filename = {} epoch_to_experiments = {} sweep_name_to_epoch = {} epoch_to_first_image = {} from dxtbx.serialize import load try: epochs = scaler._sweep_information.keys() for epoch in epochs: si = scaler._sweep_information[epoch] epoch_to_batches[epoch] = si['batches'] epoch_to_integrated_intensities[epoch] = si[ 'corrected_intensities'] epoch_to_sweep_name[epoch] = si['sname'] sweep_name_to_epoch[si['name']] = epoch intgr = si['integrater'] epoch_to_experiments_filename[ epoch] = intgr.get_integrated_experiments() epoch_to_experiments[epoch] = load.experiment_list( intgr.get_integrated_experiments()) except AttributeError: epochs = scaler._sweep_handler.get_epochs() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) epoch_to_batches[epoch] = si.get_batches() epoch_to_integrated_intensities[epoch] = si.get_reflections() epoch_to_sweep_name[epoch] = si.get_sweep_name() sweep_name_to_epoch[si.get_sweep_name()] = epoch intgr = si.get_integrater() epoch_to_experiments_filename[ epoch] = intgr.get_integrated_experiments() epoch_to_experiments[epoch] = load.experiment_list( intgr.get_integrated_experiments()) from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection sp_json_files = {} for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): sp = StereographicProjection() auto_logfiler(sp) sp.set_working_directory(working_directory) for experiments in epoch_to_experiments_filename.values(): sp.add_experiments(experiments) sp.set_hkl(hkl) sp.run() sp_json_files[hkl] = sp.get_json_filename() unmerged_mtz = scaler.get_scaled_reflections( 'mtz_unmerged').values()[0] from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(unmerged_mtz) from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity mult_json_files = {} for axis in ('h', 'k', 'l'): pm = PlotMultiplicity() auto_logfiler(pm) pm.set_working_directory(working_directory) pm.set_mtz_filename(unmerged_mtz) pm.set_slice_axis(axis) pm.set_show_missing(True) pm.run() mult_json_files[axis] = pm.get_json_filename() intensities = None batches = None assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma from xia2.Wrappers.CCP4.Blend import Blend hand_blender = Blend() hand_blender.set_working_directory(working_directory) auto_logfiler(hand_blender) Citations.cite('blend') from xia2.Handlers.Environment import which Rscript_binary = which('Rscript', debug=False) if Rscript_binary is None: Chatter.write('Skipping BLEND analysis: Rscript not available') else: for epoch in epochs: hand_blender.add_hklin(epoch_to_integrated_intensities[epoch], label=epoch_to_sweep_name[epoch]) hand_blender.analysis() Chatter.write("Dendrogram saved to: %s" % hand_blender.get_dendrogram_file()) analysis = hand_blender.get_analysis() summary = hand_blender.get_summary() clusters = hand_blender.get_clusters() ddict = hand_blender.plot_dendrogram() phil_files_dir = 'phil_files' if not os.path.exists(phil_files_dir): os.makedirs(phil_files_dir) rows = [] headers = [ 'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV', 'Average unit cell' ] completeness = flex.double() average_unit_cell_params = [] for i, cluster in clusters.iteritems(): print i sel_cluster = flex.bool(batches.size(), False) cluster_uc_params = [flex.double() for k in range(6)] for j in cluster['dataset_ids']: epoch = epochs[j - 1] batch_start, batch_end = epoch_to_batches[epoch] sel_cluster |= ((batches.data() >= batch_start) & (batches.data() <= batch_end)) expts = epoch_to_experiments.get(epoch) assert expts is not None, (epoch) assert len(expts) == 1, len(expts) expt = expts[0] uc_params = expt.crystal.get_unit_cell().parameters() for k in range(6): cluster_uc_params[k].append(uc_params[k]) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) average_unit_cell_params.append( tuple(flex.mean(p) for p in cluster_uc_params)) dataset_ids = cluster['dataset_ids'] assert min(dataset_ids) > 0 with open( os.path.join(phil_files_dir, 'blend_cluster_%i_images.phil' % i), 'wb') as f: sweep_names = [ hand_blender._labels[dataset_id - 1] for dataset_id in dataset_ids ] for sweep_name in sweep_names: expts = epoch_to_experiments.get( sweep_name_to_epoch.get(sweep_name)) assert expts is not None, ( sweep_name, sweep_name_to_epoch.get(sweep_name)) assert len(expts) == 1, len(expts) expt = expts[0] print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path( 0) rows.append([ '%i' % i, ' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids), '%.1f' % flex.mean(multiplicities.data().as_double()), '%.2f' % completeness[-1], '%.2f' % cluster['lcv'], '%.2f' % cluster['alcv'], '%g %g %g %g %g %g' % average_unit_cell_params[-1] ]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print print 'Unit cell clustering summary:' print tabulate(rows, headers, tablefmt='rst') print blend_html = tabulate(rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">') # XXX what about multiple wavelengths? with open('batches.phil', 'wb') as f: try: for epoch, si in scaler._sweep_information.iteritems(): print >> f, "batch {" print >> f, " id=%s" % si['sname'] print >> f, " range=%i,%i" % tuple(si['batches']) print >> f, "}" except AttributeError: for epoch in scaler._sweep_handler.get_epochs(): si = scaler._sweep_handler.get_sweep_information(epoch) print >> f, "batch {" print >> f, " id=%s" % si.get_sweep_name() print >> f, " range=%i,%i" % tuple(si.get_batches()) print >> f, "}" from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis mca = MultiCrystalAnalysis() auto_logfiler(mca, extra="MultiCrystalAnalysis") mca.add_command_line_args([ scaler.get_scaled_reflections(format="sca_unmerged").values()[0], "unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()), "batches.phil" ]) mca.set_working_directory(working_directory) mca.run() intensity_clusters = mca.get_clusters() rows = [] headers = [ 'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height', 'Average unit cell' ] completeness = flex.double() average_unit_cell_params = [] for i, cluster in intensity_clusters.iteritems(): sel_cluster = flex.bool(batches.size(), False) cluster_uc_params = [flex.double() for k in range(6)] for j in cluster['datasets']: epoch = epochs[j - 1] batch_start, batch_end = epoch_to_batches[epoch] sel_cluster |= ((batches.data() >= batch_start) & (batches.data() <= batch_end)) expts = epoch_to_experiments.get(epoch) assert expts is not None, (epoch) assert len(expts) == 1, len(expts) expt = expts[0] uc_params = expt.crystal.get_unit_cell().parameters() for k in range(6): cluster_uc_params[k].append(uc_params[k]) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) average_unit_cell_params.append( tuple(flex.mean(p) for p in cluster_uc_params)) dataset_ids = cluster['datasets'] rows.append([ '%i' % int(i), ' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids), '%.1f' % flex.mean(multiplicities.data().as_double()), '%.2f' % completeness[-1], '%.2f' % cluster['height'], '%g %g %g %g %g %g' % average_unit_cell_params[-1] ]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print 'Intensity clustering summary:' print tabulate(rows, headers, tablefmt='rst') print intensity_clustering_html = tabulate( rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">') import json json_data = {} if ddict is not None: from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict) else: json_data['blend_dendrogram'] = {'data': [], 'layout': {}} json_data['intensity_clustering'] = mca.get_dict() del json_data['intensity_clustering']['clusters'] for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): with open(sp_json_files[hkl], 'rb') as f: d = json.load(f) d['layout'][ 'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl json_data['stereographic_projection_%s%s%s' % hkl] = d for axis in ('h', 'k', 'l'): with open(mult_json_files[axis], 'rb') as f: json_data['multiplicity_%s' % axis] = json.load(f) json_str = json.dumps(json_data, indent=2) javascript = ['var graphs = %s' % (json_str)] javascript.append( 'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);' ) javascript.append( 'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);' ) for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): javascript.append( 'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);' % ({ 'hkl': "%s%s%s" % hkl })) for axis in ('h', 'k', 'l'): javascript.append( 'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);' % ({ 'axis': axis })) html_header = ''' <head> <!-- Plotly.js --> <script src="https://cdn.plot.ly/plotly-latest.min.js"></script> <meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8"> <link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script> <script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> <style type="text/css"> body { /*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/ margin: 8px; min-width: 240px; margin-left: 5%; margin-right: 5%; } .plot { float: left; width: 1200px; height: 800px; margin-bottom: 20px; } .square_plot { float: left; width: 800px; height: 800px; margin-bottom: 20px; } </style> </head> ''' html_body = ''' <body> <div class="page-header"> <h1>Multi-crystal analysis report</h1> </div> <div class="panel-group"> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity"> <h4 class="panel-title"> <a>Multiplicity plots</a> </h4> </div> <div id="collapse_multiplicity" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection"> <h4 class="panel-title"> <a>Stereographic projections</a> </h4> </div> <div id="collapse_stereographic_projection" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_cell"> <h4 class="panel-title"> <a>Unit cell clustering</a> </h4> </div> <div id="collapse_cell" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div> <div class="table-responsive" style="width: 800px"> %(blend_html)s </div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_intensity"> <h4 class="panel-title"> <a>Intensity clustering</a> </h4> </div> <div id="collapse_intensity" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div> <div class="table-responsive" style="width: 800px"> %(intensity_clustering_html)s </div> </div> </div> </div> </div> <script> %(script)s </script> </body> ''' % { 'script': '\n'.join(javascript), 'blend_html': blend_html, 'intensity_clustering_html': intensity_clustering_html } html = '\n'.join([html_header, html_body]) print "Writing html report to: %s" % 'multi-crystal-report.html' with open('multi-crystal-report.html', 'wb') as f: print >> f, html.encode('ascii', 'xmlcharrefreplace') write_citations() Environment.cleanup() return
def multi_crystal_analysis(stop_after=None): '''Actually process something...''' assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): cwd = os.path.abspath(os.curdir) working_directory = Environment.generate_directory( [crystal.get_name(), 'analysis']) os.chdir(working_directory) from xia2.Wrappers.CCP4.Blend import Blend from xia2.lib.bits import auto_logfiler hand_blender = Blend() hand_blender.set_working_directory(working_directory) auto_logfiler(hand_blender) Citations.cite('blend') scaler = crystal._get_scaler() #epoch_to_si = {} epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} try: epochs = scaler._sweep_information.keys() for epoch in epochs: si = scaler._sweep_information[epoch] epoch_to_batches[epoch] = si['batches'] epoch_to_integrated_intensities[epoch] = si['corrected_intensities'] epoch_to_sweep_name[epoch] = si['sname'] except AttributeError, e: epochs = scaler._sweep_handler.get_epochs() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) epoch_to_batches[epoch] = si.get_batches() epoch_to_integrated_intensities[epoch] = si.get_reflections() epoch_to_sweep_name[epoch] = si.get_sweep_name() unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0] from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(unmerged_mtz) intensities = None batches = None assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma from xia2.Handlers.Environment import which Rscript_binary = which('Rscript', debug=False) if Rscript_binary is None: Chatter.write('Skipping BLEND analysis: Rscript not available') else: for epoch in epochs: hand_blender.add_hklin(epoch_to_integrated_intensities[epoch], label=epoch_to_sweep_name[epoch]) hand_blender.analysis() Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file()) analysis = hand_blender.get_analysis() summary = hand_blender.get_summary() clusters = hand_blender.get_clusters() linkage_matrix = hand_blender.get_linkage_matrix() ddict = hand_blender.plot_dendrogram() rows = [] headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV'] completeness = flex.double() for i, cluster in clusters.iteritems(): sel_cluster = flex.bool(batches.size(), False) for j in cluster['dataset_ids']: batch_start, batch_end = epoch_to_batches[epochs[j-1]] sel_cluster |= ( (batches.data() >= batch_start) & (batches.data() <= batch_end)) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) dataset_ids = cluster['dataset_ids'] rows.append( ['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids), '%.1f' %flex.mean(multiplicities.data().as_double()), '%.2f' %completeness[-1], '%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print print 'Unit cell clustering summary:' print tabulate(rows, headers, tablefmt='rst') print blend_html = tabulate(rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">')