def test_blend_wrapper(xia2_regression_build, tmpdir):
if not cmd_exists('blend'):
pytest.skip('blend not available')
tmp_dir = tmpdir.strpath
os.chdir(tmp_dir)
blend_tutorial_dir = os.path.join(xia2_regression_build, 'blend_tutorial')
tar_gz = os.path.join(blend_tutorial_dir, 'data02.tgz')
import tarfile
tar = tarfile.open(tar_gz, "r:gz")
for tarinfo in tar:
if tarinfo.isreg():
# extract the file
print("Extracting", tarinfo.name)
tar.extract(tarinfo, path=tmp_dir)
tar.close()
g = sorted(
glob.glob(os.path.join(tmp_dir, 'data', 'lysozyme', 'dataset_*.mtz')))
from xia2.Wrappers.CCP4.Blend import Blend
b = Blend()
for f in g:
b.add_hklin(f)
b.analysis()
#print("".join(b.get_all_output()))
analysis = b.get_analysis()
summary = b.get_summary()
clusters = b.get_clusters()
linkage_matrix = b.get_linkage_matrix()
assert analysis.keys() == range(1, 29)
assert analysis[1] == {
'start_image': 1,
'radiation_damage_cutoff': 50,
'd_min': 1.739,
'final_image': 50,
'input_file': os.path.join(tmp_dir, 'data', 'lysozyme',
'dataset_001.mtz')
}, analysis[1]
assert summary.keys() == range(1, 29)
assert summary[1] == {
'volume': 238834.27,
'distance': 308.73,
'd_max': 24.854,
'cell': (78.595, 78.595, 38.664, 90.0, 90.0, 90.0),
'd_min': 1.692,
'mosaicity': 0.0,
'wavelength': 0.9173
}
assert clusters.keys() == range(1, 28)
if 'furthest_datasets' in clusters[1]:
del clusters[1]['furthest_datasets']
del clusters[27]['furthest_datasets']
assert clusters[1] == {
'lcv': 0.05,
'alcv': 0.06,
'n_datasets': 2,
'dataset_ids': [13, 14],
'height': 0.001
}
assert clusters[27] == {
'lcv':
13.98,
'alcv':
15.44,
'n_datasets':
28,
'dataset_ids': [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28
],
'height':
17.335
}
import numpy
assert approx_equal(
list(linkage_matrix.flat),
list(
numpy.array([[1.2000e+01, 1.3000e+01, 1.0000e-03, 2.0000e+00],
[1.0000e+00, 1.1000e+01, 6.0000e-03, 2.0000e+00],
[2.2000e+01, 2.6000e+01, 7.0000e-03, 2.0000e+00],
[0.0000e+00, 4.0000e+00, 9.0000e-03, 2.0000e+00],
[3.0000e+01, 2.5000e+01, 1.3000e-02, 3.0000e+00],
[3.1000e+01, 6.0000e+00, 1.6000e-02, 3.0000e+00],
[2.9000e+01, 2.0000e+00, 2.2000e-02, 3.0000e+00],
[1.6000e+01, 1.9000e+01, 2.4000e-02, 2.0000e+00],
[2.8000e+01, 2.1000e+01, 2.4000e-02, 3.0000e+00],
[5.0000e+00, 9.0000e+00, 2.6000e-02, 2.0000e+00],
[1.4000e+01, 1.7000e+01, 3.1000e-02, 2.0000e+00],
[3.2000e+01, 2.7000e+01, 3.2000e-02, 4.0000e+00],
[3.3000e+01, 1.0000e+01, 3.6000e-02, 4.0000e+00],
[3.7000e+01, 7.0000e+00, 6.9000e-02, 3.0000e+00],
[1.5000e+01, 2.4000e+01, 7.5000e-02, 2.0000e+00],
[3.4000e+01, 3.0000e+00, 8.4000e-02, 4.0000e+00],
[3.5000e+01, 3.6000e+01, 9.3000e-02, 5.0000e+00],
[4.0000e+01, 8.0000e+00, 1.5700e-01, 5.0000e+00],
[4.2000e+01, 2.3000e+01, 1.7500e-01, 3.0000e+00],
[3.8000e+01, 3.9000e+01, 2.0200e-01, 6.0000e+00],
[4.3000e+01, 4.5000e+01, 2.8100e-01, 9.0000e+00],
[4.6000e+01, 4.7000e+01, 4.7600e-01, 9.0000e+00],
[4.1000e+01, 4.8000e+01, 7.2000e-01, 1.2000e+01],
[4.4000e+01, 4.9000e+01, 9.9300e-01, 1.4000e+01],
[1.8000e+01, 2.0000e+01, 4.3150e+00, 2.0000e+00],
[5.0000e+01, 5.1000e+01, 7.5490e+00, 2.6000e+01],
[5.2000e+01, 5.3000e+01, 1.7335e+01,
2.8000e+01]]).flat))
assert os.path.exists(b.get_dendrogram_file())
def exercise_blend_wrapper():
if not cmd_exists("blend"):
print "Skipping exercise_blend_wrapper(): blend not available"
exit(0)
if (not libtbx.env.has_module("xia2_regression")) or (xia2_regression is None):
print "Skipping exercise_blend_wrapper(): xia2_regression not present."
exit(0)
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
blend_tutorial_dir = os.path.join(xia2_regression, "blend_tutorial")
tar_gz = os.path.join(blend_tutorial_dir, "data02.tgz")
import tarfile
tar = tarfile.open(tar_gz, "r:gz")
for tarinfo in tar:
if tarinfo.isreg():
# extract the file
print "Extracting", tarinfo.name
tar.extract(tarinfo, path=tmp_dir)
tar.close()
g = sorted(glob.glob(os.path.join(tmp_dir, "data", "lysozyme", "dataset_*.mtz")))
from xia2.Wrappers.CCP4.Blend import Blend
b = Blend()
for f in g:
b.add_hklin(f)
b.analysis()
# print "".join(b.get_all_output())
analysis = b.get_analysis()
summary = b.get_summary()
clusters = b.get_clusters()
linkage_matrix = b.get_linkage_matrix()
assert analysis.keys() == range(1, 29)
assert analysis[1] == {
"start_image": 1,
"radiation_damage_cutoff": 50,
"d_min": 1.739,
"final_image": 50,
"input_file": os.path.join(tmp_dir, "data", "lysozyme", "dataset_001.mtz"),
}, analysis[1]
assert summary.keys() == range(1, 29)
assert summary[1] == {
"volume": 238834.27,
"distance": 308.73,
"d_max": 24.854,
"cell": (78.595, 78.595, 38.664, 90.0, 90.0, 90.0),
"d_min": 1.692,
"mosaicity": 0.0,
"wavelength": 0.9173,
}
assert clusters.keys() == range(1, 28)
if "furthest_datasets" in clusters[1]:
del clusters[1]["furthest_datasets"]
del clusters[27]["furthest_datasets"]
assert clusters[1] == {"lcv": 0.05, "alcv": 0.06, "n_datasets": 2, "dataset_ids": [13, 14], "height": 0.001}
assert clusters[27] == {
"lcv": 13.98,
"alcv": 15.44,
"n_datasets": 28,
"dataset_ids": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
],
"height": 17.335,
}
from libtbx.test_utils import approx_equal
import numpy
assert approx_equal(
list(linkage_matrix.flat),
list(
numpy.array(
[
[1.2000e01, 1.3000e01, 1.0000e-03, 2.0000e00],
[1.0000e00, 1.1000e01, 6.0000e-03, 2.0000e00],
[2.2000e01, 2.6000e01, 7.0000e-03, 2.0000e00],
[0.0000e00, 4.0000e00, 9.0000e-03, 2.0000e00],
[3.0000e01, 2.5000e01, 1.3000e-02, 3.0000e00],
[3.1000e01, 6.0000e00, 1.6000e-02, 3.0000e00],
[2.9000e01, 2.0000e00, 2.2000e-02, 3.0000e00],
[1.6000e01, 1.9000e01, 2.4000e-02, 2.0000e00],
[2.8000e01, 2.1000e01, 2.4000e-02, 3.0000e00],
[5.0000e00, 9.0000e00, 2.6000e-02, 2.0000e00],
[1.4000e01, 1.7000e01, 3.1000e-02, 2.0000e00],
[3.2000e01, 2.7000e01, 3.2000e-02, 4.0000e00],
[3.3000e01, 1.0000e01, 3.6000e-02, 4.0000e00],
[3.7000e01, 7.0000e00, 6.9000e-02, 3.0000e00],
[1.5000e01, 2.4000e01, 7.5000e-02, 2.0000e00],
[3.4000e01, 3.0000e00, 8.4000e-02, 4.0000e00],
[3.5000e01, 3.6000e01, 9.3000e-02, 5.0000e00],
[4.0000e01, 8.0000e00, 1.5700e-01, 5.0000e00],
[4.2000e01, 2.3000e01, 1.7500e-01, 3.0000e00],
[3.8000e01, 3.9000e01, 2.0200e-01, 6.0000e00],
[4.3000e01, 4.5000e01, 2.8100e-01, 9.0000e00],
[4.6000e01, 4.7000e01, 4.7600e-01, 9.0000e00],
[4.1000e01, 4.8000e01, 7.2000e-01, 1.2000e01],
[4.4000e01, 4.9000e01, 9.9300e-01, 1.4000e01],
[1.8000e01, 2.0000e01, 4.3150e00, 2.0000e00],
[5.0000e01, 5.1000e01, 7.5490e00, 2.6000e01],
[5.2000e01, 5.3000e01, 1.7335e01, 2.8000e01],
]
).flat
),
)
assert os.path.exists(b.get_dendrogram_file())
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
epoch_to_experiments_filename = {}
epoch_to_experiments = {}
sweep_name_to_epoch = {}
epoch_to_first_image = {}
from dxtbx.serialize import load
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si[
'corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
sweep_name_to_epoch[si['name']] = epoch
intgr = si['integrater']
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
except AttributeError:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
sweep_name_to_epoch[si.get_sweep_name()] = epoch
intgr = si.get_integrater()
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection
sp_json_files = {}
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
sp = StereographicProjection()
auto_logfiler(sp)
sp.set_working_directory(working_directory)
for experiments in epoch_to_experiments_filename.values():
sp.add_experiments(experiments)
sp.set_hkl(hkl)
sp.run()
sp_json_files[hkl] = sp.get_json_filename()
unmerged_mtz = scaler.get_scaled_reflections(
'mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity
mult_json_files = {}
for axis in ('h', 'k', 'l'):
pm = PlotMultiplicity()
auto_logfiler(pm)
pm.set_working_directory(working_directory)
pm.set_mtz_filename(unmerged_mtz)
pm.set_slice_axis(axis)
pm.set_show_missing(True)
pm.run()
mult_json_files[axis] = pm.get_json_filename()
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Wrappers.CCP4.Blend import Blend
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %
hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
ddict = hand_blender.plot_dendrogram()
phil_files_dir = 'phil_files'
if not os.path.exists(phil_files_dir):
os.makedirs(phil_files_dir)
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV',
'aLCV', 'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in clusters.iteritems():
print i
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['dataset_ids']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['dataset_ids']
assert min(dataset_ids) > 0
with open(
os.path.join(phil_files_dir,
'blend_cluster_%i_images.phil' % i),
'wb') as f:
sweep_names = [
hand_blender._labels[dataset_id - 1]
for dataset_id in dataset_ids
]
for sweep_name in sweep_names:
expts = epoch_to_experiments.get(
sweep_name_to_epoch.get(sweep_name))
assert expts is not None, (
sweep_name, sweep_name_to_epoch.get(sweep_name))
assert len(expts) == 1, len(expts)
expt = expts[0]
print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path(
0)
rows.append([
'%i' % i,
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['lcv'],
'%.2f' % cluster['alcv'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
# XXX what about multiple wavelengths?
with open('batches.phil', 'wb') as f:
try:
for epoch, si in scaler._sweep_information.iteritems():
print >> f, "batch {"
print >> f, " id=%s" % si['sname']
print >> f, " range=%i,%i" % tuple(si['batches'])
print >> f, "}"
except AttributeError:
for epoch in scaler._sweep_handler.get_epochs():
si = scaler._sweep_handler.get_sweep_information(epoch)
print >> f, "batch {"
print >> f, " id=%s" % si.get_sweep_name()
print >> f, " range=%i,%i" % tuple(si.get_batches())
print >> f, "}"
from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis
mca = MultiCrystalAnalysis()
auto_logfiler(mca, extra="MultiCrystalAnalysis")
mca.add_command_line_args([
scaler.get_scaled_reflections(format="sca_unmerged").values()[0],
"unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()),
"batches.phil"
])
mca.set_working_directory(working_directory)
mca.run()
intensity_clusters = mca.get_clusters()
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height',
'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in intensity_clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['datasets']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['datasets']
rows.append([
'%i' % int(i),
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['height'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print 'Intensity clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
intensity_clustering_html = tabulate(
rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
import json
json_data = {}
if ddict is not None:
from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json
json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict)
else:
json_data['blend_dendrogram'] = {'data': [], 'layout': {}}
json_data['intensity_clustering'] = mca.get_dict()
del json_data['intensity_clustering']['clusters']
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
with open(sp_json_files[hkl], 'rb') as f:
d = json.load(f)
d['layout'][
'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl
json_data['stereographic_projection_%s%s%s' % hkl] = d
for axis in ('h', 'k', 'l'):
with open(mult_json_files[axis], 'rb') as f:
json_data['multiplicity_%s' % axis] = json.load(f)
json_str = json.dumps(json_data, indent=2)
javascript = ['var graphs = %s' % (json_str)]
javascript.append(
'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);'
)
javascript.append(
'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);'
)
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
javascript.append(
'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);'
% ({
'hkl': "%s%s%s" % hkl
}))
for axis in ('h', 'k', 'l'):
javascript.append(
'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);'
% ({
'axis': axis
}))
html_header = '''
<head>
<!-- Plotly.js -->
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8">
<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
<script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
<style type="text/css">
body {
/*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/
margin: 8px;
min-width: 240px;
margin-left: 5%;
margin-right: 5%;
}
.plot {
float: left;
width: 1200px;
height: 800px;
margin-bottom: 20px;
}
.square_plot {
float: left;
width: 800px;
height: 800px;
margin-bottom: 20px;
}
</style>
</head>
'''
html_body = '''
<body>
<div class="page-header">
<h1>Multi-crystal analysis report</h1>
</div>
<div class="panel-group">
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity">
<h4 class="panel-title">
<a>Multiplicity plots</a>
</h4>
</div>
<div id="collapse_multiplicity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection">
<h4 class="panel-title">
<a>Stereographic projections</a>
</h4>
</div>
<div id="collapse_stereographic_projection" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_cell">
<h4 class="panel-title">
<a>Unit cell clustering</a>
</h4>
</div>
<div id="collapse_cell" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div>
<div class="table-responsive" style="width: 800px">
%(blend_html)s
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_intensity">
<h4 class="panel-title">
<a>Intensity clustering</a>
</h4>
</div>
<div id="collapse_intensity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div>
<div class="table-responsive" style="width: 800px">
%(intensity_clustering_html)s
</div>
</div>
</div>
</div>
</div>
<script>
%(script)s
</script>
</body>
''' % {
'script': '\n'.join(javascript),
'blend_html': blend_html,
'intensity_clustering_html': intensity_clustering_html
}
html = '\n'.join([html_header, html_body])
print "Writing html report to: %s" % 'multi-crystal-report.html'
with open('multi-crystal-report.html', 'wb') as f:
print >> f, html.encode('ascii', 'xmlcharrefreplace')
write_citations()
Environment.cleanup()
return
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
from xia2.Wrappers.CCP4.Blend import Blend
from xia2.lib.bits import auto_logfiler
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si['corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
except AttributeError, e:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
linkage_matrix = hand_blender.get_linkage_matrix()
ddict = hand_blender.plot_dendrogram()
rows = []
headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV']
completeness = flex.double()
for i, cluster in clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
for j in cluster['dataset_ids']:
batch_start, batch_end = epoch_to_batches[epochs[j-1]]
sel_cluster |= (
(batches.data() >= batch_start) & (batches.data() <= batch_end))
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
dataset_ids = cluster['dataset_ids']
rows.append(
['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids),
'%.1f' %flex.mean(multiplicities.data().as_double()),
'%.2f' %completeness[-1],
'%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>', '<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
