def setup(self):
"""Set everything up..."""
# check arguments are all ascii
Debug.write("Start parsing command line: " + str(sys.argv))
for token in sys.argv:
try:
token.encode("ascii")
except UnicodeDecodeError:
raise RuntimeError("non-ascii characters in input")
self._argv = copy.deepcopy(sys.argv)
replacements = {
"-2d": "pipeline=2d",
"-2di": "pipeline=2di",
"-3d": "pipeline=3d",
"-3di": "pipeline=3di",
"-3dii": "pipeline=3dii",
"-3dd": "pipeline=3dd",
"-dials": "pipeline=dials",
"-quick": "dials.fast_mode=true",
"-failover": "failover=true",
"-small_molecule": "small_molecule=true",
}
for k, v in replacements.iteritems():
if k in self._argv:
print(
"***\nCommand line option %s is deprecated.\nPlease use %s instead\n***"
% (k, v))
self._argv[self._argv.index(k)] = v
if "-atom" in self._argv:
idx = self._argv.index("-atom")
element = self._argv[idx + 1]
self._argv[idx:idx + 2] = ["atom=%s" % element]
print(
"***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***"
% (element, element))
# first of all try to interpret arguments as phil parameters/files
from xia2.Handlers.Phil import master_phil
from libtbx.phil import command_line
cmd_line = command_line.argument_interpreter(master_phil=master_phil)
working_phil, self._argv = cmd_line.process_and_fetch(
args=self._argv, custom_processor="collect_remaining")
PhilIndex.merge_phil(working_phil)
try:
params = PhilIndex.get_python_object()
except RuntimeError as e:
raise Sorry(e)
# sanity check / interpret Auto in input
from libtbx import Auto
if params.xia2.settings.input.atom is None:
if params.xia2.settings.input.anomalous is Auto:
PhilIndex.update("xia2.settings.input.anomalous=false")
else:
if params.xia2.settings.input.anomalous is False:
raise Sorry(
"Setting anomalous=false and atom type inconsistent")
params.xia2.settings.input.anomalous = True
PhilIndex.update("xia2.settings.input.anomalous=true")
if params.xia2.settings.resolution.keep_all_reflections is Auto:
if (params.xia2.settings.small_molecule is True
and params.xia2.settings.resolution.d_min is None
and params.xia2.settings.resolution.d_max is None):
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=true")
else:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=false")
if params.xia2.settings.small_molecule is True:
Debug.write("Small molecule selected")
if params.xia2.settings.symmetry.chirality is None:
PhilIndex.update("xia2.settings.symmetry.chirality=nonchiral")
params = PhilIndex.get_python_object()
# pipeline options
self._read_pipeline()
for (parameter, value) in (
("project", params.xia2.settings.project),
("crystal", params.xia2.settings.crystal),
):
validate_project_crystal_name(parameter, value)
Debug.write("Project: %s" % params.xia2.settings.project)
Debug.write("Crystal: %s" % params.xia2.settings.crystal)
# FIXME add some consistency checks in here e.g. that there are
# images assigned, there is a lattice assigned if cell constants
# are given and so on
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
from xia2.Handlers.Environment import get_number_cpus
if mp_params.mode == "parallel":
if mp_params.type == "qsub":
if which("qsub") is None:
raise Sorry("qsub not available")
if mp_params.njob is Auto:
mp_params.njob = get_number_cpus()
if mp_params.nproc is Auto:
mp_params.nproc = 1
elif mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
elif mp_params.mode == "serial":
if mp_params.type == "qsub":
if which("qsub") is None:
raise Sorry("qsub not available")
if mp_params.njob is Auto:
mp_params.njob = 1
if mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
mp_params.njob)
PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
mp_params.nproc)
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if mp_params.nproc > 1 and os.name == "nt":
raise Sorry("nproc > 1 is not supported on Windows.") # #191
if params.xia2.settings.indexer is not None:
add_preference("indexer", params.xia2.settings.indexer)
if params.xia2.settings.refiner is not None:
add_preference("refiner", params.xia2.settings.refiner)
if params.xia2.settings.integrater is not None:
add_preference("integrater", params.xia2.settings.integrater)
if params.xia2.settings.scaler is not None:
add_preference("scaler", params.xia2.settings.scaler)
if params.xia2.settings.multi_sweep_indexing is Auto:
if (params.xia2.settings.small_molecule is True
and "dials" == params.xia2.settings.indexer):
PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
else:
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
if (params.xia2.settings.multi_sweep_indexing is True
and params.xia2.settings.multiprocessing.mode == "parallel"):
Chatter.write(
"Multi sweep indexing disabled:\nMSI is not available for parallel processing."
)
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
input_json = params.xia2.settings.input.json
if input_json is not None and len(input_json):
for json_file in input_json:
assert os.path.isfile(json_file)
load_experiments(json_file)
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
reference_geometries = "\n".join([
"xia2.settings.input.reference_geometry=%s" %
os.path.abspath(g)
for g in params.xia2.settings.input.reference_geometry
])
Debug.write(reference_geometries)
PhilIndex.update(reference_geometries)
Debug.write("xia2.settings.trust_beam_centre=true")
PhilIndex.update("xia2.settings.trust_beam_centre=true")
params = PhilIndex.get_python_object()
params = PhilIndex.get_python_object()
if params.xia2.settings.input.xinfo is not None:
xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
self.set_xinfo(xinfo_file)
# issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
# on commandline, set here, should be idempotent
if params.xia2.settings.input.anomalous is True:
crystals = self._xinfo.get_crystals()
for xname in crystals:
xtal = crystals[xname]
Debug.write("Setting anomalous for crystal %s" % xname)
xtal.set_anomalous(True)
else:
xinfo_file = "%s/automatic.xinfo" % os.path.abspath(os.curdir)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
if params.dials.find_spots.phil_file is not None:
PhilIndex.update(
"dials.find_spots.phil_file=%s" %
os.path.abspath(params.dials.find_spots.phil_file))
if params.dials.index.phil_file is not None:
PhilIndex.update("dials.index.phil_file=%s" %
os.path.abspath(params.dials.index.phil_file))
if params.dials.refine.phil_file is not None:
PhilIndex.update("dials.refine.phil_file=%s" %
os.path.abspath(params.dials.refine.phil_file))
if params.dials.integrate.phil_file is not None:
PhilIndex.update("dials.integrate.phil_file=%s" %
os.path.abspath(params.dials.integrate.phil_file))
if params.xds.index.xparm is not None:
Flags.set_xparm(params.xds.index.xparm)
if params.xds.index.xparm_ub is not None:
Flags.set_xparm_ub(params.xds.index.xparm_ub)
if params.xia2.settings.scale.freer_file is not None:
freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
if not os.path.exists(freer_file):
raise RuntimeError("%s does not exist" % freer_file)
from xia2.Modules.FindFreeFlag import FindFreeFlag
column = FindFreeFlag(freer_file)
Debug.write("FreeR_flag column in %s found: %s" %
(freer_file, column))
PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)
if params.xia2.settings.scale.reference_reflection_file is not None:
reference_reflection_file = os.path.abspath(
params.xia2.settings.scale.reference_reflection_file)
if not os.path.exists(reference_reflection_file):
raise RuntimeError("%s does not exist" %
reference_reflection_file)
PhilIndex.update(
"xia2.settings.scale.reference_reflection_file=%s" %
reference_reflection_file)
params = PhilIndex.get_python_object()
datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)
for dataset in datasets:
start_end = None
# here we only care about ':' which are later than C:\
if ":" in dataset[3:]:
tokens = dataset.split(":")
# cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
if len(tokens[0]) == 1:
tokens = ["%s:%s" % (tokens[0], tokens[1])] + tokens[2:]
if len(tokens) != 3:
raise RuntimeError("/path/to/image_0001.cbf:start:end")
dataset = tokens[0]
start_end = int(tokens[1]), int(tokens[2])
from xia2.Applications.xia2setup import is_hd5f_name
if os.path.exists(os.path.abspath(dataset)):
dataset = os.path.abspath(dataset)
else:
directories = [os.getcwd()] + self._argv[1:]
found = False
for d in directories:
if os.path.exists(os.path.join(d, dataset)):
dataset = os.path.join(d, dataset)
found = True
break
if not found:
raise Sorry("Could not find %s in %s" %
(dataset, " ".join(directories)))
if is_hd5f_name(dataset):
self._hdf5_master_files.append(dataset)
if start_end:
Debug.write("Image range: %d %d" % start_end)
if dataset not in self._default_start_end:
self._default_start_end[dataset] = []
self._default_start_end[dataset].append(start_end)
else:
Debug.write("No image range specified")
else:
template, directory = image2template_directory(
os.path.abspath(dataset))
self._default_template.append(os.path.join(
directory, template))
self._default_directory.append(directory)
Debug.write("Interpreted from image %s:" % dataset)
Debug.write("Template %s" % template)
Debug.write("Directory %s" % directory)
if start_end:
Debug.write("Image range: %d %d" % start_end)
key = os.path.join(directory, template)
if key not in self._default_start_end:
self._default_start_end[key] = []
self._default_start_end[key].append(start_end)
else:
Debug.write("No image range specified")
# finally, check that all arguments were read and raise an exception
# if any of them were nonsense.
with open("xia2-working.phil", "wb") as f:
f.write(PhilIndex.working_phil.as_str())
f.write(
os.linesep
) # temporarily required for https://github.com/dials/dials/issues/522
with open("xia2-diff.phil", "wb") as f:
f.write(PhilIndex.get_diff().as_str())
f.write(
os.linesep
) # temporarily required for https://github.com/dials/dials/issues/522
Debug.write("\nDifference PHIL:")
Debug.write(PhilIndex.get_diff().as_str(), strip=False)
Debug.write("Working PHIL:")
Debug.write(PhilIndex.working_phil.as_str(), strip=False)
nonsense = "Unknown command-line options:"
was_nonsense = False
for j, argv in enumerate(self._argv):
if j == 0:
continue
if argv[0] != "-" and "=" not in argv:
continue
if j not in self._understood:
nonsense += " %s" % argv
was_nonsense = True
if was_nonsense:
raise RuntimeError(nonsense)
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
from xia2.Wrappers.CCP4.Blend import Blend
from xia2.lib.bits import auto_logfiler
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si['corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
except AttributeError, e:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
linkage_matrix = hand_blender.get_linkage_matrix()
ddict = hand_blender.plot_dendrogram()
rows = []
headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV']
completeness = flex.double()
for i, cluster in clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
for j in cluster['dataset_ids']:
batch_start, batch_end = epoch_to_batches[epochs[j-1]]
sel_cluster |= (
(batches.data() >= batch_start) & (batches.data() <= batch_end))
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
dataset_ids = cluster['dataset_ids']
rows.append(
['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids),
'%.1f' %flex.mean(multiplicities.data().as_double()),
'%.2f' %completeness[-1],
'%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>', '<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
epoch_to_experiments_filename = {}
epoch_to_experiments = {}
sweep_name_to_epoch = {}
epoch_to_first_image = {}
from dxtbx.serialize import load
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si[
'corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
sweep_name_to_epoch[si['name']] = epoch
intgr = si['integrater']
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
except AttributeError:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
sweep_name_to_epoch[si.get_sweep_name()] = epoch
intgr = si.get_integrater()
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection
sp_json_files = {}
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
sp = StereographicProjection()
auto_logfiler(sp)
sp.set_working_directory(working_directory)
for experiments in epoch_to_experiments_filename.values():
sp.add_experiments(experiments)
sp.set_hkl(hkl)
sp.run()
sp_json_files[hkl] = sp.get_json_filename()
unmerged_mtz = scaler.get_scaled_reflections(
'mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity
mult_json_files = {}
for axis in ('h', 'k', 'l'):
pm = PlotMultiplicity()
auto_logfiler(pm)
pm.set_working_directory(working_directory)
pm.set_mtz_filename(unmerged_mtz)
pm.set_slice_axis(axis)
pm.set_show_missing(True)
pm.run()
mult_json_files[axis] = pm.get_json_filename()
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Wrappers.CCP4.Blend import Blend
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %
hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
ddict = hand_blender.plot_dendrogram()
phil_files_dir = 'phil_files'
if not os.path.exists(phil_files_dir):
os.makedirs(phil_files_dir)
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV',
'aLCV', 'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in clusters.iteritems():
print i
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['dataset_ids']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['dataset_ids']
assert min(dataset_ids) > 0
with open(
os.path.join(phil_files_dir,
'blend_cluster_%i_images.phil' % i),
'wb') as f:
sweep_names = [
hand_blender._labels[dataset_id - 1]
for dataset_id in dataset_ids
]
for sweep_name in sweep_names:
expts = epoch_to_experiments.get(
sweep_name_to_epoch.get(sweep_name))
assert expts is not None, (
sweep_name, sweep_name_to_epoch.get(sweep_name))
assert len(expts) == 1, len(expts)
expt = expts[0]
print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path(
0)
rows.append([
'%i' % i,
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['lcv'],
'%.2f' % cluster['alcv'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
# XXX what about multiple wavelengths?
with open('batches.phil', 'wb') as f:
try:
for epoch, si in scaler._sweep_information.iteritems():
print >> f, "batch {"
print >> f, " id=%s" % si['sname']
print >> f, " range=%i,%i" % tuple(si['batches'])
print >> f, "}"
except AttributeError:
for epoch in scaler._sweep_handler.get_epochs():
si = scaler._sweep_handler.get_sweep_information(epoch)
print >> f, "batch {"
print >> f, " id=%s" % si.get_sweep_name()
print >> f, " range=%i,%i" % tuple(si.get_batches())
print >> f, "}"
from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis
mca = MultiCrystalAnalysis()
auto_logfiler(mca, extra="MultiCrystalAnalysis")
mca.add_command_line_args([
scaler.get_scaled_reflections(format="sca_unmerged").values()[0],
"unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()),
"batches.phil"
])
mca.set_working_directory(working_directory)
mca.run()
intensity_clusters = mca.get_clusters()
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height',
'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in intensity_clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['datasets']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['datasets']
rows.append([
'%i' % int(i),
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['height'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print 'Intensity clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
intensity_clustering_html = tabulate(
rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
import json
json_data = {}
if ddict is not None:
from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json
json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict)
else:
json_data['blend_dendrogram'] = {'data': [], 'layout': {}}
json_data['intensity_clustering'] = mca.get_dict()
del json_data['intensity_clustering']['clusters']
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
with open(sp_json_files[hkl], 'rb') as f:
d = json.load(f)
d['layout'][
'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl
json_data['stereographic_projection_%s%s%s' % hkl] = d
for axis in ('h', 'k', 'l'):
with open(mult_json_files[axis], 'rb') as f:
json_data['multiplicity_%s' % axis] = json.load(f)
json_str = json.dumps(json_data, indent=2)
javascript = ['var graphs = %s' % (json_str)]
javascript.append(
'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);'
)
javascript.append(
'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);'
)
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
javascript.append(
'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);'
% ({
'hkl': "%s%s%s" % hkl
}))
for axis in ('h', 'k', 'l'):
javascript.append(
'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);'
% ({
'axis': axis
}))
html_header = '''
<head>
<!-- Plotly.js -->
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8">
<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
<script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
<style type="text/css">
body {
/*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/
margin: 8px;
min-width: 240px;
margin-left: 5%;
margin-right: 5%;
}
.plot {
float: left;
width: 1200px;
height: 800px;
margin-bottom: 20px;
}
.square_plot {
float: left;
width: 800px;
height: 800px;
margin-bottom: 20px;
}
</style>
</head>
'''
html_body = '''
<body>
<div class="page-header">
<h1>Multi-crystal analysis report</h1>
</div>
<div class="panel-group">
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity">
<h4 class="panel-title">
<a>Multiplicity plots</a>
</h4>
</div>
<div id="collapse_multiplicity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection">
<h4 class="panel-title">
<a>Stereographic projections</a>
</h4>
</div>
<div id="collapse_stereographic_projection" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_cell">
<h4 class="panel-title">
<a>Unit cell clustering</a>
</h4>
</div>
<div id="collapse_cell" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div>
<div class="table-responsive" style="width: 800px">
%(blend_html)s
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_intensity">
<h4 class="panel-title">
<a>Intensity clustering</a>
</h4>
</div>
<div id="collapse_intensity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div>
<div class="table-responsive" style="width: 800px">
%(intensity_clustering_html)s
</div>
</div>
</div>
</div>
</div>
<script>
%(script)s
</script>
</body>
''' % {
'script': '\n'.join(javascript),
'blend_html': blend_html,
'intensity_clustering_html': intensity_clustering_html
}
html = '\n'.join([html_header, html_body])
print "Writing html report to: %s" % 'multi-crystal-report.html'
with open('multi-crystal-report.html', 'wb') as f:
print >> f, html.encode('ascii', 'xmlcharrefreplace')
write_citations()
Environment.cleanup()
return
def setup(self):
'''Set everything up...'''
# check arguments are all ascii
Debug.write('Start parsing command line: ' + str(sys.argv))
for token in sys.argv:
try:
token.encode('ascii')
except UnicodeDecodeError:
raise RuntimeError('non-ascii characters in input')
self._argv = copy.deepcopy(sys.argv)
replacements = {
'-2d': 'pipeline=2d',
'-2di': 'pipeline=2di',
'-3d': 'pipeline=3d',
'-3di': 'pipeline=3di',
'-3dii': 'pipeline=3dii',
'-3dd': 'pipeline=3dd',
'-dials': 'pipeline=dials',
'-quick': 'dials.fast_mode=true',
'-failover': 'failover=true',
'-small_molecule': 'small_molecule=true'
}
for k, v in replacements.iteritems():
if k in self._argv:
print "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % (
k, v)
self._argv[self._argv.index(k)] = v
if '-atom' in self._argv:
idx = self._argv.index('-atom')
element = self._argv[idx + 1]
self._argv[idx:idx + 2] = ['atom=%s' % element]
print "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % (
element, element)
# first of all try to interpret arguments as phil parameters/files
from xia2.Handlers.Phil import master_phil
from libtbx.phil import command_line
cmd_line = command_line.argument_interpreter(master_phil=master_phil)
working_phil, self._argv = cmd_line.process_and_fetch(
args=self._argv, custom_processor="collect_remaining")
PhilIndex.merge_phil(working_phil)
try:
params = PhilIndex.get_python_object()
except RuntimeError as e:
raise Sorry(e)
# sanity check / interpret Auto in input
from libtbx import Auto
if params.xia2.settings.input.atom is None:
if params.xia2.settings.input.anomalous is Auto:
PhilIndex.update("xia2.settings.input.anomalous=false")
else:
if params.xia2.settings.input.anomalous == False:
raise Sorry(
'Setting anomalous=false and atom type inconsistent')
params.xia2.settings.input.anomalous = True
PhilIndex.update("xia2.settings.input.anomalous=true")
if params.xia2.settings.resolution.keep_all_reflections is Auto:
if params.xia2.settings.small_molecule == True and \
params.xia2.settings.resolution.d_min is None and \
params.xia2.settings.resolution.d_max is None:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=true")
else:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=false")
if params.xia2.settings.small_molecule == True:
Debug.write('Small molecule selected')
if params.ccp4.pointless.chirality is None:
PhilIndex.update("ccp4.pointless.chirality=nonchiral")
params = PhilIndex.get_python_object()
# pipeline options
self._read_pipeline()
Debug.write('Project: %s' % params.xia2.settings.project)
Debug.write('Crystal: %s' % params.xia2.settings.crystal)
# FIXME add some consistency checks in here e.g. that there are
# images assigned, there is a lattice assigned if cell constants
# are given and so on
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
from xia2.Handlers.Environment import get_number_cpus
if mp_params.mode == 'parallel':
if mp_params.type == 'qsub':
if which('qsub') is None:
raise Sorry('qsub not available')
if mp_params.njob is Auto:
mp_params.njob = get_number_cpus()
if mp_params.nproc is Auto:
mp_params.nproc = 1
elif mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
elif mp_params.mode == 'serial':
if mp_params.type == 'qsub':
if which('qsub') is None:
raise Sorry('qsub not available')
if mp_params.njob is Auto:
mp_params.njob = 1
if mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
mp_params.njob)
PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
mp_params.nproc)
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if params.xia2.settings.indexer is not None:
add_preference("indexer", params.xia2.settings.indexer)
if params.xia2.settings.refiner is not None:
add_preference("refiner", params.xia2.settings.refiner)
if params.xia2.settings.integrater is not None:
add_preference("integrater", params.xia2.settings.integrater)
if params.xia2.settings.scaler is not None:
add_preference("scaler", params.xia2.settings.scaler)
if params.xia2.settings.multi_sweep_indexing is Auto:
if params.xia2.settings.small_molecule == True and 'dials' == params.xia2.settings.indexer:
PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
else:
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
if params.xia2.settings.multi_sweep_indexing == True and \
params.xia2.settings.multiprocessing.mode == 'parallel':
Chatter.write(
'Multi sweep indexing disabled:\nMSI is not available for parallel processing.'
)
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
input_json = params.xia2.settings.input.json
if (input_json is not None and len(input_json)):
for json_file in input_json:
assert os.path.isfile(json_file)
load_datablock(json_file)
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
reference_geometries = "\n".join([
"xia2.settings.input.reference_geometry=%s" %
os.path.abspath(g)
for g in params.xia2.settings.input.reference_geometry
])
Debug.write(reference_geometries)
PhilIndex.update(reference_geometries)
Debug.write("xia2.settings.trust_beam_centre=true")
PhilIndex.update("xia2.settings.trust_beam_centre=true")
params = PhilIndex.get_python_object()
params = PhilIndex.get_python_object()
if params.xia2.settings.input.xinfo is not None:
xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
self.set_xinfo(xinfo_file)
# issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
# on commandline, set here, should be idempotent
if params.xia2.settings.input.anomalous is True:
crystals = self._xinfo.get_crystals()
for xname in crystals:
xtal = crystals[xname]
Debug.write("Setting anomalous for crystal %s" % xname)
xtal.set_anomalous(True)
else:
xinfo_file = '%s/automatic.xinfo' % os.path.abspath(os.curdir)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
if params.dials.find_spots.phil_file is not None:
PhilIndex.update(
"dials.find_spots.phil_file=%s" %
os.path.abspath(params.dials.find_spots.phil_file))
if params.dials.index.phil_file is not None:
PhilIndex.update("dials.index.phil_file=%s" %
os.path.abspath(params.dials.index.phil_file))
if params.dials.refine.phil_file is not None:
PhilIndex.update("dials.refine.phil_file=%s" %
os.path.abspath(params.dials.refine.phil_file))
if params.dials.integrate.phil_file is not None:
PhilIndex.update("dials.integrate.phil_file=%s" %
os.path.abspath(params.dials.integrate.phil_file))
if params.xds.index.xparm is not None:
Flags.set_xparm(params.xds.index.xparm)
if params.xds.index.xparm_ub is not None:
Flags.set_xparm_ub(params.xds.index.xparm_ub)
if params.xia2.settings.scale.freer_file is not None:
freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
if not os.path.exists(freer_file):
raise RuntimeError('%s does not exist' % freer_file)
from xia2.Modules.FindFreeFlag import FindFreeFlag
column = FindFreeFlag(freer_file)
Debug.write('FreeR_flag column in %s found: %s' % \
(freer_file, column))
PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)
if params.xia2.settings.scale.reference_reflection_file is not None:
reference_reflection_file = os.path.abspath(
params.xia2.settings.scale.reference_reflection_file)
if not os.path.exists(reference_reflection_file):
raise RuntimeError('%s does not exist' %
reference_reflection_file)
PhilIndex.update(
"xia2.settings.scale.reference_reflection_file=%s" %
reference_reflection_file)
params = PhilIndex.get_python_object()
datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)
for dataset in datasets:
start_end = None
if ':' in dataset:
tokens = dataset.split(':')
# cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
if len(tokens[0]) == 1:
tokens = ['%s:%s' % (tokens[0], tokens[1])] + tokens[2:]
if len(tokens) != 3:
raise RuntimeError('/path/to/image_0001.cbf:start:end')
dataset = tokens[0]
start_end = int(tokens[1]), int(tokens[2])
from xia2.Applications.xia2setup import is_hd5f_name
if os.path.exists(os.path.abspath(dataset)):
dataset = os.path.abspath(dataset)
else:
directories = [os.getcwd()] + self._argv[1:]
found = False
for d in directories:
if os.path.exists(os.path.join(d, dataset)):
dataset = os.path.join(d, dataset)
found = True
break
if not found:
raise Sorry('Cound not find %s in %s' % \
(dataset, ' '.join(directories)))
if is_hd5f_name(dataset):
self._hdf5_master_files.append(dataset)
if start_end:
Debug.write('Image range: %d %d' % start_end)
if not dataset in self._default_start_end:
self._default_start_end[dataset] = []
self._default_start_end[dataset].append(start_end)
else:
Debug.write('No image range specified')
else:
template, directory = image2template_directory(
os.path.abspath(dataset))
self._default_template.append(os.path.join(
directory, template))
self._default_directory.append(directory)
Debug.write('Interpreted from image %s:' % dataset)
Debug.write('Template %s' % template)
Debug.write('Directory %s' % directory)
if start_end:
Debug.write('Image range: %d %d' % start_end)
key = os.path.join(directory, template)
if not key in self._default_start_end:
self._default_start_end[key] = []
self._default_start_end[key].append(start_end)
else:
Debug.write('No image range specified')
# finally, check that all arguments were read and raise an exception
# if any of them were nonsense.
with open('xia2-working.phil', 'wb') as f:
print >> f, PhilIndex.working_phil.as_str()
with open('xia2-diff.phil', 'wb') as f:
print >> f, PhilIndex.get_diff().as_str()
Debug.write('\nDifference PHIL:')
Debug.write(PhilIndex.get_diff().as_str(), strip=False)
Debug.write('Working PHIL:')
Debug.write(PhilIndex.working_phil.as_str(), strip=False)
nonsense = 'Unknown command-line options:'
was_nonsense = False
for j, argv in enumerate(self._argv):
if j == 0:
continue
if argv[0] != '-' and '=' not in argv:
continue
if not j in self._understood:
nonsense += ' %s' % argv
was_nonsense = True
if was_nonsense:
raise RuntimeError(nonsense)