def cleanup(self):
out = open('xia2-files.txt', 'w')
for f in self._temporary_files:
try:
os.remove(f)
out.write('Deleted: %s\n' % f)
except Exception as e:
out.write('Failed to delete: %s (%s)\n' % \
(f, str(e)))
for f in self._output_files:
out.write('Output file (%s): %s\n' % f)
# copy the log files
log_directory = Environment.generate_directory('LogFiles')
for f in self._log_file_keys:
filename = os.path.join(log_directory,
'%s.log' % f.replace(' ', '_'))
shutil.copyfile(self._log_files[f], filename)
out.write('Copied log file %s to %s\n' %
(self._log_files[f], filename))
for f in self._xml_file_keys:
filename = os.path.join(log_directory,
'%s.xml' % f.replace(' ', '_'))
shutil.copyfile(self._xml_files[f], filename)
out.write('Copied xml file %s to %s\n' %
(self._xml_files[f], filename))
for f in self._html_file_keys:
filename = os.path.join(log_directory,
'%s.html' % f.replace(' ', '_'))
shutil.copyfile(self._html_files[f], filename)
out.write('Copied html file %s to %s\n' %
(self._html_files[f], filename))
# copy the data files
data_directory = Environment.generate_directory('DataFiles')
for f in self._data_files:
filename = os.path.join(data_directory, os.path.split(f)[-1])
shutil.copyfile(f, filename)
out.write('Copied data file %s to %s\n' % \
(f, filename))
for tag, ext in self._more_data_file_keys:
filename_out = os.path.join(data_directory,
'%s.%s' % (tag.replace(' ', '_'), ext))
filename_in = self._more_data_files[(tag, ext)]
shutil.copyfile(filename_in, filename_out)
out.write('Copied extra data file %s to %s\n' %
(filename_in, filename_out))
out.close()
def get_data_file(self, filename):
'''Return the point where this data file will end up!'''
if not filename in self._data_files:
return filename
data_directory = Environment.generate_directory('DataFiles')
return os.path.join(data_directory, os.path.split(filename)[-1])
def _get_refiner(self):
if self._refiner is None:
# set the working directory for this, based on the hierarchy
# defined herein...
# that would be CRYSTAL_ID/WAVELENGTH/SWEEP/index &c.
if not self.get_wavelength():
wavelength_id = "default"
crystal_id = "default"
else:
wavelength_id = self.get_wavelength().get_name()
crystal_id = self.get_wavelength().get_crystal().get_name()
working_directory = Environment.generate_directory(
[crystal_id, wavelength_id,
self.get_name(), "refine"])
# FIXME the indexer factory should probably be able to
# take self [this object] as input, to help with deciding
# the most appropriate indexer to use... this will certainly
# be the case for the integrater. Maintaining this link
# will also help the system cope with updates (which
# was going to be one of the big problems...)
# 06/SEP/06 no keep these interfaces separate - want to
# keep "pure" interfaces to the programs for reuse, then
# wrap in XStyle.
self._refiner = RefinerFactory.RefinerForXSweep(self)
## set the user supplied lattice if there is one
# if self._user_lattice:
# self._indexer.set_indexer_input_lattice(self._user_lattice)
# self._indexer.set_indexer_user_input_lattice(True)
## and also the cell constants - but only if lattice is
## assigned
# if self._user_cell:
# self._indexer.set_indexer_input_cell(self._user_cell)
# else:
# if self._user_cell:
# raise RuntimeError('cannot assign cell without lattice')
self._refiner.set_working_directory(working_directory)
# if self._frames_to_process:
# frames = self._frames_to_process
# self._refiner.set_frame_wedge(frames[0], frames[1])
# self._refiner.set_indexer_sweep_name(self._name)
self._refiner.add_refiner_indexer(
self.get_epoch(self._frames_to_process[0]), self._get_indexer())
return self._refiner
def _get_indexer(self):
"""Get my indexer, if set, else create a new one from the
factory."""
if self._indexer is None:
# set the working directory for this, based on the hierarchy
# defined herein...
# that would be CRYSTAL_ID/WAVELENGTH/SWEEP/index &c.
if not self.get_wavelength():
wavelength_id = "default"
crystal_id = "default"
project_id = "default"
else:
wavelength_id = self.get_wavelength().get_name()
crystal_id = self.get_wavelength().get_crystal().get_name()
project_id = (self.get_wavelength().get_crystal().get_project(
).get_name())
working_directory = Environment.generate_directory(
[crystal_id, wavelength_id,
self.get_name(), "index"])
# FIXME the indexer factory should probably be able to
# take self [this object] as input, to help with deciding
# the most appropriate indexer to use... this will certainly
# be the case for the integrater. Maintaining this link
# will also help the system cope with updates (which
# was going to be one of the big problems...)
# 06/SEP/06 no keep these interfaces separate - want to
# keep "pure" interfaces to the programs for reuse, then
# wrap in XStyle.
self._indexer = IndexerFactory.IndexerForXSweep(self)
# set the user supplied lattice if there is one
if self._user_lattice:
self._indexer.set_indexer_input_lattice(self._user_lattice)
self._indexer.set_indexer_user_input_lattice(True)
# and also the cell constants - but only if lattice is
# assigned
if self._user_cell:
self._indexer.set_indexer_input_cell(self._user_cell)
else:
if self._user_cell:
raise RuntimeError("cannot assign cell without lattice")
self._indexer.set_working_directory(working_directory)
self._indexer.set_indexer_project_info(project_id, crystal_id,
wavelength_id)
self._indexer.set_indexer_sweep_name(self._name)
return self._indexer
def _get_scaler(self):
if self._scaler is None:
# in here check if
#
# (1) self._scaled_merged_reflections is set and
# (2) there is no sweep information
#
# if both of these are true then produce a null scaler
# which will wrap this information
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = "scale"
working_directory = Environment.generate_directory(
[self._name, scale_dir])
self._scaler = Scaler()
# put an inverse link in place... to support RD analysis
# involved change to Scaler interface definition
self._scaler.set_scaler_xcrystal(self)
if self._anomalous:
self._scaler.set_scaler_anomalous(True)
# set up a sensible working directory
self._scaler.set_working_directory(working_directory)
# set the reference reflection file, if we have one...
if self._reference_reflection_file:
self._scaler.set_scaler_reference_reflection_file(
self._reference_reflection_file)
# and FreeR file
if self._freer_file:
self._scaler.set_scaler_freer_file(self._freer_file)
# and spacegroup information
if self._user_spacegroup:
# compute the lattice and pointgroup from this...
pointgroup = Syminfo.get_pointgroup(self._user_spacegroup)
self._scaler.set_scaler_input_spacegroup(self._user_spacegroup)
self._scaler.set_scaler_input_pointgroup(pointgroup)
integraters = self._get_integraters()
# then feed them to the scaler
for i in integraters:
self._scaler.add_scaler_integrater(i)
return self._scaler
def _get_refiner(self):
if self._refiner is None:
# set the working directory for this, based on the hierarchy
# defined herein...
# that would be CRYSTAL_ID/WAVELENGTH/SWEEP/index &c.
if not self.get_wavelength():
wavelength_id = "default"
crystal_id = "default"
else:
wavelength_id = self.get_wavelength().get_name()
crystal_id = self.get_wavelength().get_crystal().get_name()
working_directory = Environment.generate_directory(
[crystal_id, wavelength_id, self.get_name(), 'refine'])
# FIXME the indexer factory should probably be able to
# take self [this object] as input, to help with deciding
# the most appropriate indexer to use... this will certainly
# be the case for the integrater. Maintaining this link
# will also help the system cope with updates (which
# was going to be one of the big problems...)
# 06/SEP/06 no keep these interfaces separate - want to
# keep "pure" interfaces to the programs for reuse, then
# wrap in XStyle.
self._refiner = RefinerFactory.RefinerForXSweep(self)
## set the user supplied lattice if there is one
#if self._user_lattice:
#self._indexer.set_indexer_input_lattice(self._user_lattice)
#self._indexer.set_indexer_user_input_lattice(True)
## and also the cell constants - but only if lattice is
## assigned
#if self._user_cell:
#self._indexer.set_indexer_input_cell(self._user_cell)
#else:
#if self._user_cell:
#raise RuntimeError, 'cannot assign cell without lattice'
self._refiner.set_working_directory(working_directory)
#if self._frames_to_process:
#frames = self._frames_to_process
#self._refiner.set_frame_wedge(frames[0], frames[1])
#self._refiner.set_indexer_sweep_name(self._name)
self._refiner.add_refiner_indexer(
self.get_epoch(self._frames_to_process[0]), self._get_indexer())
return self._refiner
def run():
if os.path.exists("xia2-working.phil"):
sys.argv.append("xia2-working.phil")
try:
check_environment()
except Exception as e:
traceback.print_exc(file=open("xia2.error", "w"))
Chatter.write('Status: error "%s"' % str(e))
# print the version
Chatter.write(Version)
Citations.cite("xia2")
start_time = time.time()
assert os.path.exists("xia2.json")
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
# cwd = os.path.abspath(os.curdir)
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = "scale"
i = 0
while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
i += 1
scale_dir = "scale%i" % i
PhilIndex.params.xia2.settings.scale.directory = scale_dir
working_directory = Environment.generate_directory(
[crystal.get_name(), scale_dir])
# os.chdir(working_directory)
crystals[crystal_id]._scaler = None # reset scaler
scaler = crystal._get_scaler()
Chatter.write(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write("Processing took %s" %
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
# delete all of the temporary mtz files...
cleanup()
write_citations()
xinfo.as_json(filename="xia2.json")
Environment.cleanup()
def _get_scaler(self):
if self._scaler is None:
# in here check if
#
# (1) self._scaled_merged_reflections is set and
# (2) there is no sweep information
#
# if both of these are true then produce a null scaler
# which will wrap this information
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = 'scale'
working_directory = Environment.generate_directory([self._name, scale_dir])
self._scaler = Scaler()
# put an inverse link in place... to support RD analysis
# involved change to Scaler interface definition
self._scaler.set_scaler_xcrystal(self)
if self._anomalous:
self._scaler.set_scaler_anomalous(True)
# set up a sensible working directory
self._scaler.set_working_directory(working_directory)
# set the reference reflection file, if we have one...
if self._reference_reflection_file:
self._scaler.set_scaler_reference_reflection_file(
self._reference_reflection_file)
# and FreeR file
if self._freer_file:
self._scaler.set_scaler_freer_file(self._freer_file)
# and spacegroup information
if self._user_spacegroup:
# compute the lattice and pointgroup from this...
pointgroup = Syminfo.get_pointgroup(self._user_spacegroup)
self._scaler.set_scaler_input_spacegroup(
self._user_spacegroup)
self._scaler.set_scaler_input_pointgroup(pointgroup)
integraters = self._get_integraters()
# then feed them to the scaler
for i in integraters:
self._scaler.add_scaler_integrater(i)
return self._scaler
def write_cif(self):
'''Write CIF to file.'''
# update audit information for citations
self.collate_audit_information()
from xia2.Handlers.Environment import Environment
data_directory = Environment.generate_directory('DataFiles')
with open(os.path.join(data_directory, self._outfile), 'w') as fh:
self._cif.show(out=fh)
def _get_indexer(self):
'''Get my indexer, if set, else create a new one from the
factory.'''
if self._indexer is None:
# set the working directory for this, based on the hierarchy
# defined herein...
# that would be CRYSTAL_ID/WAVELENGTH/SWEEP/index &c.
if not self.get_wavelength():
wavelength_id = "default"
crystal_id = "default"
project_id = "default"
else:
wavelength_id = self.get_wavelength().get_name()
crystal_id = self.get_wavelength().get_crystal().get_name()
project_id = self.get_wavelength().get_crystal().get_project().get_name()
working_directory = Environment.generate_directory(
[crystal_id, wavelength_id, self.get_name(), 'index'])
# FIXME the indexer factory should probably be able to
# take self [this object] as input, to help with deciding
# the most appropriate indexer to use... this will certainly
# be the case for the integrater. Maintaining this link
# will also help the system cope with updates (which
# was going to be one of the big problems...)
# 06/SEP/06 no keep these interfaces separate - want to
# keep "pure" interfaces to the programs for reuse, then
# wrap in XStyle.
self._indexer = IndexerFactory.IndexerForXSweep(self)
# set the user supplied lattice if there is one
if self._user_lattice:
self._indexer.set_indexer_input_lattice(self._user_lattice)
self._indexer.set_indexer_user_input_lattice(True)
# and also the cell constants - but only if lattice is
# assigned
if self._user_cell:
self._indexer.set_indexer_input_cell(self._user_cell)
else:
if self._user_cell:
raise RuntimeError, 'cannot assign cell without lattice'
self._indexer.set_working_directory(working_directory)
self._indexer.set_indexer_project_info(
project_id, crystal_id, wavelength_id)
self._indexer.set_indexer_sweep_name(self._name)
return self._indexer
def _generate_absorption_map(self, scaler):
output = scaler.get_all_output()
aimless = 'AIMLESS, CCP4'
import re
pattern = re.compile(" +#+ *CCP4.*#+")
for line in output:
if pattern.search(line):
aimless = re.sub('\s\s+', ', ', line.strip("\t\n #"))
break
from xia2.Toolkit.AimlessSurface import evaluate_1degree, \
scrape_coefficients, generate_map
coefficients = scrape_coefficients(log=output)
if coefficients:
absmap = evaluate_1degree(coefficients)
absmin, absmax = absmap.min(), absmap.max()
else:
absmin, absmax = 1.0, 1.0
block = CIF.get_block('xia2')
mmblock = mmCIF.get_block('xia2')
block["_exptl_absorpt_correction_T_min"] = mmblock["_exptl.absorpt_correction_T_min"] = \
absmin / absmax # = scaled
block["_exptl_absorpt_correction_T_max"] = mmblock["_exptl.absorpt_correction_T_max"] = \
absmax / absmax # = 1
block["_exptl_absorpt_correction_type"] = mmblock["_exptl.absorpt_correction_type"] = \
"empirical"
block["_exptl_absorpt_process_details"] = mmblock["_exptl.absorpt_process_details"] = '''
%s
Scaling & analysis of unmerged intensities, absorption correction using spherical harmonics
''' % aimless
if absmax - absmin > 0.000001:
from xia2.Handlers.Environment import Environment
log_directory = Environment.generate_directory('LogFiles')
mapfile = os.path.join(log_directory, 'absorption_surface.png')
generate_map(absmap, mapfile)
else:
Debug.write("Cannot create absorption surface: map is too flat (min: %f, max: %f)" % (absmin, absmax))
def _generate_absorption_map(self, scaler):
output = scaler.get_all_output()
aimless = 'AIMLESS, CCP4'
import re
pattern = re.compile(" +#+ *CCP4.*#+")
for line in output:
if pattern.search(line):
aimless = re.sub('\s\s+', ', ', line.strip("\t\n #"))
break
from xia2.Toolkit.AimlessSurface import evaluate_1degree, \
scrape_coefficients, generate_map
coefficients = scrape_coefficients(log=output)
if coefficients:
absmap = evaluate_1degree(coefficients)
absmin, absmax = absmap.min(), absmap.max()
else:
absmin, absmax = 1.0, 1.0
block = CIF.get_block('xia2')
mmblock = mmCIF.get_block('xia2')
block["_exptl_absorpt_correction_T_min"] = mmblock["_exptl.absorpt_correction_T_min"] = \
absmin / absmax # = scaled
block["_exptl_absorpt_correction_T_max"] = mmblock["_exptl.absorpt_correction_T_max"] = \
absmax / absmax # = 1
block["_exptl_absorpt_correction_type"] = mmblock["_exptl.absorpt_correction_type"] = \
"empirical"
block["_exptl_absorpt_process_details"] = mmblock["_exptl.absorpt_process_details"] = '''
%s
Scaling & analysis of unmerged intensities, absorption correction using spherical harmonics
''' % aimless
if absmax - absmin > 0.000001:
from xia2.Handlers.Environment import Environment
log_directory = Environment.generate_directory('LogFiles')
mapfile = os.path.join(log_directory, 'absorption_surface.png')
generate_map(absmap, mapfile)
else:
Debug.write("Cannot create absorption surface: map is too flat (min: %f, max: %f)" % (absmin, absmax))
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
from xia2.Wrappers.CCP4.Blend import Blend
from xia2.lib.bits import auto_logfiler
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si['corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
except AttributeError, e:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
linkage_matrix = hand_blender.get_linkage_matrix()
ddict = hand_blender.plot_dendrogram()
rows = []
headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV']
completeness = flex.double()
for i, cluster in clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
for j in cluster['dataset_ids']:
batch_start, batch_end = epoch_to_batches[epochs[j-1]]
sel_cluster |= (
(batches.data() >= batch_start) & (batches.data() <= batch_end))
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
dataset_ids = cluster['dataset_ids']
rows.append(
['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids),
'%.1f' %flex.mean(multiplicities.data().as_double()),
'%.2f' %completeness[-1],
'%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>', '<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
def json_object(self, command_line=''):
result = {}
for crystal in sorted(self._crystals):
xcrystal = self._crystals[crystal]
cell = xcrystal.get_cell()
spacegroup = xcrystal.get_likely_spacegroups()[0]
result['AutoProc'] = {}
tmp = result['AutoProc']
tmp['spaceGroup'] = spacegroup
for name, value in zip(['a', 'b', 'c', 'alpha', 'beta', 'gamma'], cell):
tmp['refinedCell_%s' % name] = value
result['AutoProcScalingContainer'] = {}
tmp = result['AutoProcScalingContainer']
tmp['AutoProcScaling'] = {
'recordTimeStamp': time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime())
}
statistics_all = xcrystal.get_statistics()
reflection_files = xcrystal.get_scaled_merged_reflections()
wavelength_names = xcrystal.get_wavelength_names()
for key in statistics_all.keys():
pname, xname, dname = key
# FIXME should assert that the dname is a
# valid wavelength name
available = statistics_all[key].keys()
stats = []
keys = [
'High resolution limit',
'Low resolution limit',
'Completeness',
'Multiplicity',
'I/sigma',
'Rmerge(I+/-)',
'CC half',
'Anomalous completeness',
'Anomalous correlation',
'Anomalous multiplicity',
'Total observations',
'Total unique',
'Rmeas(I)',
'Rmeas(I+/-)',
'Rpim(I)',
'Rpim(I+/-)',
'Partial Bias'
]
for k in keys:
if k in available:
stats.append(k)
xwavelength = xcrystal.get_xwavelength(dname)
sweeps = xwavelength.get_sweeps()
tmp['AutoProcScalingStatistics'] = []
tmp2 = tmp['AutoProcScalingStatistics']
for j, name in enumerate(
['overall', 'innerShell', 'outerShell']):
statistics_cache = {'scalingStatisticsType':name}
for s in stats:
if s in self._name_map:
n = self._name_map[s]
else:
continue
if isinstance(statistics_all[key][s], type([])):
statistics_cache[n] = statistics_all[key][s][j]
elif isinstance(statistics_all[key][s], type(())):
statistics_cache[n] = statistics_all[key][s][j]
tmp2.append(statistics_cache)
tmp['AutoProcIntegrationContainer'] = []
tmp2 = tmp['AutoProcIntegrationContainer']
for sweep in sweeps:
if '#' in sweep.get_template():
image_name = sweep.get_image_name(0)
else:
image_name = os.path.join(sweep.get_directory(),
sweep.get_template())
cell = sweep.get_integrater_cell()
intgr_tmp = {}
for name, value in zip(['a', 'b', 'c', 'alpha', 'beta', 'gamma'],
cell):
intgr_tmp['cell_%s' % name] = value
# FIXME this is naughty
indxr = sweep._get_indexer()
intgr = sweep._get_integrater()
start, end = intgr.get_integrater_wedge()
intgr_tmp['startImageNumber'] = start
intgr_tmp['endImageNumber'] = end
intgr_tmp['refinedDetectorDistance'] = indxr.get_indexer_distance()
beam = indxr.get_indexer_beam_centre()
intgr_tmp['refinedXBeam'] = beam[0]
intgr_tmp['refinedYBeam'] = beam[1]
tmp2.append(
{'Image':{'fileName':os.path.split(image_name)[-1],
'fileLocation':sanitize(os.path.split(image_name)[0])},
'AutoProcIntegration': intgr_tmp})
# file unpacking nonsense
result['AutoProcProgramContainer'] = {}
tmp = result['AutoProcProgramContainer']
tmp2 = {}
if not command_line:
from xia2.Handlers.CommandLine import CommandLine
command_line = CommandLine.get_command_line()
tmp2['processingCommandLine'] = sanitize(command_line)
tmp2['processingProgram'] = 'xia2'
tmp['AutoProcProgram'] = tmp2
tmp['AutoProcProgramAttachment'] = []
tmp2 = tmp['AutoProcProgramAttachment']
from xia2.Handlers.Environment import Environment
data_directory = Environment.generate_directory('DataFiles')
for k in reflection_files:
reflection_file = reflection_files[k]
if not isinstance(reflection_file, type('')):
continue
reflection_file = FileHandler.get_data_file(reflection_file)
basename = os.path.basename(reflection_file)
if os.path.isfile(os.path.join(data_directory, basename)):
# Use file in DataFiles directory in preference (if it exists)
reflection_file = os.path.join(data_directory, basename)
tmp2.append({
'fileType': 'Result',
'fileName': os.path.split(reflection_file)[-1],
'filePath': sanitize(os.path.split(reflection_file)[0]),
})
tmp2.append({'fileType':'Log',
'fileName':'xia2.txt',
'filePath':sanitize(os.getcwd())})
return result
def write_xml(self, file, command_line=''):
fout = open(file, 'w')
fout.write('<?xml version="1.0"?>')
fout.write('<AutoProcContainer>\n')
for crystal in sorted(self._crystals):
xcrystal = self._crystals[crystal]
cell = xcrystal.get_cell()
spacegroup = xcrystal.get_likely_spacegroups()[0]
fout.write('<AutoProc><spaceGroup>%s</spaceGroup>' % spacegroup)
self.write_refined_cell(fout, cell)
fout.write('</AutoProc>')
fout.write('<AutoProcScalingContainer>')
fout.write('<AutoProcScaling>')
self.write_date(fout)
fout.write('</AutoProcScaling>')
statistics_all = xcrystal.get_statistics()
reflection_files = xcrystal.get_scaled_merged_reflections()
wavelength_names = xcrystal.get_wavelength_names()
for key in statistics_all.keys():
pname, xname, dname = key
# FIXME should assert that the dname is a
# valid wavelength name
available = statistics_all[key].keys()
stats = []
keys = [
'High resolution limit',
'Low resolution limit',
'Completeness',
'Multiplicity',
'I/sigma',
'Rmerge(I+/-)',
'CC half',
'Anomalous completeness',
'Anomalous correlation',
'Anomalous multiplicity',
'Total observations',
'Total unique',
'Rmeas(I)',
'Rmeas(I+/-)',
'Rpim(I)',
'Rpim(I+/-)',
'Partial Bias'
]
for k in keys:
if k in available:
stats.append(k)
xwavelength = xcrystal.get_xwavelength(dname)
sweeps = xwavelength.get_sweeps()
for j, name in enumerate(['overall', 'innerShell', 'outerShell']):
statistics_cache = {}
for s in stats:
if isinstance(statistics_all[key][s], type([])):
statistics_cache[s] = statistics_all[key][s][j]
elif isinstance(statistics_all[key][s], type(())):
statistics_cache[s] = statistics_all[key][s][j]
# send these to be written out
self.write_scaling_statistics(fout, name, statistics_cache)
for sweep in sweeps:
fout.write('<AutoProcIntegrationContainer>\n')
if '#' in sweep.get_template():
image_name = sweep.get_image_name(0)
else:
image_name = os.path.join(sweep.get_directory(),
sweep.get_template())
fout.write('<Image><fileName>%s</fileName>' % \
os.path.split(image_name)[-1])
fout.write('<fileLocation>%s</fileLocation></Image>' %
sanitize(os.path.split(image_name)[0]))
fout.write('<AutoProcIntegration>\n')
cell = sweep.get_integrater_cell()
self.write_cell(fout, cell)
# FIXME this is naughty
intgr = sweep._get_integrater()
start, end = intgr.get_integrater_wedge()
fout.write('<startImageNumber>%d</startImageNumber>' % \
start)
fout.write('<endImageNumber>%d</endImageNumber>' % \
end)
# FIXME this is naughty
indxr = sweep._get_indexer()
fout.write(
'<refinedDetectorDistance>%f</refinedDetectorDistance>' % \
indxr.get_indexer_distance())
beam = indxr.get_indexer_beam_centre()
fout.write('<refinedXBeam>%f</refinedXBeam>' % beam[0])
fout.write('<refinedYBeam>%f</refinedYBeam>' % beam[1])
fout.write('</AutoProcIntegration>\n')
fout.write('</AutoProcIntegrationContainer>\n')
fout.write('</AutoProcScalingContainer>')
# file unpacking nonsense
if not command_line:
from xia2.Handlers.CommandLine import CommandLine
command_line = CommandLine.get_command_line()
fout.write('<AutoProcProgramContainer><AutoProcProgram>')
fout.write('<processingCommandLine>%s</processingCommandLine>' \
% sanitize(command_line))
fout.write('<processingPrograms>xia2</processingPrograms>')
fout.write('</AutoProcProgram>')
from xia2.Handlers.Environment import Environment
data_directory = Environment.generate_directory('DataFiles')
log_directory = Environment.generate_directory('LogFiles')
for k in reflection_files:
reflection_file = reflection_files[k]
if not isinstance(reflection_file, type('')):
continue
reflection_file = FileHandler.get_data_file(reflection_file)
basename = os.path.basename(reflection_file)
if os.path.isfile(os.path.join(data_directory, basename)):
# Use file in DataFiles directory in preference (if it exists)
reflection_file = os.path.join(data_directory, basename)
fout.write('<AutoProcProgramAttachment><fileType>Result')
fout.write('</fileType><fileName>%s</fileName>' % \
os.path.split(reflection_file)[-1])
fout.write('<filePath>%s</filePath>' % \
sanitize(os.path.split(reflection_file)[0]))
fout.write('</AutoProcProgramAttachment>\n')
import glob
g = glob.glob(os.path.join(log_directory, '*merging-statistics.json'))
for merging_stats_json in g:
fout.write('<AutoProcProgramAttachment><fileType>Graph')
fout.write('</fileType><fileName>%s</fileName>' %
os.path.split(merging_stats_json)[-1])
fout.write('<filePath>%s</filePath>' % sanitize(log_directory))
fout.write('</AutoProcProgramAttachment>\n')
# add the xia2.txt file...
fout.write('<AutoProcProgramAttachment><fileType>Log')
fout.write('</fileType><fileName>xia2.txt</fileName>')
fout.write('<filePath>%s</filePath>' % sanitize(os.getcwd()))
fout.write('</AutoProcProgramAttachment>\n')
fout.write('</AutoProcProgramContainer>')
fout.write('</AutoProcContainer>\n')
fout.close()
def load_sweeps_with_common_indexing():
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
import dials # required for gaussian_rs warning
from xia2.Wrappers.Dials.Reindex import Reindex
Citations.cite('dials')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = next(crystals.itervalues())
working_directory = Environment.generate_directory([crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
# Aimless only
epochs = scaler._sweep_handler.get_epochs()
reference_cell = None
reference_lattice = None
reference_vectors = None
reference_wavelength = None
# Reindex each sweep to same setting
all_miller_indices = flex.miller_index()
all_two_thetas = flex.double()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
Chatter.smallbanner(si.get_sweep_name(), True)
Debug.smallbanner(si.get_sweep_name(), True)
intgr = si.get_integrater()
experiments_filename = intgr.get_integrated_experiments()
reflections_filename = intgr.get_integrated_reflections()
refiner = intgr.get_integrater_refiner()
Debug.write('experiment: %s' % experiments_filename)
Debug.write('reflection: %s' % reflections_filename)
# Use setting of first sweep as reference
if reference_vectors is None:
reference_vectors = experiments_filename
# Assume that all sweeps have the same lattice system
if reference_lattice is None:
reference_lattice = refiner.get_refiner_lattice()
else:
assert reference_lattice == refiner.get_refiner_lattice()
Debug.write("lattice: %s" % refiner.get_refiner_lattice())
# Read .json file for sweep
db = ExperimentListFactory.from_json_file(experiments_filename)
# Assume that each file only contains a single experiment
assert (len(db) == 1)
db = db[0]
# Get beam vector
s0 = db.beam.get_unit_s0()
# Use the unit cell of the first sweep as reference
if reference_cell is None:
reference_cell = db.crystal.get_unit_cell()
Debug.write("Reference cell: %s" % str(reference_cell))
dials_reindex = Reindex()
dials_reindex.set_working_directory(working_directory)
dials_reindex.set_cb_op("auto")
dials_reindex.set_reference_filename(reference_vectors)
dials_reindex.set_experiments_filename(experiments_filename)
dials_reindex.set_indexed_filename(reflections_filename)
auto_logfiler(dials_reindex)
dials_reindex.run()
# Assume that all data are collected at same wavelength
if reference_wavelength is None:
reference_wavelength = intgr.get_wavelength()
else:
assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01
Debug.write("wavelength: %f A" % intgr.get_wavelength())
Debug.write("distance: %f mm" % intgr.get_distance())
# Get integrated reflection data
import dials
with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh:
reflections = pickle.load(fh)
selection = reflections.get_flags(reflections.flags.used_in_refinement)
Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index'])))
reflections = reflections.select(selection)
# Filter bad reflections
selection = reflections['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d reflections with negative variance' % \
selection.count(True)
if 'intensity.prf.variance' in reflections:
selection = reflections['intensity.prf.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d profile reflections with negative variance' % \
selection.count(True)
# Find the observed 2theta angles
miller_indices = flex.miller_index()
two_thetas_obs = flex.double()
for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']):
assert hkl != (0, 0, 0)
two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2]))
miller_indices.append(hkl)
# Convert observed 2theta angles to degrees
two_thetas_obs = two_thetas_obs * 180 / 3.14159265359
Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs)))
all_miller_indices.extend(miller_indices)
all_two_thetas.extend(two_thetas_obs)
return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
#cwd = os.path.abspath(os.curdir)
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = 'scale'
i = 0
while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
i += 1
scale_dir = 'scale%i' %i
PhilIndex.params.xia2.settings.scale.directory = scale_dir
working_directory = Environment.generate_directory(
[crystal.get_name(), scale_dir])
#os.chdir(working_directory)
crystals[crystal_id]._scaler = None # reset scaler
scaler = crystal._get_scaler()
Chatter.write(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
# delete all of the temporary mtz files...
def _compute_scaler_statistics(self,
scaled_unmerged_mtz,
selected_band=None,
wave=None):
''' selected_band = (d_min, d_max) with None for automatic determination. '''
# mapping of expected dictionary names to iotbx.merging_statistics attributes
key_to_var = {
'I/sigma': 'i_over_sigma_mean',
'Completeness': 'completeness',
'Low resolution limit': 'd_max',
'Multiplicity': 'mean_redundancy',
'Rmerge(I)': 'r_merge',
#'Wilson B factor':,
'Rmeas(I)': 'r_meas',
'High resolution limit': 'd_min',
'Total observations': 'n_obs',
'Rpim(I)': 'r_pim',
'CC half': 'cc_one_half',
'Total unique': 'n_uniq',
}
anom_key_to_var = {
'Rmerge(I+/-)': 'r_merge',
'Rpim(I+/-)': 'r_pim',
'Rmeas(I+/-)': 'r_meas',
'Anomalous completeness': 'anom_completeness',
'Anomalous correlation': 'anom_half_corr',
'Anomalous multiplicity': 'mean_redundancy',
}
stats = {}
select_result, select_anom_result = None, None
# don't call self.get_scaler_likely_spacegroups() since that calls
# self.scale() which introduced a subtle bug
from cctbx import sgtbx
sg = sgtbx.space_group_info(str(
self._scalr_likely_spacegroups[0])).group()
from xia2.Handlers.Environment import Environment
log_directory = Environment.generate_directory('LogFiles')
merging_stats_file = os.path.join(
log_directory, '%s_%s%s_merging-statistics.txt' %
(self._scalr_pname, self._scalr_xname,
'' if wave is None else '_%s' % wave))
merging_stats_json = os.path.join(
log_directory, '%s_%s%s_merging-statistics.json' %
(self._scalr_pname, self._scalr_xname,
'' if wave is None else '_%s' % wave))
result, select_result, anom_result, select_anom_result = None, None, None, None
n_bins = PhilIndex.params.xia2.settings.merging_statistics.n_bins
import iotbx.merging_statistics
while result is None:
try:
result = self._iotbx_merging_statistics(scaled_unmerged_mtz,
anomalous=False,
n_bins=n_bins)
result.as_json(file_name=merging_stats_json)
with open(merging_stats_file, 'w') as fh:
result.show(out=fh)
four_column_output = selected_band and any(selected_band)
if four_column_output:
select_result = self._iotbx_merging_statistics(
scaled_unmerged_mtz,
anomalous=False,
d_min=selected_band[0],
d_max=selected_band[1],
n_bins=n_bins)
if sg.is_centric():
anom_result = None
anom_key_to_var = {}
else:
anom_result = self._iotbx_merging_statistics(
scaled_unmerged_mtz, anomalous=True, n_bins=n_bins)
stats['Anomalous slope'] = [anom_result.anomalous_np_slope]
if four_column_output:
select_anom_result = self._iotbx_merging_statistics(
scaled_unmerged_mtz,
anomalous=True,
d_min=selected_band[0],
d_max=selected_band[1],
n_bins=n_bins)
except iotbx.merging_statistics.StatisticsErrorNoReflectionsInRange:
# Too few reflections for too many bins. Reduce number of bins and try again.
result = None
n_bins = n_bins - 3
if n_bins > 5:
continue
else:
raise
from six.moves import cStringIO as StringIO
result_cache = StringIO()
result.show(out=result_cache)
for d, r, s in ((key_to_var, result, select_result),
(anom_key_to_var, anom_result, select_anom_result)):
for k, v in d.iteritems():
if four_column_output:
values = (getattr(s.overall, v), getattr(s.bins[0], v),
getattr(s.bins[-1], v), getattr(r.overall, v))
else:
values = (getattr(r.overall,
v), getattr(r.bins[0],
v), getattr(r.bins[-1], v))
if 'completeness' in v:
values = [v_ * 100 for v_ in values]
if values[0] is not None:
stats[k] = values
return stats
def json_object(self, command_line=""):
result = {}
for crystal in sorted(self._crystals):
xcrystal = self._crystals[crystal]
cell = xcrystal.get_cell()
spacegroup = xcrystal.get_likely_spacegroups()[0]
result["AutoProc"] = {}
tmp = result["AutoProc"]
tmp["spaceGroup"] = spacegroup
for name, value in zip(["a", "b", "c", "alpha", "beta", "gamma"],
cell):
tmp["refinedCell_%s" % name] = value
result["AutoProcScalingContainer"] = {}
tmp = result["AutoProcScalingContainer"]
tmp["AutoProcScaling"] = {
"recordTimeStamp":
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
}
statistics_all = xcrystal.get_statistics()
reflection_files = xcrystal.get_scaled_merged_reflections()
wavelength_names = xcrystal.get_wavelength_names()
for key in statistics_all.keys():
pname, xname, dname = key
# FIXME should assert that the dname is a
# valid wavelength name
available = statistics_all[key].keys()
stats = []
keys = [
"High resolution limit",
"Low resolution limit",
"Completeness",
"Multiplicity",
"I/sigma",
"Rmerge(I+/-)",
"CC half",
"Anomalous completeness",
"Anomalous correlation",
"Anomalous multiplicity",
"Total observations",
"Total unique",
"Rmeas(I)",
"Rmeas(I+/-)",
"Rpim(I)",
"Rpim(I+/-)",
"Partial Bias",
]
for k in keys:
if k in available:
stats.append(k)
xwavelength = xcrystal.get_xwavelength(dname)
sweeps = xwavelength.get_sweeps()
tmp["AutoProcScalingStatistics"] = []
tmp2 = tmp["AutoProcScalingStatistics"]
for j, name in enumerate(
["overall", "innerShell", "outerShell"]):
statistics_cache = {"scalingStatisticsType": name}
for s in stats:
if s in self._name_map:
n = self._name_map[s]
else:
continue
if isinstance(statistics_all[key][s], type([])):
statistics_cache[n] = statistics_all[key][s][j]
elif isinstance(statistics_all[key][s], type(())):
statistics_cache[n] = statistics_all[key][s][j]
tmp2.append(statistics_cache)
tmp["AutoProcIntegrationContainer"] = []
tmp2 = tmp["AutoProcIntegrationContainer"]
for sweep in sweeps:
if "#" in sweep.get_template():
image_name = sweep.get_image_name(0)
else:
image_name = os.path.join(sweep.get_directory(),
sweep.get_template())
cell = sweep.get_integrater_cell()
intgr_tmp = {}
for name, value in zip(
["a", "b", "c", "alpha", "beta", "gamma"], cell):
intgr_tmp["cell_%s" % name] = value
# FIXME this is naughty
indxr = sweep._get_indexer()
intgr = sweep._get_integrater()
start, end = intgr.get_integrater_wedge()
intgr_tmp["startImageNumber"] = start
intgr_tmp["endImageNumber"] = end
intgr_tmp[
"refinedDetectorDistance"] = indxr.get_indexer_distance(
)
beam = indxr.get_indexer_beam_centre_raw_image()
intgr_tmp["refinedXBeam"] = beam[0]
intgr_tmp["refinedYBeam"] = beam[1]
tmp2.append({
"Image": {
"fileName": os.path.split(image_name)[-1],
"fileLocation":
sanitize(os.path.split(image_name)[0]),
},
"AutoProcIntegration": intgr_tmp,
})
# file unpacking nonsense
result["AutoProcProgramContainer"] = {}
tmp = result["AutoProcProgramContainer"]
tmp2 = {}
if not command_line:
from xia2.Handlers.CommandLine import CommandLine
command_line = CommandLine.get_command_line()
tmp2["processingCommandLine"] = sanitize(command_line)
tmp2["processingProgram"] = "xia2"
tmp["AutoProcProgram"] = tmp2
tmp["AutoProcProgramAttachment"] = []
tmp2 = tmp["AutoProcProgramAttachment"]
from xia2.Handlers.Environment import Environment
data_directory = Environment.generate_directory("DataFiles")
for k in reflection_files:
reflection_file = reflection_files[k]
if not isinstance(reflection_file, type("")):
continue
reflection_file = FileHandler.get_data_file(reflection_file)
basename = os.path.basename(reflection_file)
if os.path.isfile(os.path.join(data_directory, basename)):
# Use file in DataFiles directory in preference (if it exists)
reflection_file = os.path.join(data_directory, basename)
tmp2.append({
"fileType":
"Result",
"fileName":
os.path.split(reflection_file)[-1],
"filePath":
sanitize(os.path.split(reflection_file)[0]),
})
tmp2.append({
"fileType": "Log",
"fileName": "xia2.txt",
"filePath": sanitize(os.getcwd()),
})
return result
def write_xml(self, file, command_line="", working_phil=None):
if working_phil is not None:
PhilIndex.merge_phil(working_phil)
params = PhilIndex.get_python_object()
fout = open(file, "w")
fout.write('<?xml version="1.0"?>')
fout.write("<AutoProcContainer>\n")
for crystal in sorted(self._crystals):
xcrystal = self._crystals[crystal]
cell = xcrystal.get_cell()
spacegroup = xcrystal.get_likely_spacegroups()[0]
fout.write("<AutoProc><spaceGroup>%s</spaceGroup>" % spacegroup)
self.write_refined_cell(fout, cell)
fout.write("</AutoProc>")
fout.write("<AutoProcScalingContainer>")
fout.write("<AutoProcScaling>")
self.write_date(fout)
fout.write("</AutoProcScaling>")
statistics_all = xcrystal.get_statistics()
reflection_files = xcrystal.get_scaled_merged_reflections()
wavelength_names = xcrystal.get_wavelength_names()
for key in statistics_all.keys():
pname, xname, dname = key
# FIXME should assert that the dname is a
# valid wavelength name
available = statistics_all[key].keys()
stats = []
keys = [
"High resolution limit",
"Low resolution limit",
"Completeness",
"Multiplicity",
"I/sigma",
"Rmerge(I+/-)",
"CC half",
"Anomalous completeness",
"Anomalous correlation",
"Anomalous multiplicity",
"Total observations",
"Total unique",
"Rmeas(I)",
"Rmeas(I+/-)",
"Rpim(I)",
"Rpim(I+/-)",
"Partial Bias",
]
for k in keys:
if k in available:
stats.append(k)
xwavelength = xcrystal.get_xwavelength(dname)
sweeps = xwavelength.get_sweeps()
for j, name in enumerate(
["overall", "innerShell", "outerShell"]):
statistics_cache = {}
for s in stats:
if isinstance(statistics_all[key][s], type([])):
statistics_cache[s] = statistics_all[key][s][j]
elif isinstance(statistics_all[key][s], type(())):
statistics_cache[s] = statistics_all[key][s][j]
# send these to be written out
self.write_scaling_statistics(fout, name, statistics_cache)
for sweep in sweeps:
fout.write("<AutoProcIntegrationContainer>\n")
if "#" in sweep.get_template():
image_name = sweep.get_image_name(0)
else:
image_name = os.path.join(sweep.get_directory(),
sweep.get_template())
fout.write("<Image><fileName>%s</fileName>" %
os.path.split(image_name)[-1])
fout.write("<fileLocation>%s</fileLocation></Image>" %
sanitize(os.path.split(image_name)[0]))
fout.write("<AutoProcIntegration>\n")
cell = sweep.get_integrater_cell()
self.write_cell(fout, cell)
# FIXME this is naughty
intgr = sweep._get_integrater()
start, end = intgr.get_integrater_wedge()
fout.write("<startImageNumber>%d</startImageNumber>" %
start)
fout.write("<endImageNumber>%d</endImageNumber>" % end)
# FIXME this is naughty
indxr = sweep._get_indexer()
fout.write(
"<refinedDetectorDistance>%f</refinedDetectorDistance>"
% indxr.get_indexer_distance())
beam = indxr.get_indexer_beam_centre_raw_image()
fout.write("<refinedXBeam>%f</refinedXBeam>" % beam[0])
fout.write("<refinedYBeam>%f</refinedYBeam>" % beam[1])
fout.write("</AutoProcIntegration>\n")
fout.write("</AutoProcIntegrationContainer>\n")
fout.write("</AutoProcScalingContainer>")
# file unpacking nonsense
if not command_line:
from xia2.Handlers.CommandLine import CommandLine
command_line = CommandLine.get_command_line()
pipeline = params.xia2.settings.pipeline
fout.write("<AutoProcProgramContainer><AutoProcProgram>")
fout.write("<processingCommandLine>%s</processingCommandLine>" %
sanitize(command_line))
fout.write("<processingPrograms>xia2 %s</processingPrograms>" %
pipeline)
fout.write("</AutoProcProgram>")
from xia2.Handlers.Environment import Environment
data_directory = Environment.generate_directory("DataFiles")
log_directory = Environment.generate_directory("LogFiles")
for k in reflection_files:
reflection_file = reflection_files[k]
if not isinstance(reflection_file, type("")):
continue
reflection_file = FileHandler.get_data_file(reflection_file)
basename = os.path.basename(reflection_file)
if os.path.isfile(os.path.join(data_directory, basename)):
# Use file in DataFiles directory in preference (if it exists)
reflection_file = os.path.join(data_directory, basename)
fout.write("<AutoProcProgramAttachment><fileType>Result")
fout.write("</fileType><fileName>%s</fileName>" %
os.path.split(reflection_file)[-1])
fout.write("<filePath>%s</filePath>" %
sanitize(os.path.split(reflection_file)[0]))
fout.write("</AutoProcProgramAttachment>\n")
import glob
g = glob.glob(
os.path.join(log_directory, "*merging-statistics.json"))
for merging_stats_json in g:
fout.write("<AutoProcProgramAttachment><fileType>Graph")
fout.write("</fileType><fileName>%s</fileName>" %
os.path.split(merging_stats_json)[-1])
fout.write("<filePath>%s</filePath>" % sanitize(log_directory))
fout.write("</AutoProcProgramAttachment>\n")
# add the xia2.txt file...
fout.write("<AutoProcProgramAttachment><fileType>Log")
fout.write("</fileType><fileName>xia2.txt</fileName>")
fout.write("<filePath>%s</filePath>" % sanitize(os.getcwd()))
fout.write("</AutoProcProgramAttachment>\n")
fout.write("</AutoProcProgramContainer>")
fout.write("</AutoProcContainer>\n")
fout.close()
def _get_integrater(self):
'''Get my integrater, and if it is not set, create one.'''
if self._integrater is None:
# set the working directory for this, based on the hierarchy
# defined herein...
# that would be CRYSTAL_ID/WAVELENGTH/SWEEP/index &c.
if not self.get_wavelength():
wavelength_id = "default"
crystal_id = "default"
project_id = "default"
else:
wavelength_id = self.get_wavelength().get_name()
crystal_id = self.get_wavelength().get_crystal().get_name()
project_id = self.get_wavelength().get_crystal().get_project().get_name()
working_directory = Environment.generate_directory(
[crystal_id, wavelength_id, self.get_name(), 'integrate'])
self._integrater = IntegraterFactory.IntegraterForXSweep(self)
# configure the integrater with the indexer - unless
# we don't want to...
self._integrater.set_integrater_refiner(self._get_refiner())
Debug.write('Integrater / refiner / indexer for sweep %s: %s/%s/%s' % \
(self._name, self._integrater.__class__.__name__,
self._get_refiner().__class__.__name__,
self._get_indexer().__class__.__name__))
# or if we have been told this on the command-line -
# N.B. should really add a mechanism to specify the ice
# rings we want removing, #1317.
if PhilIndex.params.xia2.settings.integration.exclude_ice_regions:
Debug.write('Ice ring region exclusion ON')
self._integrater.set_integrater_ice(True)
# or if we were told about ice or specific excluded resolution
# ranges via the xinfo file
if self._ice:
self._integrater.set_integrater_ice(self._ice)
if self._excluded_regions:
self._integrater.set_integrater_excluded_regions(
self._excluded_regions)
self._integrater.set_integrater_project_info(
project_id, crystal_id, wavelength_id)
self._integrater.set_integrater_sweep_name(self._name)
# copy across anomalous flags in case it's useful - #871
self._integrater.set_integrater_anomalous(
self.get_wavelength().get_crystal().get_anomalous())
# see if we have any useful detector parameters to pass on
if self.get_gain():
self._integrater.set_gain(self.get_gain())
if self.get_polarization():
self._integrater.set_polarization(self.get_polarization())
# look to see if there are any global integration parameters
# we can set...
if global_integration_parameters.get_parameters(crystal_id):
Debug.write('Using integration parameters for crystal %s' \
% crystal_id)
self._integrater.set_integrater_parameters(
global_integration_parameters.get_parameters(crystal_id))
# frames to process...
if self._frames_to_process:
self._integrater._setup_from_imageset(self.get_imageset())
#frames = self._frames_to_process
#self._integrater.set_integrater_wedge(frames[0],
#frames[1])
#self._integrater.set_frame_wedge(frames[0],
#frames[1])
self._integrater.set_integrater_epoch(
self.get_epoch(self._frames_to_process[0]))
self._integrater.set_working_directory(working_directory)
return self._integrater
def _get_integrater(self):
"""Get my integrater, and if it is not set, create one."""
if self._integrater is None:
# set the working directory for this, based on the hierarchy
# defined herein...
# that would be CRYSTAL_ID/WAVELENGTH/SWEEP/index &c.
if not self.get_wavelength():
wavelength_id = "default"
crystal_id = "default"
project_id = "default"
else:
wavelength_id = self.get_wavelength().get_name()
crystal_id = self.get_wavelength().get_crystal().get_name()
project_id = (self.get_wavelength().get_crystal().get_project(
).get_name())
working_directory = Environment.generate_directory(
[crystal_id, wavelength_id,
self.get_name(), "integrate"])
self._integrater = IntegraterFactory.IntegraterForXSweep(self)
# configure the integrater with the indexer - unless
# we don't want to...
self._integrater.set_integrater_refiner(self._get_refiner())
Debug.write(
"Integrater / refiner / indexer for sweep %s: %s/%s/%s" % (
self._name,
self._integrater.__class__.__name__,
self._get_refiner().__class__.__name__,
self._get_indexer().__class__.__name__,
))
# or if we have been told this on the command-line -
# N.B. should really add a mechanism to specify the ice
# rings we want removing, #1317.
if PhilIndex.params.xia2.settings.integration.exclude_ice_regions:
Debug.write("Ice ring region exclusion ON")
self._integrater.set_integrater_ice(True)
# or if we were told about ice or specific excluded resolution
# ranges via the xinfo file
if self._ice:
self._integrater.set_integrater_ice(self._ice)
if self._excluded_regions:
self._integrater.set_integrater_excluded_regions(
self._excluded_regions)
self._integrater.set_integrater_project_info(
project_id, crystal_id, wavelength_id)
self._integrater.set_integrater_sweep_name(self._name)
# copy across anomalous flags in case it's useful - #871
self._integrater.set_integrater_anomalous(
self.get_wavelength().get_crystal().get_anomalous())
# see if we have any useful detector parameters to pass on
if self.get_gain():
self._integrater.set_gain(self.get_gain())
if self.get_polarization():
self._integrater.set_polarization(self.get_polarization())
# look to see if there are any global integration parameters
# we can set...
if global_integration_parameters.get_parameters(crystal_id):
Debug.write("Using integration parameters for crystal %s" %
crystal_id)
self._integrater.set_integrater_parameters(
global_integration_parameters.get_parameters(crystal_id))
# frames to process...
if self._frames_to_process:
self._integrater._setup_from_imageset(self.get_imageset())
# frames = self._frames_to_process
# self._integrater.set_integrater_wedge(frames[0],
# frames[1])
# self._integrater.set_frame_wedge(frames[0],
# frames[1])
self._integrater.set_integrater_epoch(
self.get_epoch(self._frames_to_process[0]))
self._integrater.set_working_directory(working_directory)
return self._integrater
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
epoch_to_experiments_filename = {}
epoch_to_experiments = {}
sweep_name_to_epoch = {}
epoch_to_first_image = {}
from dxtbx.serialize import load
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si[
'corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
sweep_name_to_epoch[si['name']] = epoch
intgr = si['integrater']
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
except AttributeError:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
sweep_name_to_epoch[si.get_sweep_name()] = epoch
intgr = si.get_integrater()
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection
sp_json_files = {}
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
sp = StereographicProjection()
auto_logfiler(sp)
sp.set_working_directory(working_directory)
for experiments in epoch_to_experiments_filename.values():
sp.add_experiments(experiments)
sp.set_hkl(hkl)
sp.run()
sp_json_files[hkl] = sp.get_json_filename()
unmerged_mtz = scaler.get_scaled_reflections(
'mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity
mult_json_files = {}
for axis in ('h', 'k', 'l'):
pm = PlotMultiplicity()
auto_logfiler(pm)
pm.set_working_directory(working_directory)
pm.set_mtz_filename(unmerged_mtz)
pm.set_slice_axis(axis)
pm.set_show_missing(True)
pm.run()
mult_json_files[axis] = pm.get_json_filename()
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Wrappers.CCP4.Blend import Blend
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %
hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
ddict = hand_blender.plot_dendrogram()
phil_files_dir = 'phil_files'
if not os.path.exists(phil_files_dir):
os.makedirs(phil_files_dir)
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV',
'aLCV', 'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in clusters.iteritems():
print i
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['dataset_ids']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['dataset_ids']
assert min(dataset_ids) > 0
with open(
os.path.join(phil_files_dir,
'blend_cluster_%i_images.phil' % i),
'wb') as f:
sweep_names = [
hand_blender._labels[dataset_id - 1]
for dataset_id in dataset_ids
]
for sweep_name in sweep_names:
expts = epoch_to_experiments.get(
sweep_name_to_epoch.get(sweep_name))
assert expts is not None, (
sweep_name, sweep_name_to_epoch.get(sweep_name))
assert len(expts) == 1, len(expts)
expt = expts[0]
print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path(
0)
rows.append([
'%i' % i,
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['lcv'],
'%.2f' % cluster['alcv'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
# XXX what about multiple wavelengths?
with open('batches.phil', 'wb') as f:
try:
for epoch, si in scaler._sweep_information.iteritems():
print >> f, "batch {"
print >> f, " id=%s" % si['sname']
print >> f, " range=%i,%i" % tuple(si['batches'])
print >> f, "}"
except AttributeError:
for epoch in scaler._sweep_handler.get_epochs():
si = scaler._sweep_handler.get_sweep_information(epoch)
print >> f, "batch {"
print >> f, " id=%s" % si.get_sweep_name()
print >> f, " range=%i,%i" % tuple(si.get_batches())
print >> f, "}"
from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis
mca = MultiCrystalAnalysis()
auto_logfiler(mca, extra="MultiCrystalAnalysis")
mca.add_command_line_args([
scaler.get_scaled_reflections(format="sca_unmerged").values()[0],
"unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()),
"batches.phil"
])
mca.set_working_directory(working_directory)
mca.run()
intensity_clusters = mca.get_clusters()
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height',
'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in intensity_clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['datasets']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['datasets']
rows.append([
'%i' % int(i),
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['height'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print 'Intensity clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
intensity_clustering_html = tabulate(
rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
import json
json_data = {}
if ddict is not None:
from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json
json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict)
else:
json_data['blend_dendrogram'] = {'data': [], 'layout': {}}
json_data['intensity_clustering'] = mca.get_dict()
del json_data['intensity_clustering']['clusters']
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
with open(sp_json_files[hkl], 'rb') as f:
d = json.load(f)
d['layout'][
'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl
json_data['stereographic_projection_%s%s%s' % hkl] = d
for axis in ('h', 'k', 'l'):
with open(mult_json_files[axis], 'rb') as f:
json_data['multiplicity_%s' % axis] = json.load(f)
json_str = json.dumps(json_data, indent=2)
javascript = ['var graphs = %s' % (json_str)]
javascript.append(
'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);'
)
javascript.append(
'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);'
)
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
javascript.append(
'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);'
% ({
'hkl': "%s%s%s" % hkl
}))
for axis in ('h', 'k', 'l'):
javascript.append(
'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);'
% ({
'axis': axis
}))
html_header = '''
<head>
<!-- Plotly.js -->
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8">
<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
<script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
<style type="text/css">
body {
/*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/
margin: 8px;
min-width: 240px;
margin-left: 5%;
margin-right: 5%;
}
.plot {
float: left;
width: 1200px;
height: 800px;
margin-bottom: 20px;
}
.square_plot {
float: left;
width: 800px;
height: 800px;
margin-bottom: 20px;
}
</style>
</head>
'''
html_body = '''
<body>
<div class="page-header">
<h1>Multi-crystal analysis report</h1>
</div>
<div class="panel-group">
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity">
<h4 class="panel-title">
<a>Multiplicity plots</a>
</h4>
</div>
<div id="collapse_multiplicity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection">
<h4 class="panel-title">
<a>Stereographic projections</a>
</h4>
</div>
<div id="collapse_stereographic_projection" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_cell">
<h4 class="panel-title">
<a>Unit cell clustering</a>
</h4>
</div>
<div id="collapse_cell" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div>
<div class="table-responsive" style="width: 800px">
%(blend_html)s
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_intensity">
<h4 class="panel-title">
<a>Intensity clustering</a>
</h4>
</div>
<div id="collapse_intensity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div>
<div class="table-responsive" style="width: 800px">
%(intensity_clustering_html)s
</div>
</div>
</div>
</div>
</div>
<script>
%(script)s
</script>
</body>
''' % {
'script': '\n'.join(javascript),
'blend_html': blend_html,
'intensity_clustering_html': intensity_clustering_html
}
html = '\n'.join([html_header, html_body])
print "Writing html report to: %s" % 'multi-crystal-report.html'
with open('multi-crystal-report.html', 'wb') as f:
print >> f, html.encode('ascii', 'xmlcharrefreplace')
write_citations()
Environment.cleanup()
return