def _scale_finish_chunk_5_finish_small_molecule(self):
# keep 'mtz' and remove 'mtz_merged' from the dictionary for
# consistency with non-small-molecule workflow
self._scalr_scaled_reflection_files['mtz'] = \
self._scalr_scaled_reflection_files['mtz_merged']
del self._scalr_scaled_reflection_files['mtz_merged']
FileHandler.record_data_file(self._scalr_scaled_reflection_files['mtz'])
def _scale(self):
"Perform all of the operations required to deliver the scaled data."
epochs = self._sweep_handler.get_epochs()
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_chef_unmerged(True)
sc.set_new_scales_file("%s.scales" % self._scalr_xname)
user_resolution_limits = {}
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution, name=sname)
else:
sc.add_run(start, end, name=sname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled_test.mtz",
)
)
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
failover = PhilIndex.params.xia2.settings.failover
if failover:
try:
sc.scale()
except RuntimeError as e:
es = str(e)
if (
"bad batch" in es
or "negative scales run" in es
or "no observations" in es
):
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del self._scalr_integraters[epoch]
# then tell the user what is happening
logger.info(
"Sweep %s gave negative scales - removing", sweep.get_name()
)
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
loggraph = sc.parse_ccp4_loggraph()
resolution_info = {}
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if "Analysis against resolution" in key:
dataset = key.split(",")[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
# check in here that there is actually some data to scale..!
if not resolution_info:
raise RuntimeError("no resolution info")
highest_suggested_resolution = self.assess_resolution_limits(
sc.get_unmerged_reflection_file(), user_resolution_limits
)
if not self.get_scaler_done():
logger.debug("Returning as scaling not finished...")
return
batch_info = {}
for key in loggraph:
if "Analysis against Batch" in key:
dataset = key.split(",")[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file(
f"{self._scalr_pname} {self._scalr_xname} aimless", sc.get_log_file()
)
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file("%s_final.scales" % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=xname
)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled.mtz",
)
)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
FileHandler.record_xml_file(
f"{self._scalr_pname} {self._scalr_xname} aimless", sc.get_xmlout()
)
data = sc.get_summary()
scales_file = sc.get_new_scales_file()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = {}
for key in loggraph:
if "standard deviation v. Intensity" in key:
dataset = key.split(",")[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(loggraph[key])
resolution_info = {}
for key in loggraph:
if "Analysis against resolution" in key:
dataset = key.split(",")[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
batch_info = {}
for key in loggraph:
if "Analysis against Batch" in key:
dataset = key.split(",")[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(sc.get_scaled_reflection_files())
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=sname
)
if dname not in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled.mtz",
)
)
sc.set_scalepack()
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
self._update_scaled_unit_cell()
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files["sca"] = {}
self._scalr_scaled_reflection_files["sca_unmerged"] = {}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {}
for key in self._scalr_scaled_refl_files:
hklout = self._scalr_scaled_refl_files[key]
scaout = "%s.sca" % hklout[:-4]
self._scalr_scaled_reflection_files["sca"][key] = scaout
FileHandler.record_data_file(scaout)
scalepack = os.path.join(
os.path.split(hklout)[0],
os.path.split(hklout)[1]
.replace("_scaled", "_scaled_unmerged")
.replace(".mtz", ".sca"),
)
self._scalr_scaled_reflection_files["sca_unmerged"][key] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + ".mtz"
self._scalr_scaled_reflection_files["mtz_unmerged"][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if self._scalr_cell_esd is not None:
# patch .mtz and overwrite unit cell information
import xia2.Modules.Scaler.tools as tools
override_cell = self._scalr_cell_dict.get(
f"{self._scalr_pname}_{self._scalr_xname}_{key}"
)[0]
tools.patch_mtz_unit_cell(mtz_unmerged, override_cell)
tools.patch_mtz_unit_cell(hklout, override_cell)
self._scalr_scaled_reflection_files["mtz_unmerged"][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == "cctbx":
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files["mtz_unmerged"][key],
selected_band=(highest_suggested_resolution, None),
wave=key,
)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
] = stats
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=sname
)
if dname not in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_chef.mtz",
)
)
sc.set_chef_unmerged(True)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
if not PhilIndex.params.dials.fast_mode:
try:
self._generate_absorption_map(sc)
except Exception as e:
# Map generation may fail for number of reasons, eg. matplotlib borken
logger.debug("Could not generate absorption map (%s)", e)
self._scalr_scaled_reflection_files = { }
self._scalr_scaled_reflection_files['sca'] = { }
for key in self._scalr_scaled_refl_files:
f = self._scalr_scaled_refl_files[key]
scaout = '%s.sca' % f[:-4]
m2v = self._factory.Mtz2various()
m2v.set_hklin(f)
m2v.set_hklout(scaout)
m2v.convert()
self._scalr_scaled_reflection_files['sca'][key] = scaout
FileHandler.record_data_file(scaout)
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_intensities(PhilIndex.params.ccp4.aimless.intensities)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit = self._scalr_resolution_limits[(dname, sname)]
def _scale_finish_chunk_6_add_free_r(self):
hklout = os.path.join(
self.get_working_directory(),
'%s_%s_free_temp.mtz' % (self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
scale_params = PhilIndex.params.xia2.settings.scale
if self.get_scaler_freer_file():
# e.g. via .xinfo file
freein = self.get_scaler_freer_file()
Debug.write('Copying FreeR_flag from %s' % freein)
c = self._factory.Cad()
c.set_freein(freein)
c.add_hklin(self._scalr_scaled_reflection_files['mtz_merged'])
c.set_hklout(hklout)
c.copyfree()
elif scale_params.freer_file is not None:
# e.g. via -freer_file command line argument
freein = scale_params.freer_file
Debug.write('Copying FreeR_flag from %s' % freein)
c = self._factory.Cad()
c.set_freein(freein)
c.add_hklin(self._scalr_scaled_reflection_files['mtz_merged'])
c.set_hklout(hklout)
c.copyfree()
else:
if scale_params.free_total:
ntot = scale_params.free_total
# need to get a fraction, so...
nref = MtzUtils.nref_from_mtz(hklin)
free_fraction = float(ntot) / float(nref)
else:
free_fraction = scale_params.free_fraction
f = self._factory.Freerflag()
f.set_free_fraction(free_fraction)
f.set_hklin(self._scalr_scaled_reflection_files['mtz_merged'])
f.set_hklout(hklout)
f.add_free_flag()
# then check that this FreeR set is complete
hklin = hklout
hklout = os.path.join(
self.get_working_directory(),
'%s_%s_free.mtz' % (self._scalr_pname, self._scalr_xname))
# default fraction of 0.05
free_fraction = 0.05
if scale_params.free_fraction:
free_fraction = scale_params.free_fraction
elif scale_params.free_total:
ntot = scale_params.free_total()
# need to get a fraction, so...
nref = MtzUtils.nref_from_mtz(hklin)
free_fraction = float(ntot) / float(nref)
f = self._factory.Freerflag()
f.set_free_fraction(free_fraction)
f.set_hklin(hklin)
f.set_hklout(hklout)
f.complete_free_flag()
# remove 'mtz_merged' from the dictionary - this is made
# redundant by the merged free...
del self._scalr_scaled_reflection_files['mtz_merged']
# changed from mtz_merged_free to plain ol' mtz
self._scalr_scaled_reflection_files['mtz'] = hklout
# record this for future reference
FileHandler.record_data_file(hklout)
def _scale_finish_export_shelxt(self):
'''Read hklin (unmerged reflection file) and generate SHELXT input file
and HKL file'''
from iotbx.reflection_file_reader import any_reflection_file
from iotbx.shelx import writer
from iotbx.shelx.hklf import miller_array_export_as_shelx_hklf
from cctbx.xray.structure import structure
from cctbx.xray import scatterer
for wavelength_name in self._scalr_scaled_refl_files.keys():
prefix = wavelength_name
if len(self._scalr_scaled_refl_files.keys()) == 1:
prefix = 'shelxt'
prefixpath = os.path.join(self.get_working_directory(), prefix)
mtz_unmerged = self._scalr_scaled_reflection_files['mtz_unmerged'][
wavelength_name]
reader = any_reflection_file(mtz_unmerged)
intensities = [
ma for ma in reader.as_miller_arrays(merge_equivalents=False)
if ma.info().labels == ['I', 'SIGI']
][0]
# FIXME do I need to reindex to a conventional setting here
indices = reader.file_content(
).extract_original_index_miller_indices()
intensities = intensities.customized_copy(indices=indices,
info=intensities.info())
with open('%s.hkl' % prefixpath, 'wb') as hkl_file_handle:
# limit values to 4 digits (before decimal point), as this is what shelxt
# writes in its output files, and shelxl seems to read. ShelXL apparently
# does not read values >9999 properly
miller_array_export_as_shelx_hklf(
intensities,
hkl_file_handle,
scale_range=(-9999., 9999.),
normalise_if_format_overflow=True)
crystal_symm = intensities.crystal_symmetry()
unit_cell_dims = self._scalr_cell
unit_cell_esds = self._scalr_cell_esd
cb_op = crystal_symm.change_of_basis_op_to_reference_setting()
if cb_op.c().r().as_hkl() == 'h,k,l':
print('Change of basis to reference setting: %s' % cb_op)
crystal_symm = crystal_symm.change_basis(cb_op)
if str(cb_op) != "a,b,c":
unit_cell_dims = None
unit_cell_esds = None
# Would need to apply operation to cell errors, too. Need a test case for this
# crystal_symm.show_summary()
xray_structure = structure(crystal_symmetry=crystal_symm)
compound = 'CNOH'
if compound:
from xia2.command_line.to_shelx import parse_compound
result = parse_compound(compound)
for element in result:
xray_structure.add_scatterer(
scatterer(label=element, occupancy=result[element]))
wavelength = self._scalr_xcrystal.get_xwavelength(
wavelength_name).get_wavelength()
with open('%s.ins' % prefixpath, 'w') as insfile:
insfile.write(''.join(
writer.generator(xray_structure,
wavelength=wavelength,
full_matrix_least_squares_cycles=0,
title=prefix,
unit_cell_dims=unit_cell_dims,
unit_cell_esds=unit_cell_esds)))
FileHandler.record_data_file('%s.ins' % prefixpath)
FileHandler.record_data_file('%s.hkl' % prefixpath)
def _scale(self):
'''Actually scale all of the data together.'''
from xia2.Handlers.Environment import debug_memory_usage
debug_memory_usage()
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'XSCALE',
{'scaling model':'default (all)'})
epochs = self._sweep_information.keys()
epochs.sort()
xscale = self.XScale()
xscale.set_spacegroup_number(self._xds_spacegroup)
xscale.set_cell(self._scalr_cell)
Debug.write('Set CELL: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
Debug.write('Set SPACEGROUP_NUMBER: %d' % \
self._xds_spacegroup)
Debug.write('Gathering measurements for scaling')
for epoch in epochs:
# get the prepared reflections
reflections = self._sweep_information[epoch][
'prepared_reflections']
# and the get wavelength that this belongs to
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
# and the resolution range for the reflections
intgr = self._sweep_information[epoch]['integrater']
Debug.write('Epoch: %d' % epoch)
Debug.write('HKL: %s (%s/%s)' % (reflections, dname, sname))
resolution_low = intgr.get_integrater_low_resolution()
resolution_high, _ = self._scalr_resolution_limits.get((dname, sname), (0.0, None))
resolution = (resolution_high, resolution_low)
xscale.add_reflection_file(reflections, dname, resolution)
# set the global properties of the sample
xscale.set_crystal(self._scalr_xname)
xscale.set_anomalous(self._scalr_anomalous)
debug_memory_usage()
xscale.run()
scale_factor = xscale.get_scale_factor()
Debug.write('XSCALE scale factor found to be: %e' % scale_factor)
# record the log file
pname = self._scalr_pname
xname = self._scalr_xname
FileHandler.record_log_file('%s %s XSCALE' % \
(pname, xname),
os.path.join(self.get_working_directory(),
'XSCALE.LP'))
# check for outlier reflections and if a number are found
# then iterate (that is, rerun XSCALE, rejecting these outliers)
if not PhilIndex.params.dials.fast_mode and not PhilIndex.params.xds.keep_outliers:
xscale_remove = xscale.get_remove()
if xscale_remove:
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in xscale_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(xscale_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = xscale_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = PhilIndex.params.xds.z_min
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections
if used:
self.set_scaler_done(False)
if not self.get_scaler_done():
Chatter.write('Excluding outlier reflections Z > %.2f' %
PhilIndex.params.xds.z_min)
return
debug_memory_usage()
# now get the reflection files out and merge them with aimless
output_files = xscale.get_output_reflection_files()
wavelength_names = output_files.keys()
# these are per wavelength - also allow for user defined resolution
# limits a la bug # 3183. No longer...
for epoch in self._sweep_information.keys():
input = self._sweep_information[epoch]
intgr = input['integrater']
rkey = input['dname'], input['sname']
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if rkey not in self._user_resolution_limits:
self._scalr_resolution_limits[rkey] = (dmin, None)
self._user_resolution_limits[rkey] = dmin
elif dmin < self._user_resolution_limits[rkey]:
self._scalr_resolution_limits[rkey] = (dmin, None)
self._user_resolution_limits[rkey] = dmin
self._scalr_scaled_refl_files = { }
self._scalr_statistics = { }
max_batches = 0
mtz_dict = { }
project_info = { }
for epoch in self._sweep_information.keys():
pname = self._scalr_pname
xname = self._scalr_xname
dname = self._sweep_information[epoch]['dname']
reflections = os.path.split(
self._sweep_information[epoch]['prepared_reflections'])[-1]
project_info[reflections] = (pname, xname, dname)
for epoch in self._sweep_information.keys():
self._sweep_information[epoch]['scaled_reflections'] = None
debug_memory_usage()
for wavelength in wavelength_names:
hklin = output_files[wavelength]
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
ref = xsh.split_and_convert_xscale_output(
hklin, 'SCALED_', project_info, 1.0 / scale_factor)
for hklout in ref.keys():
for epoch in self._sweep_information.keys():
if os.path.split(self._sweep_information[epoch][
'prepared_reflections'])[-1] == \
os.path.split(hklout)[-1]:
if self._sweep_information[epoch][
'scaled_reflections'] is not None:
raise RuntimeError, 'duplicate entries'
self._sweep_information[epoch][
'scaled_reflections'] = ref[hklout]
del(xsh)
debug_memory_usage()
for epoch in self._sweep_information.keys():
hklin = self._sweep_information[epoch]['scaled_reflections']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
hkl_copy = os.path.join(self.get_working_directory(),
'R_%s' % os.path.split(hklin)[-1])
if not os.path.exists(hkl_copy):
shutil.copyfile(hklin, hkl_copy)
# let's properly listen to the user's resolution limit needs...
if self._user_resolution_limits.get((dname, sname), False):
resolution = self._user_resolution_limits[(dname, sname)]
else:
if PhilIndex.params.xia2.settings.resolution.keep_all_reflections == True:
try:
resolution = intgr.get_detector().get_max_resolution(intgr.get_beam_obj().get_s0())
Debug.write('keep_all_reflections set, using detector limits')
except Exception:
resolution = self._estimate_resolution_limit(hklin)
else:
resolution = self._estimate_resolution_limit(hklin)
Chatter.write('Resolution for sweep %s/%s: %.2f' % \
(dname, sname, resolution))
if (dname, sname) not in self._scalr_resolution_limits:
self._scalr_resolution_limits[(dname, sname)] = (resolution, None)
self.set_scaler_done(False)
else:
if resolution < self._scalr_resolution_limits[(dname, sname)][0]:
self._scalr_resolution_limits[(dname, sname)] = (resolution, None)
self.set_scaler_done(False)
debug_memory_usage()
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
self._sort_together_data_xds()
highest_resolution = min(limit for limit, _ in self._scalr_resolution_limits.values())
self._scalr_highest_resolution = highest_resolution
Debug.write('Scaler highest resolution set to %5.2f' % \
highest_resolution)
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
sdadd_full = 0.0
sdb_full = 0.0
# ---------- FINAL MERGING ----------
sc = self._factory.Aimless()
FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
self._scalr_xname),
sc.get_log_file())
sc.set_resolution(highest_resolution)
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)
if sdadd_full == 0.0 and sdb_full == 0.0:
pass
else:
sc.add_sd_correction('both', 1.0, sdadd_full, sdb_full)
for epoch in epochs:
input = self._sweep_information[epoch]
start, end = (min(input['batches']), max(input['batches']))
rkey = input['dname'], input['sname']
run_resolution_limit, _ = self._scalr_resolution_limits[rkey]
sc.add_run(start, end, exclude = False,
resolution = run_resolution_limit,
name = input['sname'])
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.multi_merge()
FileHandler.record_xml_file('%s %s aimless xml' % (self._scalr_pname,
self._scalr_xname),
sc.get_xmlout())
data = sc.get_summary()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = { }
for key in loggraph.keys():
if 'standard deviation v. Intensity' in key:
dataset = key.split(',')[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(
loggraph[key])
resolution_info = { }
for key in loggraph.keys():
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(
loggraph[key])
# and also radiation damage stuff...
batch_info = { }
for key in loggraph.keys():
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(
loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(
sc.get_scaled_reflection_files())
self._scalr_scaled_reflection_files = { }
# also output the unmerged scalepack format files...
sc = self._factory.Aimless()
sc.set_resolution(highest_resolution)
sc.set_hklin(self._prepared_reflections)
sc.set_scalepack()
for epoch in epochs:
input = self._sweep_information[epoch]
start, end = (min(input['batches']), max(input['batches']))
rkey = input['dname'], input['sname']
run_resolution_limit, _ = self._scalr_resolution_limits[rkey]
sc.add_run(start, end, exclude = False,
resolution = run_resolution_limit,
name = input['sname'])
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.multi_merge()
self._scalr_scaled_reflection_files['sca_unmerged'] = { }
self._scalr_scaled_reflection_files['mtz_unmerged'] = { }
for dataset in sc.get_scaled_reflection_files().keys():
hklout = sc.get_scaled_reflection_files()[dataset]
# then mark the scalepack files for copying...
scalepack = os.path.join(os.path.split(hklout)[0],
os.path.split(hklout)[1].replace(
'_scaled', '_scaled_unmerged').replace('.mtz', '.sca'))
self._scalr_scaled_reflection_files['sca_unmerged'][
dataset] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + '.mtz'
self._scalr_scaled_reflection_files['mtz_unmerged'][dataset] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == 'cctbx':
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files['mtz_unmerged'][key], wave=key)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)] = stats
# convert reflection files to .sca format - use mtz2various for this
self._scalr_scaled_reflection_files['sca'] = { }
self._scalr_scaled_reflection_files['hkl'] = { }
for key in self._scalr_scaled_refl_files:
f = self._scalr_scaled_refl_files[key]
scaout = '%s.sca' % f[:-4]
m2v = self._factory.Mtz2various()
m2v.set_hklin(f)
m2v.set_hklout(scaout)
m2v.convert()
self._scalr_scaled_reflection_files['sca'][key] = scaout
FileHandler.record_data_file(scaout)
if PhilIndex.params.xia2.settings.small_molecule == True:
hklout = '%s.hkl' % f[:-4]
m2v = self._factory.Mtz2various()
m2v.set_hklin(f)
m2v.set_hklout(hklout)
m2v.convert_shelx()
self._scalr_scaled_reflection_files['hkl'][key] = hklout
FileHandler.record_data_file(hklout)
def _scale(self):
'''Perform all of the operations required to deliver the scaled
data.'''
epochs = self._sweep_handler.get_epochs()
if self._scalr_corrections:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'automatic',
'absorption':self._scalr_correct_absorption,
'decay':self._scalr_correct_decay
})
else:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'default'})
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_chef_unmerged(True)
sc.set_new_scales_file('%s.scales' % self._scalr_xname)
user_resolution_limits = {}
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution, name=sname)
else:
sc.add_run(start, end, name=sname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled_test.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
failover = PhilIndex.params.xia2.settings.failover
if failover:
try:
sc.scale()
except RuntimeError as e:
es = str(e)
if 'bad batch' in es or \
'negative scales run' in es or \
'no observations' in es:
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del self._scalr_integraters[epoch]
# then tell the user what is happening
Chatter.write(
'Sweep %s gave negative scales - removing' % \
sweep.get_name())
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
data = sc.get_summary()
loggraph = sc.parse_ccp4_loggraph()
resolution_info = {}
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
highest_resolution = 100.0
highest_suggested_resolution = None
# check in here that there is actually some data to scale..!
if len(resolution_info) == 0:
raise RuntimeError('no resolution info')
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
continue
elif (dname, sname) in user_resolution_limits:
limit = user_resolution_limits[(dname, sname)]
self._scalr_resolution_limits[(dname, sname)] = (limit, None)
if limit < highest_resolution:
highest_resolution = limit
Chatter.write('Resolution limit for %s: %5.2f (user provided)' % \
(dname, limit))
continue
hklin = sc.get_unmerged_reflection_file()
limit, reasoning = self._estimate_resolution_limit(
hklin, batch_range=(start, end))
if PhilIndex.params.xia2.settings.resolution.keep_all_reflections == True:
suggested = limit
if highest_suggested_resolution is None or limit < highest_suggested_resolution:
highest_suggested_resolution = limit
limit = intgr.get_detector().get_max_resolution(intgr.get_beam_obj().get_s0())
self._scalr_resolution_limits[(dname, sname)] = (limit, suggested)
Debug.write('keep_all_reflections set, using detector limits')
Debug.write('Resolution for sweep %s: %.2f' % \
(sname, limit))
if not (dname, sname) in self._scalr_resolution_limits:
self._scalr_resolution_limits[(dname, sname)] = (limit, None)
self.set_scaler_done(False)
if limit < highest_resolution:
highest_resolution = limit
limit, suggested = self._scalr_resolution_limits[(dname, sname)]
if suggested is None or limit == suggested:
reasoning_str = ''
if reasoning:
reasoning_str = ' (%s)' % reasoning
Chatter.write('Resolution for sweep %s/%s: %.2f%s' % \
(dname, sname, limit, reasoning_str))
else:
Chatter.write('Resolution limit for %s/%s: %5.2f (%5.2f suggested)' % \
(dname, sname, limit, suggested))
if highest_suggested_resolution is not None and \
highest_resolution >= (highest_suggested_resolution - 0.004):
Debug.write('Dropping resolution cut-off suggestion since it is'
' essentially identical to the actual resolution limit.')
highest_suggested_resolution = None
self._scalr_highest_resolution = highest_resolution
self._scalr_highest_suggested_resolution = highest_suggested_resolution
if highest_suggested_resolution is not None:
Debug.write('Suggested highest resolution is %5.2f (%5.2f suggested)' % \
(highest_resolution, highest_suggested_resolution))
else:
Debug.write('Scaler highest resolution set to %5.2f' % \
highest_resolution)
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
batch_info = {}
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
self._scalr_xname),
sc.get_log_file())
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=xname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
FileHandler.record_xml_file('%s %s aimless xml' % (self._scalr_pname,
self._scalr_xname),
sc.get_xmlout())
data = sc.get_summary()
scales_file = sc.get_new_scales_file()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = {}
for key in loggraph:
if 'standard deviation v. Intensity' in key:
dataset = key.split(',')[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(loggraph[key])
resolution_info = {}
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
batch_info = {}
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(
sc.get_scaled_reflection_files())
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=sname)
if not dname in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
sc.set_scalepack()
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
self._update_scaled_unit_cell()
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files['sca'] = {}
self._scalr_scaled_reflection_files['sca_unmerged'] = {}
self._scalr_scaled_reflection_files['mtz_unmerged'] = {}
for key in self._scalr_scaled_refl_files:
hklout = self._scalr_scaled_refl_files[key]
scaout = '%s.sca' % hklout[:-4]
self._scalr_scaled_reflection_files['sca'][key] = scaout
FileHandler.record_data_file(scaout)
scalepack = os.path.join(os.path.split(hklout)[0],
os.path.split(hklout)[1].replace(
'_scaled', '_scaled_unmerged').replace('.mtz', '.sca'))
self._scalr_scaled_reflection_files['sca_unmerged'][key] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + '.mtz'
self._scalr_scaled_reflection_files['mtz_unmerged'][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if self._scalr_cell_esd is not None:
# patch .mtz and overwrite unit cell information
import xia2.Modules.Scaler.tools as tools
override_cell = self._scalr_cell_dict.get('%s_%s_%s' %
(self._scalr_pname, self._scalr_xname, key))[0]
tools.patch_mtz_unit_cell(mtz_unmerged, override_cell)
tools.patch_mtz_unit_cell(hklout, override_cell)
self._scalr_scaled_reflection_files['mtz_unmerged'][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == 'cctbx':
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files['mtz_unmerged'][key],
selected_band=(highest_suggested_resolution, None), wave=key)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)] = stats
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=sname)
if not dname in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_chef.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
sc.set_chef_unmerged(True)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
if not PhilIndex.params.dials.fast_mode:
try:
self._generate_absorption_map(sc)
except Exception as e:
# Map generation may fail for number of reasons, eg. matplotlib borken
Debug.write("Could not generate absorption map (%s)" % e)
def _update_scaled_unit_cell(self):
# FIXME this could be brought in-house
params = PhilIndex.params
fast_mode = params.dials.fast_mode
if (params.xia2.settings.integrater == 'dials' and not fast_mode
and params.xia2.settings.scale.two_theta_refine):
from xia2.Wrappers.Dials.TwoThetaRefine import TwoThetaRefine
from xia2.lib.bits import auto_logfiler
Chatter.banner('Unit cell refinement')
# Collect a list of all sweeps, grouped by project, crystal, wavelength
groups = {}
self._scalr_cell_dict = {}
tt_refine_experiments = []
tt_refine_pickles = []
tt_refine_reindex_ops = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pi = '_'.join(si.get_project_info())
intgr = si.get_integrater()
groups[pi] = groups.get(pi, []) + \
[(intgr.get_integrated_experiments(),
intgr.get_integrated_reflections(),
intgr.get_integrater_reindex_operator())]
# Two theta refine the unit cell for each group
p4p_file = os.path.join(self.get_working_directory(),
'%s_%s.p4p' % (self._scalr_pname, self._scalr_xname))
for pi in groups.keys():
tt_grouprefiner = TwoThetaRefine()
tt_grouprefiner.set_working_directory(self.get_working_directory())
auto_logfiler(tt_grouprefiner)
args = zip(*groups[pi])
tt_grouprefiner.set_experiments(args[0])
tt_grouprefiner.set_pickles(args[1])
tt_grouprefiner.set_output_p4p(p4p_file)
tt_refine_experiments.extend(args[0])
tt_refine_pickles.extend(args[1])
tt_refine_reindex_ops.extend(args[2])
reindex_ops = args[2]
from cctbx.sgtbx import change_of_basis_op as cb_op
if self._spacegroup_reindex_operator is not None:
reindex_ops = [(
cb_op(str(self._spacegroup_reindex_operator)) * cb_op(str(op))).as_hkl()
if op is not None else self._spacegroup_reindex_operator
for op in reindex_ops]
tt_grouprefiner.set_reindex_operators(reindex_ops)
tt_grouprefiner.run()
Chatter.write('%s: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % \
tuple([''.join(pi.split('_')[2:])] + list(tt_grouprefiner.get_unit_cell())))
self._scalr_cell_dict[pi] = (tt_grouprefiner.get_unit_cell(), tt_grouprefiner.get_unit_cell_esd(), tt_grouprefiner.import_cif(), tt_grouprefiner.import_mmcif())
if len(groups) > 1:
cif_in = tt_grouprefiner.import_cif()
cif_out = CIF.get_block(pi)
for key in sorted(cif_in.keys()):
cif_out[key] = cif_in[key]
mmcif_in = tt_grouprefiner.import_mmcif()
mmcif_out = mmCIF.get_block(pi)
for key in sorted(mmcif_in.keys()):
mmcif_out[key] = mmcif_in[key]
# Two theta refine everything together
if len(groups) > 1:
tt_refiner = TwoThetaRefine()
tt_refiner.set_working_directory(self.get_working_directory())
tt_refiner.set_output_p4p(p4p_file)
auto_logfiler(tt_refiner)
tt_refiner.set_experiments(tt_refine_experiments)
tt_refiner.set_pickles(tt_refine_pickles)
if self._spacegroup_reindex_operator is not None:
reindex_ops = [(
cb_op(str(self._spacegroup_reindex_operator)) * cb_op(str(op))).as_hkl()
if op is not None else self._spacegroup_reindex_operator
for op in tt_refine_reindex_ops]
tt_refiner.set_reindex_operators(reindex_ops)
tt_refiner.run()
self._scalr_cell = tt_refiner.get_unit_cell()
Chatter.write('Overall: %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f' % tt_refiner.get_unit_cell())
self._scalr_cell_esd = tt_refiner.get_unit_cell_esd()
cif_in = tt_refiner.import_cif()
mmcif_in = tt_refiner.import_mmcif()
else:
self._scalr_cell, self._scalr_cell_esd, cif_in, mmcif_in = self._scalr_cell_dict.values()[0]
if params.xia2.settings.small_molecule == True:
FileHandler.record_data_file(p4p_file)
import dials.util.version
cif_out = CIF.get_block('xia2')
mmcif_out = mmCIF.get_block('xia2')
cif_out['_computing_cell_refinement'] = mmcif_out['_computing.cell_refinement'] = 'DIALS 2theta refinement, %s' % dials.util.version.dials_version()
for key in sorted(cif_in.keys()):
cif_out[key] = cif_in[key]
for key in sorted(mmcif_in.keys()):
mmcif_out[key] = mmcif_in[key]
Debug.write('Unit cell obtained by two-theta refinement')
else:
ami = AnalyseMyIntensities()
ami.set_working_directory(self.get_working_directory())
average_unit_cell, ignore_sg = ami.compute_average_cell(
[self._scalr_scaled_refl_files[key] for key in
self._scalr_scaled_refl_files])
Debug.write('Computed average unit cell (will use in all files)')
self._scalr_cell = average_unit_cell
self._scalr_cell_esd = None
# Write average unit cell to .cif
cif_out = CIF.get_block('xia2')
cif_out['_computing_cell_refinement'] = 'AIMLESS averaged unit cell'
for cell, cifname in zip(self._scalr_cell,
['length_a', 'length_b', 'length_c', 'angle_alpha', 'angle_beta', 'angle_gamma']):
cif_out['_cell_%s' % cifname] = cell
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
self._scalr_cell)
def _scale(self):
"""Perform all of the operations required to deliver the scaled
data."""
sweep_infos = [
self._sweep_handler.get_sweep_information(e)
for e in self._sweep_handler.get_epochs()
]
if self._scalr_corrections:
Journal.block(
"scaling",
self.get_scaler_xcrystal().get_name(),
"Dials",
{
"scaling model": "automatic",
"absorption": self._scalr_correct_absorption,
"decay": self._scalr_correct_decay,
},
)
else:
Journal.block(
"scaling",
self.get_scaler_xcrystal().get_name(),
"Dials",
{"scaling model": "default"},
)
### Set the parameters and datafiles for dials.scale
self._scaler = DialsScale()
self._scaler = self._updated_dials_scaler()
if self._scaled_experiments and self._scaled_reflections:
# going to continue-where-left-off
self._scaler.add_experiments_json(self._scaled_experiments)
self._scaler.add_reflections_file(self._scaled_reflections)
else:
for si in sweep_infos:
self._scaler.add_experiments_json(si.get_experiments())
self._scaler.add_reflections_file(si.get_reflections())
### Set the unmerged mtz filepath
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {}
# First set the unmerged mtz output filename. Note that this is the
# same for MAD datasets too, as need a single unmerged for merging
# stats calc. For the merged mtz this is different.
scaled_unmerged_mtz_path = os.path.join(
self.get_working_directory(),
"%s_%s_scaled_unmerged.mtz" % (self._scalr_pname, self._scalr_xname),
)
self._scaler.set_scaled_unmerged_mtz([scaled_unmerged_mtz_path])
self._scaler.set_crystal_name(self._scalr_xname) # Name goes in mtz
### Set the merged mtz filepath(s), making into account MAD case.
# Find number of dnames (i.e. number of wavelengths)
dnames_set = OrderedSet()
for si in sweep_infos:
dnames_set.add(si.get_project_info()[2])
scaled_mtz_path = os.path.join(
self.get_working_directory(),
"%s_%s_scaled.mtz" % (self._scalr_pname, self._scalr_xname),
)
if len(dnames_set) == 1:
self._scaler.set_scaled_mtz([scaled_mtz_path])
self._scalr_scaled_reflection_files["mtz"] = {
dnames_set[0]: scaled_mtz_path
}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {
dnames_set[0]: scaled_unmerged_mtz_path
}
else:
merged_mtz_files = []
self._scalr_scaled_reflection_files["mtz"] = {}
for dname in dnames_set:
this_mtz_path = scaled_mtz_path.rstrip(".mtz") + ("_%s.mtz" % dname)
merged_mtz_files.append(this_mtz_path)
self._scalr_scaled_reflection_files["mtz"][dname] = scaled_mtz_path
# Note - we aren't logging individual unmerged here as not
# generating until later.
self._scaler.set_scaled_mtz(merged_mtz_files)
### Set the resolution limit if applicable
user_resolution_limits = {}
highest_resolution = 100.0
for si in sweep_infos:
dname = si.get_project_info()[2]
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
# record user resolution here but don't use it until later - why?
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
if (dname, sname) in self._scalr_resolution_limits:
d_min, _ = self._scalr_resolution_limits[(dname, sname)]
if d_min < highest_resolution:
highest_resolution = d_min
if highest_resolution < 99.9:
self._scaler.set_resolution(d_min=highest_resolution)
### Setup final job details and run scale
self._scaler.set_working_directory(self.get_working_directory())
auto_logfiler(self._scaler)
FileHandler.record_log_file(
"%s %s SCALE" % (self._scalr_pname, self._scalr_xname),
self._scaler.get_log_file(),
)
self._scaler.scale()
self._scaled_experiments = self._scaler.get_scaled_experiments()
self._scaled_reflections = self._scaler.get_scaled_reflections()
FileHandler.record_data_file(scaled_unmerged_mtz_path)
# make it so that only scaled.expt and scaled.refl are
# the files that dials.scale knows about, so that if scale is called again,
# scaling resumes from where it left off.
self._scaler.clear_datafiles()
# log datafiles here, picked up from here in commonscaler methods.
if len(dnames_set) == 1:
hklout = copy.deepcopy(self._scaler.get_scaled_mtz()[0])
self._scalr_scaled_refl_files = {dnames_set[0]: hklout}
FileHandler.record_data_file(hklout)
else:
self._scalr_scaled_refl_files = {}
for i, dname in enumerate(dnames_set):
hklout = copy.deepcopy(self._scaler.get_scaled_mtz()[i])
self._scalr_scaled_refl_files[dname] = hklout
FileHandler.record_data_file(hklout)
### Calculate the resolution limit and set done False if applicable
highest_suggested_resolution = self.assess_resolution_limits(
self._scaler.get_unmerged_reflection_file(),
user_resolution_limits,
use_misigma=False,
)
if not self.get_scaler_done():
# reset for when resolution limit applied
Debug.write("Returning as scaling not finished...")
return
### For MAD case, generate individual unmerged mtz for stats.
if len(dnames_set) > 1:
unmerged_mtz_files = []
scaler = DialsScale()
scaler.set_working_directory(self.get_working_directory())
scaler.set_export_mtz_only()
scaler.add_experiments_json(self._scaled_experiments)
scaler.add_reflections_file(self._scaled_reflections)
for dname in dnames_set:
this_mtz_path = scaled_unmerged_mtz_path.rstrip(".mtz") + (
"_%s.mtz" % dname
)
unmerged_mtz_files.append(this_mtz_path)
self._scalr_scaled_reflection_files["mtz_unmerged"][
dname
] = this_mtz_path
scaler.set_scaled_unmerged_mtz(unmerged_mtz_files)
scaler.scale()
for f in scaler.get_scaled_unmerged_mtz(): # a list
FileHandler.record_data_file(f)
# set refls, exps & unmerged mtz names"
if PhilIndex.params.xia2.settings.merging_statistics.source == "cctbx":
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files["mtz_unmerged"][key],
selected_band=(highest_suggested_resolution, None),
wave=key,
)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
] = stats
# Run twotheta refine
self._update_scaled_unit_cell()
