def as_miller_array(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if (base_array_info is None):
base_array_info = miller.array_info(
source_type="scalepack_no_merge_original_index")
crystal_symmetry_from_file = self.crystal_symmetry()
crystal_symmetry = crystal_symmetry_from_file.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry)
result = miller.array(
miller_set=self.unmerged_miller_set(
crystal_symmetry=crystal_symmetry,
force_symmetry=True),
data=self.i_obs,
sigmas=self.sigmas)
if (merge_equivalents):
result = result.merge_equivalents().array()
return (result
.set_info(base_array_info.customized_copy(
labels=["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"],
merged=merge_equivalents,
crystal_symmetry_from_file=crystal_symmetry_from_file))
.set_observation_type_xray_intensity())
def load_reflections_file (self, file_name, **kwds) :
if (isinstance(file_name, unicode)) :
file_name = str(file_name)
if (file_name != "") :
from iotbx.reflection_file_reader import any_reflection_file
from cctbx import miller
from scitbx.array_family import flex
try :
hkl_file = any_reflection_file(file_name)
except Exception, e :
raise Sorry(str(e))
arrays = hkl_file.as_miller_arrays(merge_equivalents=True)
f_obs = f_model = None
for array in arrays :
labels = array.info().label_string()
if labels.startswith("F-obs-filtered") :
f_obs = array
elif labels.startswith("F-model") :
f_model = array
if (f_obs is None) or (f_model is None) :
raise Sorry("This does not appear to be a phenix.refine output "+
"file. The MTZ file should contain data arrays for the filtered "+
"amplitudes (F-obs) and F-model.")
f_delta = f_obs.customized_copy(sigmas=None,
data=flex.abs(f_obs.data()-abs(f_model).data())).set_info(
miller.array_info(labels=["abs(F_obs - F_model)"]))
self.set_miller_array(f_delta)
def as_miller_arrays(self, data_block_name=None,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if base_array_info is None:
base_array_info = miller.array_info(
source=self.file_path, source_type="cif")
if data_block_name is not None:
arrays = self.build_miller_arrays(
data_block_name=data_block_name,
base_array_info=base_array_info).values()
else:
arrays = flat_list([
arrays.values() for arrays in
self.build_miller_arrays(base_array_info=base_array_info).values()])
other_symmetry=crystal_symmetry
for i, array in enumerate(arrays):
if crystal_symmetry is not None:
crystal_symmetry_from_file = array.crystal_symmetry()
crystal_symmetry = crystal_symmetry_from_file.join_symmetry(
other_symmetry=other_symmetry,
force=force_symmetry)
arrays[i] = array.customized_copy(crystal_symmetry=crystal_symmetry)
arrays[i].set_info(array.info())
return arrays
def as_miller_array(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if (base_array_info is None):
base_array_info = miller.array_info(source_type="scalepack_merge")
crystal_symmetry_from_file = self.crystal_symmetry()
if (self.anomalous):
labels = ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]
else:
labels = ["I", "SIGI"]
return (miller.array(
miller_set=miller.set(
crystal_symmetry=crystal_symmetry_from_file.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry),
indices=self.miller_indices,
anomalous_flag=self.anomalous),
data=self.i_obs,
sigmas=self.sigmas)
.set_info(base_array_info.customized_copy(
labels=labels,
crystal_symmetry_from_file=crystal_symmetry_from_file))
.set_observation_type_xray_intensity())
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
from cctbx import miller
from cctbx import crystal
if (crystal_symmetry is None):
crystal_symmetry = crystal.symmetry()
if (base_array_info is None):
base_array_info = miller.array_info(source_type="shelx_hklf")
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=self.indices()).auto_anomalous()
miller_arrays = []
obs = (miller.array(
miller_set=miller_set,
data=self.data(),
sigmas=self.sigmas())
.set_info(base_array_info.customized_copy(labels=["obs", "sigmas"])))
miller_arrays.append(obs)
if (self.alphas() is not None):
miller_arrays.append(miller.array(
miller_set=miller_set,
data=self.alphas())
.set_info(base_array_info.customized_copy(labels=["alphas"])))
return miller_arrays
def get_resolve_map (self) :
map_coeffs, null_value = self.get_ccp4_maps()
if (map_coeffs is None) :
map_coeffs = self._convert_amplitudes_and_phases(f_label="FP",
phi_label="PHIM", fom_label="FOMM", weighted=True)
from cctbx.miller import array_info
info = array_info(labels=["FWT","PHWT"])
return map_coeffs.set_info(info)
def i_obs(self, anomalous_flag=None):
assert "IOBS" in self.data
assert "SIGMA" in self.data
array_info = miller.array_info(source_type="xds_integrate")#, wavelength=)
return miller.array(miller_set=miller.set(crystal_symmetry=self.crystal_symmetry(),
indices=self.hkl,
anomalous_flag=anomalous_flag),
data=self.data["IOBS"],
sigmas=self.data["SIGMA"]).set_info(array_info).set_observation_type_xray_intensity()
def exercise_wavelength():
miller_set = crystal.symmetry(unit_cell=(10, 10, 10, 90, 90, 90), space_group_symbol="P1").miller_set(
indices=flex.miller_index([(1, 2, 3), (4, 5, 6)]), anomalous_flag=False
)
data = flex.double([1, 2])
info = miller.array_info(wavelength=0.9792)
miller_array = miller_set.array(data=data).set_info(info)
mtz_dataset = miller_array.as_mtz_dataset(column_root_label="F")
mtz_dataset.mtz_object().write("tst_iotbx_mtz_wavelength.mtz")
mtz_object = mtz.object(file_name="tst_iotbx_mtz_wavelength.mtz")
miller_array = mtz_object.as_miller_arrays()[0]
assert approx_equal(miller_array.info().wavelength, 0.9792)
def run(sf_file_name,pdb_file_name):
# check if files exist
if not isfile(sf_file_name): raise Sorry('{} is not a file'.format(sf_file_name))
if not isfile(pdb_file_name): raise Sorry('{} is not a file'.format(pdb_file_name))
# start processing file
cs = crystal_symmetry_from_any.extract_from(pdb_file_name)
base_array_info = miller.array_info(crystal_symmetry_from_file=cs)
all_miller_arrays = cif.reader(file_path=sf_file_name).build_miller_arrays(base_array_info=base_array_info)
#
for (data_name, miller_arrays) in all_miller_arrays.iteritems():
print data_name
for ma in miller_arrays.values():
print get_label(ma),ma
print 'wait here'
def __init__(self, input, n_bins, lattice_symmetry_max_delta, completeness_as_non_anomalous=None):
self.completeness_as_non_anomalous = completeness_as_non_anomalous
self.input = input.eliminate_sys_absent(integral_only=True, log=sys.stdout)
self.lattice_symmetry_max_delta = lattice_symmetry_max_delta
if not self.input.is_unique_set_under_symmetry():
print "Merging symmetry-equivalent reflections:"
merged = self.input.merge_equivalents()
merged.show_summary(prefix=" ")
print
self.input = merged.array()
del merged
if input.info() is not None:
self.input.set_info(input.info().customized_copy(merged=True))
else:
self.input.set_info(miller.array_info(merged=True))
self.input.show_comprehensive_summary()
print
self.input.setup_binner(n_bins=n_bins)
self.resolution_range = self.input.resolution_range()
self.change_of_basis_op_to_minimum_cell = self.input.change_of_basis_op_to_minimum_cell()
self.observations = (
self.input.change_basis(cb_op=self.change_of_basis_op_to_minimum_cell).expand_to_p1().map_to_asu()
)
if self.input.anomalous_flag():
self.anom_diffs = (
abs(self.input.anomalous_differences())
.change_basis(cb_op=self.change_of_basis_op_to_minimum_cell)
.expand_to_p1()
.map_to_asu()
)
else:
self.anom_diffs = None
self.minimum_cell_symmetry = crystal.symmetry.change_basis(
self.input, cb_op=self.change_of_basis_op_to_minimum_cell
)
self.intensity_symmetry = self.minimum_cell_symmetry.reflection_intensity_symmetry(
anomalous_flag=self.input.anomalous_flag()
)
self.lattice_group = sgtbx.lattice_symmetry.group(
self.minimum_cell_symmetry.unit_cell(), max_delta=self.lattice_symmetry_max_delta
)
self.lattice_group.expand_inv(sgtbx.tr_vec((0, 0, 0)))
self.lattice_group.make_tidy()
self.lattice_symmetry = crystal.symmetry(
unit_cell=self.minimum_cell_symmetry.unit_cell(),
space_group_info=sgtbx.space_group_info(group=self.lattice_group),
assert_is_compatible_unit_cell=False,
)
def batch_as_miller_array (self,
crystal_symmetry=None,
force_symmetry=False,
base_array_info=None) :
if (base_array_info is None):
base_array_info = miller.array_info(source_type="xds_ascii")
crystal_symmetry_from_file = self.crystal_symmetry()
return miller.array(
miller_set=self.miller_set(
crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry),
data=self.zd).set_info(
base_array_info.customized_copy(
labels=["ZD"],
crystal_symmetry_from_file=crystal_symmetry_from_file,
wavelength=self.wavelength))
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None,
include_unmerged_data=False,
):
assert not include_unmerged_data, "Unmerged data not supported in MTZ"
other_symmetry = crystal_symmetry
if (base_array_info is None):
base_array_info = miller.array_info(source_type="ccp4_mtz")
result = []
for crystal in self.crystals():
try :
unit_cell = crystal.unit_cell()
except ValueError, e :
raise Sorry(str(e))
crystal_symmetry_from_file = cctbx.crystal.symmetry(
unit_cell=unit_cell,
space_group_info=self.space_group_info(),
raise_sorry_if_incompatible_unit_cell=True)
crystal_symmetry = crystal_symmetry_from_file.join_symmetry(
other_symmetry=other_symmetry,
force=force_symmetry)
for dataset in crystal.datasets():
base_dataset_info = base_array_info.customized_copy(
wavelength=dataset.wavelength())
column_groups = self.group_columns(
crystal_symmetry_from_file=crystal_symmetry_from_file,
crystal_symmetry=crystal_symmetry,
base_array_info=base_dataset_info,
dataset=dataset)
for column_group in column_groups:
if (merge_equivalents
and isinstance(column_group.data(), flex.double)
and isinstance(column_group.sigmas(), flex.double)
and column_group.sigmas().size() != 0
and flex.min(column_group.sigmas()) > 0):
merged_column_group = column_group.merge_equivalents().array()
if (merged_column_group.indices().size()
!= column_group.indices().size()):
merged_column_group.set_info(
column_group.info().customized_copy(merged=True))
column_group = merged_column_group
result.append(column_group)
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if (crystal_symmetry is None):
crystal_symmetry = crystal.symmetry()
if (base_array_info is None):
base_array_info = miller.array_info(source_type="cns_index_fobs_sigma")
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=self.indices()).auto_anomalous()
return [miller.array(
miller_set=miller_set,
data=self.data(),
sigmas=self.sigmas())
.set_info(base_array_info.customized_copy(labels=self._names))
.set_observation_type_xray_amplitude()]
def batch_as_miller_array (self,
crystal_symmetry=None,
force_symmetry=False,
base_array_info=None) :
if (base_array_info is None):
base_array_info = miller.array_info(
source_type="scalepack_no_merge_original_index")
crystal_symmetry_from_file = self.crystal_symmetry()
crystal_symmetry = crystal_symmetry_from_file.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry)
return miller.array(
miller_set=self.unmerged_miller_set(
crystal_symmetry=crystal_symmetry,
force_symmetry=True),
data=self.batch_numbers).set_info(
base_array_info.customized_copy(
labels=["BATCH"],
crystal_symmetry_from_file=crystal_symmetry_from_file))
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
crystal_symmetry = self.crystal_symmetry(
crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry)
if (crystal_symmetry is None):
crystal_symmetry = crystal.symmetry(
unit_cell=None,
space_group_info=None)
if (base_array_info is None):
base_array_info = miller.array_info(source_type="cns_reflection_file")
result = []
done = set()
for group_index in xrange(len(self.groups)):
names, miller_indices, hl = self.join_hl_group(group_index)
result.append(self._as_miller_array(
crystal_symmetry, miller_indices, hl).set_info(
base_array_info.customized_copy(labels=names)))
for name in names:
done.add(name)
real_arrays = {}
for rso in self.reciprocal_space_objects.values():
if (rso.name in done): continue
if (not rso.is_real()): continue
real_arrays[rso.name.lower()] = rso
for obs,sigma,obs_type in group_obs_sigma(real_arrays):
result.append(self._as_miller_array(
crystal_symmetry, obs.indices,
obs.real_data(), sigma.real_data(), obs_type).set_info(
base_array_info.customized_copy(labels=[obs.name, sigma.name])))
done.add(obs.name)
done.add(sigma.name)
for rso in self.reciprocal_space_objects.values():
if (rso.name in done): continue
result.append(self._as_miller_array(
crystal_symmetry, rso.indices, rso.data).set_info(
base_array_info.customized_copy(labels=[rso.name])))
done.add(rso.name)
return result
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if (crystal_symmetry is None):
crystal_symmetry = crystal.symmetry()
if (base_array_info is None):
base_array_info = miller.array_info(source_type="solve_fpfm")
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=self.indices(),
anomalous_flag=True)
return [miller.array(
miller_set=miller_set,
data=self.data(),
sigmas=self.sigmas())
.set_info(base_array_info.customized_copy(
labels=["fpfm", "sigma_fpfm"]))
.set_observation_type_xray_amplitude()]
def as_miller_array(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if (base_array_info is None):
base_array_info = miller.array_info(source_type="xds_ascii")
crystal_symmetry_from_file = self.crystal_symmetry()
array = (miller.array(
miller_set=self.miller_set(
crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry),
data=self.iobs,
sigmas=self.sigma_iobs)
.set_info(base_array_info.customized_copy(
labels=["iobs", "sigma_iobs"],
crystal_symmetry_from_file=crystal_symmetry_from_file,
wavelength=self.wavelength))
.set_observation_type_xray_intensity())
if (merge_equivalents) :
info = array.info()
info.merged = True
array = array.merge_equivalents().array().set_info(info)
return array
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
crystal_symmetry_from_file = self.crystal_symmetry()
crystal_symmetry = crystal_symmetry_from_file.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry)
if (base_array_info is None):
base_array_info = miller.array_info(source_type="dtrek_reflnlist")
miller_arrays = []
sigmas=self.column_dict["fSigmaI"]
miller_arrays.append(miller.array(
miller_set=miller.set(
crystal_symmetry=crystal_symmetry,
indices=self.miller_indices,
anomalous_flag=False),
data=self.column_dict["fIntensity"],
sigmas=sigmas)
.select(sigmas > 0)
.set_info(base_array_info.customized_copy(
labels=["Intensity", "SigmaI"],
crystal_symmetry_from_file=crystal_symmetry_from_file))
.set_observation_type_xray_intensity())
if ("fIntensity+" in self.column_dict):
assert "fSigmaI+" in self.column_dict
assert "fIntensity-" in self.column_dict
assert "fSigmaI-" in self.column_dict
if (crystal_symmetry.space_group_info() is None):
space_group = sgtbx.space_group() # P 1
else:
space_group = crystal_symmetry.space_group()
ac = dtrek_ext.anomalous_combined(
space_group,
self.miller_indices,
self.column_dict["fIntensity+"],
self.column_dict["fSigmaI+"],
self.column_dict["fIntensity-"],
self.column_dict["fSigmaI-"])
miller_arrays.append(miller.array(
miller_set=miller.set(
crystal_symmetry=crystal_symmetry,
indices=ac.miller_indices(),
anomalous_flag=True),
data=ac.data(),
sigmas=ac.sigmas())
.set_info(base_array_info.customized_copy(
labels=["Intensity+-", "SigmaI+-"],
crystal_symmetry_from_file=crystal_symmetry_from_file))
.set_observation_type_xray_intensity())
for column_name in self.column_names:
if (column_name in ("nH", "nK", "nL",
"fSigmaI", "fIntensity",
"fIntensity+", "fSigmaI+",
"fIntensity-", "fSigmaI-")):
continue
miller_arrays.append(miller.array(
miller_set=miller.set(
crystal_symmetry=crystal_symmetry,
indices=self.miller_indices,
anomalous_flag=False),
data=self.column_dict[column_name])
.set_info(base_array_info.customized_copy(
labels=[column_name[1:]],
crystal_symmetry_from_file=crystal_symmetry_from_file)))
return miller_arrays
def i_obs(self, anomalous_flag=None):
array_info = miller.array_info(source_type="xds_ascii")#, wavelength=)
return miller.array(self.as_miller_set(anomalous_flag),
data=self.iobs, sigmas=self.sigma_iobs).set_info(array_info).set_observation_type_xray_intensity()
def extract(file_name,
crystal_symmetry,
wavelength_id,
crystal_id,
show_details_if_error,
output_r_free_label,
merge_non_unique_under_symmetry,
map_to_asu,
remove_systematic_absences,
incompatible_flags_to_work_set=False,
ignore_bad_sigmas=False,
extend_flags=False,
return_as_miller_arrays=False,
log=sys.stdout):
import iotbx.cif
from cctbx import miller
base_array_info = miller.array_info(
crystal_symmetry_from_file=crystal_symmetry)
all_miller_arrays = iotbx.cif.reader(file_path=file_name).build_miller_arrays(
base_array_info=base_array_info)
if (len(all_miller_arrays) == 0) :
raise Sorry("No data arrays were found in this CIF file. Please make "+
"sure that the file contains reflection data, rather than the refined "+
"model.")
column_labels = set()
if (extend_flags) :
map_to_asu = True
def get_label(miller_array):
label = None
for l in miller_array.info().labels:
if ('_meas' in l) :
if miller_array.is_xray_amplitude_array():
label = "FOBS"
elif miller_array.is_xray_intensity_array():
label = "IOBS"
elif l.endswith(".phase_meas") :
label = "PHIM"
break
elif ("_calc" in l) :
if miller_array.is_xray_amplitude_array():
label = "FC"
elif miller_array.is_xray_intensity_array():
label = "ICALC"
elif l.endswith(".F_calc") :
label = "FC"
elif l.endswith(".phase_calc") :
label = "PHIC"
break
elif miller_array.anomalous_flag() :
if miller_array.is_xray_amplitude_array():
label = "F"
elif miller_array.is_xray_intensity_array():
label = "I"
break
elif 'status' in l or '_free' in l:
label = output_r_free_label
break
elif miller_array.is_hendrickson_lattman_array():
label = "HL"
elif (miller_array.is_complex_array()) :
if (l.endswith("DELFWT")) :
label = "DELFWT"
break
elif (l.endswith("FWT")) :
label = "FWT"
break
elif (miller_array.is_real_array()) :
if ("pdbx_anom_difference" in l) :
label = "DANO"
break
elif (l.endswith(".fom")) :
label = "FOM"
break
return label
for (data_name, miller_arrays) in all_miller_arrays.iteritems():
for ma in miller_arrays.values():
other_symmetry = crystal_symmetry
try:
crystal_symmetry = other_symmetry.join_symmetry(
other_symmetry=ma.crystal_symmetry(),
force=True)
except AssertionError, e:
str_e = str(e)
from cStringIO import StringIO
s = StringIO()
if "Space group is incompatible with unit cell parameters." in str_e:
other_symmetry.show_summary(f=s)
ma.crystal_symmetry().show_summary(f=s)
str_e += "\n%s" %(s.getvalue())
raise Sorry(str_e)
else:
raise
def exercise_miller_arrays_as_cif_block():
from iotbx.cif import reader
cif_model = reader(input_string=cif_miller_array,
builder=cif.builders.cif_model_builder()).model()
ma_builder = cif.builders.miller_array_builder(cif_model['global'])
ma1 = ma_builder.arrays()['_refln_F_squared_meas']
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1, array_type='meas')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1-1j]*ma1.size())), array_type='calc')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1-2j]*ma1.size())), column_names=[
'_refln_A_calc', '_refln_B_calc'])
for key in ('_refln_F_squared_meas', '_refln_F_squared_sigma',
'_refln_F_calc', '_refln_phase_calc',
'_refln_A_calc', '_refln_A_calc'):
assert key in mas_as_cif_block.cif_block.keys()
#
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1, array_type='meas', format="mmcif")
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1-1j]*ma1.size())), array_type='calc')
for key in ('_refln.F_squared_meas', '_refln.F_squared_sigma',
'_refln.F_calc', '_refln.phase_calc',
'_space_group_symop.operation_xyz',
'_cell.length_a', '_refln.index_h'):
assert key in mas_as_cif_block.cif_block.keys()
#
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1, column_names=['_diffrn_refln_intensity_net',
'_diffrn_refln_intensity_sigma'],
miller_index_prefix='_diffrn_refln')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.std_string(ma1.size(), 'om')),
column_name='_diffrn_refln_intensity_u')
for key in ('_diffrn_refln_intensity_net', '_diffrn_refln_intensity_sigma',
'_diffrn_refln_intensity_u'):
assert key in mas_as_cif_block.cif_block.keys()
#
try: reader(input_string=cif_global)
except CifParserError, e: pass
else: raise Exception_expected
cif_model = reader(input_string=cif_global, strict=False).model()
assert not show_diff(str(cif_model), """\
data_1
_c 3
_d 4
""")
# exercise adding miller arrays with non-matching indices
cs = crystal.symmetry(unit_cell=uctbx.unit_cell((10, 10, 10, 90, 90, 90)),
space_group_info=sgtbx.space_group_info(symbol="P1"))
mi = flex.miller_index(((1,0,0), (1,2,3), (2,3,4)))
ms1 = miller.set(cs, mi)
ma1 = miller.array(ms1, data=flex.double((1,2,3)))
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1, column_name="_refln.F_meas_au")
ms2 = miller.set(cs, mi[:2])
ma2 = miller.array(ms2, data=flex.complex_double([1-2j]*ms2.size()))
mas_as_cif_block.add_miller_array(
ma2, column_names=("_refln.F_calc_au", "_refln.phase_calc")),
ms3 = miller.set(cs, flex.miller_index(((1,0,0), (5,6,7), (2,3,4))))
ma3 = miller.array(ms3, data=flex.double((4,5,6)))
mas_as_cif_block.add_miller_array(ma3, column_name="_refln.F_squared_meas")
ms4 = miller.set(cs, flex.miller_index(((1,2,3), (5,6,7), (1,1,1), (1,0,0), (2,3,4))))
ma4 = ms4.d_spacings()
mas_as_cif_block.add_miller_array(ma4, column_name="_refln.d_spacing")
# extract arrays from cif block and make sure we get back what we started with
arrays = cif.builders.miller_array_builder(mas_as_cif_block.cif_block).arrays()
recycled_arrays = (arrays['_refln.F_meas_au'],
arrays['_refln.F_calc_au'],
arrays['_refln.F_squared_meas'],
arrays['_refln.d_spacing'])
for orig, recycled in zip((ma1, ma2, ma3, ma4), recycled_arrays):
assert orig.size() == recycled.size()
recycled = recycled.customized_copy(anomalous_flag=orig.anomalous_flag())
orig, recycled = orig.common_sets(recycled)
assert orig.indices().all_eq(recycled.indices())
assert approx_equal(orig.data(), recycled.data(), eps=1e-5)
#
cif_model = reader(input_string=r3adrsf,
builder=cif.builders.cif_model_builder()).model()
cs = cif.builders.crystal_symmetry_builder(cif_model["r3adrsf"]).crystal_symmetry
ma_builder = cif.builders.miller_array_builder(
cif_model['r3adrAsf'],
base_array_info=miller.array_info(crystal_symmetry_from_file=cs))
miller_arrays = ma_builder.arrays().values()
assert len(miller_arrays) == 4
mas_as_cif_block = cif.miller_arrays_as_cif_block(
miller_arrays[0].map_to_asu(), column_names=miller_arrays[0].info().labels)
for array in miller_arrays[1:]:
labels = array.info().labels
if len(labels) > 1 :
for label in labels :
if label.startswith("wavelength_id") :
labels.remove(label)
mas_as_cif_block.add_miller_array(
array=array.map_to_asu(), column_names=array.info().labels)
s = StringIO()
print >> s, mas_as_cif_block.refln_loop
assert not show_diff(s.getvalue(), """\
loop_
_refln_index_h
_refln_index_k
_refln_index_l
_refln.crystal_id
_refln.scale_group_code
_refln.wavelength_id
_refln.pdbx_I_plus
_refln.pdbx_I_plus_sigma
_refln.pdbx_I_minus
_refln.pdbx_I_minus_sigma
-87 5 46 1 1 3 40.2 40.4 6.7 63.9
-87 5 45 1 1 3 47.8 29.7 35.1 30.5
-87 5 44 1 1 3 18.1 33.2 0.5 34.6
-87 5 43 1 1 3 6.1 45.4 12.9 51.6
-87 5 42 1 1 3 -6.6 45.6 -15.5 55.8
-87 7 37 1 1 3 6.3 43.4 ? ?
-87 7 36 1 1 3 -67.2 55.4 ? ?
-88 2 44 1 1 3 0 -1 35 38.5
-88 2 43 1 1 3 0 -1 57.4 41.5
-88 4 45 1 1 3 -1 46.1 -9.1 45.6
-88 4 44 1 1 3 -19.8 49.2 0.3 34.7
-88 6 44 1 1 3 -1.8 34.8 ? ?
""")
def __init__(self, cif_block, base_array_info=None, wavelengths=None):
crystal_symmetry_builder.__init__(self, cif_block)
if base_array_info is not None:
self.crystal_symmetry = self.crystal_symmetry.join_symmetry(
other_symmetry=base_array_info.crystal_symmetry_from_file,
force=True)
self._arrays = OrderedDict()
if (wavelengths is None):
wavelengths = {}
if base_array_info is None:
base_array_info = miller.array_info(source_type="cif")
refln_containing_loops = self.get_miller_indices_containing_loops()
for self.indices, refln_loop in refln_containing_loops:
self.wavelength_id_array = None
self.crystal_id_array = None
self.scale_group_array = None
wavelength_ids = [None]
crystal_ids = [None]
scale_groups = [None]
for key, value in refln_loop.iteritems():
# need to get these arrays first
if (key.endswith('wavelength_id') or key.endswith('crystal_id')
or key.endswith('scale_group_code')):
data = as_int_or_none_if_all_question_marks(
value, column_name=key)
if data is None:
continue
counts = data.counts()
if key.endswith('wavelength_id'):
wavelength_ids = counts.keys()
if len(counts) == 1: continue
array = miller.array(
miller.set(self.crystal_symmetry,
self.indices).auto_anomalous(), data)
if key.endswith('wavelength_id'):
self.wavelength_id_array = array
wavelength_ids = counts.keys()
elif key.endswith('crystal_id'):
self.crystal_id_array = array
crystal_ids = counts.keys()
elif key.endswith('scale_group_code'):
self.scale_group_array = array
scale_groups = counts.keys()
for label, value in sorted(refln_loop.items()):
for w_id in wavelength_ids:
for crys_id in crystal_ids:
for scale_group in scale_groups:
if 'index_' in label: continue
key = label
labels = [label]
wavelength = None
if (key.endswith('wavelength_id')
or key.endswith('crystal_id')
or key.endswith('scale_group_code')):
w_id = None
crys_id = None
scale_group = None
key_suffix = ''
if w_id is not None:
key_suffix += '_%i' % w_id
labels.insert(0, "wavelength_id=%i" % w_id)
wavelength = wavelengths.get(w_id, None)
if crys_id is not None:
key_suffix += '_%i' % crys_id
labels.insert(0, "crystal_id=%i" % crys_id)
if scale_group is not None:
key_suffix += '_%i' % scale_group
labels.insert(
0, "scale_group_code=%i" % scale_group)
key += key_suffix
sigmas = None
if key in self._arrays: continue
array = self.flex_std_string_as_miller_array(
value,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
if array is None: continue
if '_sigma' in key:
sigmas_label = label
key = None
for suffix in ('', '_meas', '_calc'):
if sigmas_label.replace(
'_sigma', suffix) in refln_loop:
key = sigmas_label.replace(
'_sigma', suffix) + key_suffix
break
if key is None:
key = sigmas_label + key_suffix
elif key in self._arrays and self._arrays[
key].sigmas() is None:
sigmas = array
array = self._arrays[key]
if (not check_array_sizes(
array, sigmas, key, sigmas_label)):
continue
sigmas = as_flex_double(
sigmas, sigmas_label)
array.set_sigmas(sigmas.data())
info = array.info()
array.set_info(
info.customized_copy(
labels=info.labels +
[sigmas_label],
wavelength=wavelength))
continue
elif 'PHWT' in key:
phwt_label = label
fwt_label = label.replace('PHWT', 'FWT')
if fwt_label not in refln_loop: continue
phwt_array = array
if fwt_label in self._arrays:
array = self._arrays[fwt_label]
if (not check_array_sizes(
array, phwt_array, fwt_label,
phwt_label)):
continue
phases = as_flex_double(
phwt_array, phwt_label)
info = array.info()
array = array.phase_transfer(phases,
deg=True)
array.set_info(
info.customized_copy(
labels=info.labels + [phwt_label]))
self._arrays[fwt_label] = array
continue
elif 'HL_' in key:
hl_letter = key[key.find('HL_') + 3]
hl_key = 'HL_' + hl_letter
key = key.replace(hl_key, 'HL_A')
if key in self._arrays:
continue # this array is already dealt with
hl_labels = [
label.replace(hl_key, 'HL_' + letter)
for letter in 'ABCD'
]
hl_keys = [
key.replace(hl_key, 'HL_' + letter)
for letter in 'ABCD'
]
hl_values = [
cif_block.get(hl_key)
for hl_key in hl_labels
]
if hl_values.count(None) == 0:
selection = self.get_selection(
hl_values[0],
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
hl_values = [
as_double_or_none_if_all_question_marks(
hl.select(selection),
column_name=lab)
for hl, lab in zip(
hl_values, hl_labels)
]
array = miller.array(
miller.set(
self.crystal_symmetry,
self.indices.select(
selection)).auto_anomalous(),
flex.hendrickson_lattman(*hl_values))
labels = labels[:-1] + hl_labels
elif '.B_' in key or '_B_' in key:
if '.B_' in key:
key, key_b = key.replace('.B_', '.A_'), key
label, label_b = label.replace(
'.B_', '.A_'), label
elif '_B_' in key:
key, key_b = key.replace('_B', '_A'), key
label, label_b = label.replace('_B',
'_A'), label
if key in refln_loop and key_b in refln_loop:
b_part = array.data()
if key in self._arrays:
info = self._arrays[key].info()
a_part = self._arrays[key].data()
self._arrays[key] = self._arrays[
key].array(
data=flex.complex_double(
a_part, b_part))
self._arrays[key].set_info(
info.customized_copy(
labels=info.labels + [key_b]))
continue
elif ('phase_' in key and not "_meas" in key
and self.crystal_symmetry.space_group()
is not None):
alt_key1 = label.replace('phase_', 'F_')
alt_key2 = alt_key1 + '_au'
if alt_key1 in refln_loop:
phase_key = label
key = alt_key1 + key_suffix
elif alt_key2 in refln_loop:
phase_key = label
key = alt_key2 + key_suffix
else:
phase_key = None
if phase_key is not None:
phases = array.data()
if key in self._arrays:
array = self._arrays[key]
array = as_flex_double(array, key)
if (not check_array_sizes(
array, phases, key,
phase_key)):
continue
info = self._arrays[key].info()
self._arrays[
key] = array.phase_transfer(
phases, deg=True)
self._arrays[key].set_info(
info.customized_copy(
labels=info.labels +
[phase_key]))
else:
array = self.flex_std_string_as_miller_array(
refln_loop[label],
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
if (not check_array_sizes(
array, phases, key,
phase_key)):
continue
array.phase_transfer(phases, deg=True)
labels = labels + [label, phase_key]
if base_array_info.labels is not None:
labels = base_array_info.labels + labels
def rstrip_substrings(string, substrings):
for substr in substrings:
if substr == '': continue
if string.endswith(substr):
string = string[:-len(substr)]
return string
# determine observation type
stripped_key = rstrip_substrings(
key, [
key_suffix, '_au', '_meas', '_calc',
'_plus', '_minus'
])
if (stripped_key.endswith('F_squared')
or stripped_key.endswith('intensity')
or stripped_key.endswith('.I')
or stripped_key.endswith('_I')) and (
array.is_real_array()
or array.is_integer_array()):
array.set_observation_type_xray_intensity()
elif (stripped_key.endswith('F')
and (array.is_real_array()
or array.is_integer_array())):
array.set_observation_type_xray_amplitude()
if (array.is_xray_amplitude_array()
or array.is_xray_amplitude_array()):
# e.g. merge_equivalents treats integer arrays differently, so must
# convert integer observation arrays here to be safe
if isinstance(array.data(), flex.int):
array = array.customized_copy(
data=array.data().as_double())
array.set_info(
base_array_info.customized_copy(labels=labels))
if (array.is_xray_amplitude_array()
or array.is_xray_amplitude_array()):
info = array.info()
array.set_info(
info.customized_copy(
wavelength=wavelength))
self._arrays.setdefault(key, array)
for key, array in self._arrays.copy().iteritems():
if (key.endswith('_minus') or '_minus_' in key
or key.endswith('_plus') or '_plus_' in key):
if '_minus' in key:
minus_key = key
plus_key = key.replace('_minus', '_plus')
elif '_plus' in key:
plus_key = key
minus_key = key.replace('_plus', '_minus')
if plus_key in self._arrays and minus_key in self._arrays:
plus_array = self._arrays.pop(plus_key)
minus_array = self._arrays.pop(minus_key)
minus_array = minus_array.customized_copy(
indices=-minus_array.indices()).set_info(
minus_array.info())
array = plus_array.concatenate(
minus_array, assert_is_similar_symmetry=False)
array = array.customized_copy(anomalous_flag=True)
array.set_info(
minus_array.info().customized_copy(labels=list(
OrderedSet(plus_array.info().labels +
minus_array.info().labels))))
array.set_observation_type(plus_array.observation_type())
self._arrays.setdefault(key, array)
if len(self._arrays) == 0:
raise CifBuilderError("No reflection data present in cif block")
def run(args):
parser = OptionParser(
#usage=usage,
phil=phil_scope,
read_reflections=True,
read_datablocks=False,
read_experiments=True,
check_format=False,
#epilog=help_message
)
params, options, args = parser.parse_args(show_diff_phil=False,
return_unhandled=True)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
datasets_input = []
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) or len(reflections):
if len(reflections) == 1:
reflections_input = reflections[0]
reflections = []
for i in range(len(experiments)):
reflections.append(
reflections_input.select(reflections_input['id'] == i))
assert len(experiments) == len(reflections)
from cctbx import crystal, miller
for expt, refl in zip(experiments, reflections):
crystal_symmetry = crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group())
# filtering of intensities similar to that done in export_mtz
# FIXME this function should be renamed/moved elsewhere
from dials.util.export_mtz import _apply_data_filters
refl = _apply_data_filters(refl,
ignore_profile_fitting=False,
filter_ice_rings=False,
min_isigi=-5,
include_partials=False,
keep_partials=False,
scale_partials=True)
assert 'intensity.sum.value' in refl
sel = refl.get_flags(refl.flags.integrated_sum)
data = refl['intensity.sum.value']
variances = refl['intensity.sum.variance']
if 'intensity.prf.value' in refl:
prf_sel = refl.get_flags(refl.flags.integrated_prf)
data.set_selected(prf_sel, refl['intensity.prf.value'])
variances.set_selected(prf_sel, refl['intensity.prf.variance'])
sel |= prf_sel
refl = refl.select(sel)
data = data.select(sel)
variances = variances.select(sel)
if 'lp' in refl and 'qe' in refl:
lp = refl['lp']
qe = refl['qe']
assert qe.all_gt(0)
scale = lp / qe
data *= scale
variances *= (flex.pow2(scale))
miller_indices = refl['miller_index']
assert variances.all_gt(0)
sigmas = flex.sqrt(variances)
miller_set = miller.set(crystal_symmetry,
miller_indices,
anomalous_flag=True)
intensities = miller.array(miller_set, data=data, sigmas=sigmas)
intensities.set_observation_type_xray_intensity()
intensities.set_info(
miller.array_info(source='DIALS', source_type='pickle'))
datasets_input.append(intensities)
files = args
for file_name in files:
reader = any_reflection_file(file_name)
assert reader.file_type() == 'ccp4_mtz'
as_miller_arrays = reader.as_miller_arrays(merge_equivalents=False)
intensities_prf = [
ma for ma in as_miller_arrays
if ma.info().labels == ['IPR', 'SIGIPR']
]
intensities_sum = [
ma for ma in as_miller_arrays if ma.info().labels == ['I', 'SIGI']
]
if len(intensities_prf):
intensities = intensities_prf[0]
else:
assert len(intensities_sum), 'No intensities found in input file.'
intensities = intensities_sum[0]
batches = [
ma for ma in as_miller_arrays if ma.info().labels == ['BATCH']
]
if len(batches):
batches = batches[0]
else:
batches = None
mtz_object = reader.file_content()
intensities = intensities.customized_copy(
anomalous_flag=True,
indices=mtz_object.extract_original_index_miller_indices(
)).set_info(intensities.info())
intensities.set_observation_type_xray_intensity()
if params.batch is not None:
assert batches is not None
bmin, bmax = params.batch
assert bmax >= bmin
sel = (batches.data() >= bmin) & (batches.data() <= bmax)
assert sel.count(True) > 0
intensities = intensities.select(sel)
datasets_input.append(intensities)
datasets = datasets_input
assert len(datasets) == 1
result = determine_space_group(
datasets[0],
normalisation=params.normalisation,
d_min=params.d_min,
min_i_mean_over_sigma_mean=params.min_i_mean_over_sigma_mean)
if (len(experiments) and len(reflections)
and params.output.reflections is not None
and params.output.experiments is not None):
from dxtbx.serialize import dump
from rstbx.symmetry.constraints import parameter_reduction
reindexed_experiments = copy.deepcopy(experiments)
reindexed_reflections = copy.deepcopy(reflections[0])
cb_op_inp_best = result.best_solution.subgroup[
'cb_op_inp_best'] * result.cb_op_inp_min
best_subsym = result.best_solution.subgroup['best_subsym']
for expt in reindexed_experiments:
expt.crystal = expt.crystal.change_basis(cb_op_inp_best)
expt.crystal.set_space_group(
best_subsym.space_group().build_derived_acentric_group())
S = parameter_reduction.symmetrize_reduce_enlarge(
expt.crystal.get_space_group())
S.set_orientation(expt.crystal.get_B())
S.symmetrize()
expt.crystal.set_B(S.orientation.reciprocal_matrix())
reindexed_reflections['miller_index'] = cb_op_inp_best.apply(
reindexed_reflections['miller_index'])
logger.info('Saving reindexed experiments to %s' %
params.output.experiments)
dump.experiment_list(reindexed_experiments, params.output.experiments)
logger.info('Saving reindexed reflections to %s' %
params.output.reflections)
reindexed_reflections.as_pickle(params.output.reflections)
elif params.output.suffix is not None:
cb_op_inp_best = result.best_solution.subgroup[
'cb_op_inp_best'] * result.cb_op_inp_min
best_subsym = result.best_solution.subgroup['best_subsym']
space_group = best_subsym.space_group().build_derived_acentric_group()
for file_name in files:
basename = os.path.basename(file_name)
out_name = os.path.splitext(
basename)[0] + params.output.suffix + ".mtz"
reader = any_reflection_file(file_name)
assert reader.file_type() == 'ccp4_mtz'
mtz_object = reader.file_content()
if not cb_op_inp_best.is_identity_op():
mtz_object.change_basis_in_place(cb_op_inp_best)
mtz_object.set_space_group_info(space_group.info())
for crystal in mtz_object.crystals():
crystal.set_unit_cell_parameters(
best_subsym.unit_cell().parameters())
mtz_object.write(out_name)
logger.info('Saving reindexed reflections to %s' % out_name)
def __init__(self, cif_block, base_array_info=None, wavelengths=None):
crystal_symmetry_builder.__init__(self, cif_block)
if base_array_info is not None:
self.crystal_symmetry = self.crystal_symmetry.join_symmetry(
other_symmetry=base_array_info.crystal_symmetry_from_file,
force=True)
self._arrays = OrderedDict()
if (wavelengths is None) :
wavelengths = {}
if base_array_info is None:
base_array_info = miller.array_info(source_type="cif")
refln_containing_loops = self.get_miller_indices_containing_loops()
for self.indices, refln_loop in refln_containing_loops:
self.wavelength_id_array = None
self.crystal_id_array = None
self.scale_group_array = None
wavelength_ids = [None]
crystal_ids = [None]
scale_groups = [None]
for key, value in refln_loop.iteritems():
# need to get these arrays first
if (key.endswith('wavelength_id') or
key.endswith('crystal_id') or
key.endswith('scale_group_code')):
data = as_int_or_none_if_all_question_marks(value, column_name=key)
if data is None:
continue
counts = data.counts()
if key.endswith('wavelength_id'):
wavelength_ids = counts.keys()
if len(counts) == 1: continue
array = miller.array(
miller.set(self.crystal_symmetry, self.indices).auto_anomalous(), data)
if key.endswith('wavelength_id'):
self.wavelength_id_array = array
wavelength_ids = counts.keys()
elif key.endswith('crystal_id'):
self.crystal_id_array = array
crystal_ids = counts.keys()
elif key.endswith('scale_group_code'):
self.scale_group_array = array
scale_groups = counts.keys()
for label, value in sorted(refln_loop.items()):
for w_id in wavelength_ids:
for crys_id in crystal_ids:
for scale_group in scale_groups:
if 'index_' in label: continue
key = label
labels = [label]
wavelength = None
if (key.endswith('wavelength_id') or
key.endswith('crystal_id') or
key.endswith('scale_group_code')):
w_id = None
crys_id = None
scale_group = None
key_suffix = ''
if w_id is not None:
key_suffix += '_%i' %w_id
labels.insert(0, "wavelength_id=%i" %w_id)
wavelength = wavelengths.get(w_id, None)
if crys_id is not None:
key_suffix += '_%i' %crys_id
labels.insert(0, "crystal_id=%i" %crys_id)
if scale_group is not None:
key_suffix += '_%i' %scale_group
labels.insert(0, "scale_group_code=%i" %scale_group)
key += key_suffix
sigmas = None
if key in self._arrays: continue
array = self.flex_std_string_as_miller_array(
value, wavelength_id=w_id, crystal_id=crys_id,
scale_group_code=scale_group)
if array is None: continue
if '_sigma' in key:
sigmas_label = label
key = None
for suffix in ('', '_meas', '_calc'):
if sigmas_label.replace('_sigma', suffix) in refln_loop:
key = sigmas_label.replace('_sigma', suffix) + key_suffix
break
if key is None:
key = sigmas_label + key_suffix
elif key in self._arrays and self._arrays[key].sigmas() is None:
sigmas = array
array = self._arrays[key]
check_array_sizes(array, sigmas, key, sigmas_label)
sigmas = as_flex_double(sigmas, sigmas_label)
array.set_sigmas(sigmas.data())
info = array.info()
array.set_info(
info.customized_copy(labels=info.labels+[sigmas_label],
wavelength=wavelength))
continue
elif 'PHWT' in key:
phwt_label = label
fwt_label = label.replace('PHWT', 'FWT')
if fwt_label not in refln_loop: continue
phwt_array = array
if fwt_label in self._arrays:
array = self._arrays[fwt_label]
check_array_sizes(array, phwt_array, fwt_label, phwt_label)
phases = as_flex_double(phwt_array, phwt_label)
info = array.info()
array = array.phase_transfer(phases, deg=True)
array.set_info(
info.customized_copy(labels=info.labels+[phwt_label]))
self._arrays[fwt_label] = array
continue
elif 'HL_' in key:
hl_letter = key[key.find('HL_')+3]
hl_key = 'HL_' + hl_letter
key = key.replace(hl_key, 'HL_A')
if key in self._arrays:
continue # this array is already dealt with
hl_labels = [label.replace(hl_key, 'HL_'+letter) for letter in 'ABCD']
hl_keys = [key.replace(hl_key, 'HL_'+letter) for letter in 'ABCD']
hl_values = [cif_block.get(hl_key) for hl_key in hl_labels]
if hl_values.count(None) == 0:
selection = self.get_selection(
hl_values[0], wavelength_id=w_id,
crystal_id=crys_id, scale_group_code=scale_group)
hl_values = [as_double_or_none_if_all_question_marks(
hl.select(selection), column_name=lab)
for hl, lab in zip(hl_values, hl_labels)]
array = miller.array(miller.set(
self.crystal_symmetry, self.indices.select(selection)
).auto_anomalous(), flex.hendrickson_lattman(*hl_values))
labels = labels[:-1]+hl_labels
elif '.B_' in key or '_B_' in key:
if '.B_' in key:
key, key_b = key.replace('.B_', '.A_'), key
label, label_b = label.replace('.B_', '.A_'), label
elif '_B_' in key:
key, key_b = key.replace('_B', '_A'), key
label, label_b = label.replace('_B', '_A'), label
if key in refln_loop and key_b in refln_loop:
b_part = array.data()
if key in self._arrays:
info = self._arrays[key].info()
a_part = self._arrays[key].data()
self._arrays[key] = self._arrays[key].array(
data=flex.complex_double(a_part, b_part))
self._arrays[key].set_info(
info.customized_copy(labels=info.labels+[key_b]))
continue
elif ('phase_' in key and not "_meas" in key and
self.crystal_symmetry.space_group() is not None):
alt_key1 = label.replace('phase_', 'F_')
alt_key2 = alt_key1 + '_au'
if alt_key1 in refln_loop:
phase_key = label
key = alt_key1+key_suffix
elif alt_key2 in refln_loop:
phase_key = label
key = alt_key2+key_suffix
else: phase_key = None
if phase_key is not None:
phases = array.data()
if key in self._arrays:
array = self._arrays[key]
array = as_flex_double(array, key)
check_array_sizes(array, phases, key, phase_key)
info = self._arrays[key].info()
self._arrays[key] = array.phase_transfer(phases, deg=True)
self._arrays[key].set_info(
info.customized_copy(labels=info.labels+[phase_key]))
else:
array = self.flex_std_string_as_miller_array(
refln_loop[label], wavelength_id=w_id, crystal_id=crys_id,
scale_group_code=scale_group)
check_array_sizes(array, phases, key, phase_key)
array.phase_transfer(phases, deg=True)
labels = labels+[label, phase_key]
if base_array_info.labels is not None:
labels = base_array_info.labels + labels
def rstrip_substrings(string, substrings):
for substr in substrings:
if substr == '': continue
if string.endswith(substr):
string = string[:-len(substr)]
return string
# determine observation type
stripped_key = rstrip_substrings(
key, [key_suffix, '_au', '_meas', '_calc', '_plus', '_minus'])
if (stripped_key.endswith('F_squared') or
stripped_key.endswith('intensity') or
stripped_key.endswith('.I') or
stripped_key.endswith('_I')) and (
array.is_real_array() or array.is_integer_array()):
array.set_observation_type_xray_intensity()
elif (stripped_key.endswith('F') and (
array.is_real_array() or array.is_integer_array())):
array.set_observation_type_xray_amplitude()
if (array.is_xray_amplitude_array() or
array.is_xray_amplitude_array()):
# e.g. merge_equivalents treats integer arrays differently, so must
# convert integer observation arrays here to be safe
if isinstance(array.data(), flex.int):
array = array.customized_copy(data=array.data().as_double())
array.set_info(base_array_info.customized_copy(labels=labels))
if (array.is_xray_amplitude_array() or
array.is_xray_amplitude_array()):
info = array.info()
array.set_info(info.customized_copy(wavelength=wavelength))
self._arrays.setdefault(key, array)
for key, array in self._arrays.copy().iteritems():
if ( key.endswith('_minus') or '_minus_' in key
or key.endswith('_plus') or '_plus_' in key):
if '_minus' in key:
minus_key = key
plus_key = key.replace('_minus', '_plus')
elif '_plus' in key:
plus_key = key
minus_key = key.replace('_plus', '_minus')
if plus_key in self._arrays and minus_key in self._arrays:
plus_array = self._arrays.pop(plus_key)
minus_array = self._arrays.pop(minus_key)
minus_array = minus_array.customized_copy(
indices=-minus_array.indices()).set_info(minus_array.info())
array = plus_array.concatenate(
minus_array, assert_is_similar_symmetry=False)
array = array.customized_copy(anomalous_flag=True)
array.set_info(minus_array.info().customized_copy(
labels=list(
OrderedSet(plus_array.info().labels+minus_array.info().labels))))
array.set_observation_type(plus_array.observation_type())
self._arrays.setdefault(key, array)
if len(self._arrays) == 0:
raise CifBuilderError("No reflection data present in cif block")
def __init__(self, cif_block, base_array_info=None, wavelengths=None):
crystal_symmetry_builder.__init__(self, cif_block)
self._arrays = OrderedDict()
self._origarrays = OrderedDict(
) # used for presenting raw data tables in HKLviewer
basearraylabels = []
if base_array_info is not None:
self.crystal_symmetry = self.crystal_symmetry.join_symmetry(
other_symmetry=base_array_info.crystal_symmetry_from_file,
force=True)
if base_array_info.labels:
basearraylabels = base_array_info.labels
if (wavelengths is None):
wavelengths = {}
if base_array_info is None:
base_array_info = miller.array_info(source_type="cif")
refln_containing_loops = self.get_miller_indices_containing_loops()
for self.indices, refln_loop in refln_containing_loops:
self.wavelength_id_array = None
self.crystal_id_array = None
self.scale_group_array = None
wavelength_ids = [None]
crystal_ids = [None]
scale_groups = [None]
for key, value in six.iteritems(refln_loop):
# Get wavelength_ids, crystal_id, scale_group_code columns for selecting data of other
# columns in self.get_selection() used by self.flex_std_string_as_miller_array()
if (key.endswith('wavelength_id') or key.endswith('crystal_id')
or key.endswith('scale_group_code')):
data = as_int_or_none_if_all_question_marks(
value, column_name=key)
if data is None:
continue
counts = data.counts()
if key.endswith('wavelength_id'):
wavelength_ids = list(counts.keys())
if len(counts) == 1: continue
array = miller.array(
miller.set(self.crystal_symmetry,
self.indices).auto_anomalous(), data)
if key.endswith('wavelength_id'):
self.wavelength_id_array = array
wavelength_ids = list(counts.keys())
elif key.endswith('crystal_id'):
self.crystal_id_array = array
crystal_ids = list(counts.keys())
elif key.endswith('scale_group_code'):
self.scale_group_array = array
scale_groups = list(counts.keys())
labelsuffix = []
wavelbl = []
cryslbl = []
scalegrplbl = []
self._origarrays["HKLs"] = self.indices
alllabels = list(sorted(refln_loop.keys()))
remaininglabls = alllabels[:] # deep copy the list
# Parse labels matching cif column conventions
# https://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/refln.html
# and extract groups of labels or just single columns.
# Groups corresponds to the map coefficients, phase and amplitudes,
# amplitudes or intensities with sigmas and hendrickson-lattman columns.
phaseamplabls, remaininglabls = self.get_phase_amplitude_labels(
remaininglabls)
mapcoefflabls, remaininglabls = self.get_mapcoefficient_labels(
remaininglabls)
HLcoefflabls, remaininglabls = self.get_HL_labels(remaininglabls)
data_sig_obstype_labls, remaininglabls = self.get_FSigF_ISigI_labels(
remaininglabls)
for w_id in wavelength_ids:
for crys_id in crystal_ids:
for scale_group in scale_groups:
# If reflection data files contain more than one crystal, wavelength or scalegroup
# then add their id(s) as a suffix to data labels computed below. Needed for avoiding
# ambuguity but avoid when not needed to make labels more human readable!
if (len(wavelength_ids) > 1
or len(wavelengths) > 1) and w_id is not None:
wavelbl = ["wavelength_id=%i" % w_id]
if len(crystal_ids) > 1 and crys_id is not None:
cryslbl = ["crystal_id=%i" % crys_id]
if len(scale_groups) > 1 and scale_group is not None:
scalegrplbl = ["scale_group_code=%i" % scale_group]
labelsuffix = scalegrplbl + cryslbl + wavelbl
jlablsufx = ""
if len(labelsuffix):
jlablsufx = "," + ",".join(labelsuffix)
for mapcoefflabl in mapcoefflabls:
A_array = refln_loop[mapcoefflabl[0]]
B_array = refln_loop[mapcoefflabl[1]]
# deselect any ? marks in the two arrays, assuming both A and B have the same ? marks
selection = self.get_selection(
A_array,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
A_array = A_array.select(selection)
B_array = B_array.select(selection)
# form the miller array with map coefficients
data = flex.complex_double(flex.double(A_array),
flex.double(B_array))
millarr = miller.array(
miller.set(self.crystal_symmetry,
self.indices.select(
selection)).auto_anomalous(),
data)
# millarr will be None for column data not matching w_id,crys_id,scale_group values
if millarr is None: continue
labl = basearraylabels + mapcoefflabl + labelsuffix
millarr.set_info(
base_array_info.customized_copy(
labels=labl,
wavelength=wavelengths.get(w_id, None)))
self._arrays[mapcoefflabl[0] + jlablsufx] = millarr
for phaseamplabl in phaseamplabls:
amplitudestrarray = refln_loop[phaseamplabl[0]]
phasestrarray = refln_loop[phaseamplabl[1]]
millarr = self.flex_std_string_as_miller_array(
amplitudestrarray,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
phasesmillarr = self.flex_std_string_as_miller_array(
phasestrarray,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
# millarr will be None for column data not matching w_id,crys_id,scale_group values
if millarr is None or phasesmillarr is None:
continue
phases = as_flex_double(phasesmillarr,
phaseamplabl[1])
millarr = millarr.phase_transfer(phases, deg=True)
labl = basearraylabels + phaseamplabl + labelsuffix
millarr.set_info(
base_array_info.customized_copy(
labels=labl,
wavelength=wavelengths.get(w_id, None)))
self._arrays[phaseamplabl[0] + jlablsufx] = millarr
for datlabl, siglabl, otype in data_sig_obstype_labls:
datastrarray = refln_loop[datlabl]
millarr = self.flex_std_string_as_miller_array(
datastrarray,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
# millarr will be None for column data not matching w_id,crys_id,scale_group values
if millarr is None: continue
millarr = as_flex_double(millarr, datlabl)
datsiglabl = [datlabl]
if siglabl:
sigmasstrarray = refln_loop[siglabl]
sigmas = self.flex_std_string_as_miller_array(
sigmasstrarray,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
sigmas = as_flex_double(sigmas, siglabl)
millarr.set_sigmas(sigmas.data())
datsiglabl = [datlabl, siglabl]
datsiglabl = basearraylabels + datsiglabl + labelsuffix
millarr.set_info(
base_array_info.customized_copy(
labels=datsiglabl,
wavelength=wavelengths.get(w_id, None)))
if otype is not None:
millarr.set_observation_type(otype)
self._arrays[datlabl + jlablsufx] = millarr
for hl_labels in HLcoefflabls:
hl_values = [
cif_block.get(hl_key) for hl_key in hl_labels
]
if hl_values.count(None) == 0:
selection = self.get_selection(
hl_values[0],
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
hl_values = [
as_double_or_none_if_all_question_marks(
hl.select(selection), column_name=lab)
for hl, lab in zip(hl_values, hl_labels)
]
# hl_values will be None for column data not matching w_id,crys_id,scale_group values
if hl_values == [None, None, None, None]:
continue
millarr = miller.array(
miller.set(
self.crystal_symmetry,
self.indices.select(
selection)).auto_anomalous(),
flex.hendrickson_lattman(*hl_values))
hlabels = basearraylabels + hl_labels + labelsuffix
millarr.set_info(
base_array_info.customized_copy(
labels=hlabels,
wavelength=wavelengths.get(w_id,
None)))
self._arrays[hl_labels[0] +
jlablsufx] = millarr
# pick up remaining columns if any that weren't identified above
for label in alllabels:
if "index_" in label:
continue
datastrarray = refln_loop[label]
if label in remaininglabls:
labels = basearraylabels + [label
] + labelsuffix
lablsufx = jlablsufx
millarr = self.flex_std_string_as_miller_array(
datastrarray,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
# millarr will be None for column data not matching w_id,crys_id,scale_group values
if (label.endswith(
'wavelength_id'
) or label.endswith(
'crystal_id'
) or # get full array if any of these labels, not just subsets
label.endswith('scale_group_code')):
millarr = self.flex_std_string_as_miller_array(
datastrarray,
wavelength_id=None,
crystal_id=None,
scale_group_code=None)
lablsufx = ""
labels = basearraylabels + [label]
if millarr is None: continue
otype = self.guess_observationtype(label)
if otype is not None:
millarr.set_observation_type(otype)
millarr.set_info(
base_array_info.customized_copy(
labels=labels,
wavelength=wavelengths.get(w_id,
None)))
self._arrays[label + lablsufx] = millarr
origarr = self.flex_std_string_as_miller_array(
datastrarray,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
newlabel = label.replace("_refln.", "")
newlabel2 = newlabel.replace("_refln_", "")
if origarr: # want only genuine miller arrays
self._origarrays[newlabel2 +
jlablsufx] = origarr.data()
# Convert any groups of I+,I-,SigI+,SigI- (or amplitudes) arrays into anomalous arrays
# i.e. both friedel mates in the same array
for key, array in six.iteritems(self._arrays.copy()):
plus_key = ""
if '_minus' in key:
minus_key = key
plus_key = key.replace('_minus', '_plus')
elif '-' in key:
minus_key = key
plus_key = key.replace('-', '+')
elif '_plus' in key:
plus_key = key
minus_key = key.replace('_plus', '_minus')
elif '+' in key:
plus_key = key
minus_key = key.replace('+', '-')
if plus_key in self._arrays and minus_key in self._arrays:
plus_array = self._arrays.pop(plus_key)
minus_array = self._arrays.pop(minus_key)
minus_array = minus_array.customized_copy(
indices=-minus_array.indices()).set_info(
minus_array.info())
array = plus_array.concatenate(
minus_array, assert_is_similar_symmetry=False)
array = array.customized_copy(anomalous_flag=True)
array.set_info(minus_array.info().customized_copy(labels=list(
OrderedSet(plus_array.info().labels +
minus_array.info().labels))))
array.set_observation_type(plus_array.observation_type())
self._arrays.setdefault(key, array)
if len(self._arrays) == 0:
raise CifBuilderError("No reflection data present in cif block")
# Sort the ordered dictionary to resemble the order of columns in the cif file
# This is to avoid any F_meas arrays accidentally being put adjacent to
# pdbx_anom_difference arrays in the self._arrays OrderedDict. Otherwise these
# arrays may unintentionally be combined into a reconstructed anomalous amplitude
# array when saving as an mtz file due to a problem in the iotbx/mtz module.
# See http://phenix-online.org/pipermail/cctbxbb/2021-March/002289.html
arrlstord = []
arrlst = list(self._arrays)
for arr in arrlst:
for i, k in enumerate(refln_loop.keys()):
if arr.split(",")[0] == k:
arrlstord.append((arr, i))
# arrlstord must have the same keys as in the self._arrays dictionary
assert sorted(arrlst) == sorted([e[0] for e in arrlstord])
sortarrlst = sorted(arrlstord, key=lambda arrord: arrord[1])
self._ordarrays = OrderedDict()
for sortkey, i in sortarrlst:
self._ordarrays.setdefault(sortkey, self._arrays[sortkey])
self._arrays = self._ordarrays
def get_solve_map(self):
map_coeffs = self._convert_amplitudes_and_phases(f_label="FP",
phi_label="PHIB", fom_label="FOM", weighted=True)
from cctbx.miller import array_info
info = array_info(labels=["FWT","PHWT"])
return map_coeffs.set_info(info)
def list_6_as_miller_arrays(file_name):
""" Read the file of given name and return a pair of miller arrays
(F_obs^2, F_cal) """
# potentially iotbx.cif could be used here
fcf = iter(open(file_name))
space_group = sgtbx.space_group()
unit_cell_params = {}
indices = flex.miller_index()
f_obs_squares = flex.double()
sigma_f_obs_squares = flex.double()
f_calc_amplitudes = flex.double()
f_calc_phases = flex.double()
for li in fcf:
if li.startswith('loop_'):
for li in fcf:
li = li.strip()
if li == '_symmetry_equiv_pos_as_xyz':
for li in fcf:
li = li.strip()
if not li: break
space_group.expand_smx(li[1:-1])
else:
for i in xrange(6): fcf.next()
for li in fcf:
items = li.split()
if not items: break
h,k,l, fo, sig_fo, fc, phase = items
indices.append((int(h), int(k), int(l)))
f_obs_squares.append(float(fo))
sigma_f_obs_squares.append(float(sig_fo))
f_calc_amplitudes.append(float(fc))
f_calc_phases.append(float(phase))
if not li: break
elif li.startswith('_cell'):
lbl, value = li.split()
unit_cell_params[lbl] = float(value)
unit_cell = uctbx.unit_cell(
[ unit_cell_params[p]
for p in ( "_cell_length_a","_cell_length_b","_cell_length_c",
"_cell_angle_alpha","_cell_angle_beta","_cell_angle_gamma" )
])
crystal_symmetry = crystal.symmetry(
unit_cell=unit_cell,
space_group=space_group)
f_calc_phases *= pi/180
f_calc = flex.complex_double(
reals=f_calc_amplitudes * flex.cos(f_calc_phases),
imags=f_calc_amplitudes * flex.sin(f_calc_phases) )
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=indices).auto_anomalous()
f_obs_squares = miller.array(
miller_set=miller_set,
data=f_obs_squares,
sigmas=sigma_f_obs_squares)
f_obs_squares.set_observation_type_xray_intensity()
f_obs_squares.set_info(miller.array_info(
source=file_name,
labels=["F_squared_meas", "F_squared_sigma"]))
f_calc = miller.array(
miller_set=miller_set,
data=f_calc)
f_obs_squares.set_info(miller.array_info(
source=file_name,
labels=["F_calc"]))
return f_obs_squares, f_calc
def extract(file_name,
crystal_symmetry,
wavelength_id,
crystal_id,
show_details_if_error,
output_r_free_label,
merge_non_unique_under_symmetry,
map_to_asu,
remove_systematic_absences,
incompatible_flags_to_work_set=False,
ignore_bad_sigmas=False,
extend_flags=False,
return_as_miller_arrays=False,
log=sys.stdout):
import iotbx.cif
from cctbx import miller
base_array_info = miller.array_info(
crystal_symmetry_from_file=crystal_symmetry)
all_miller_arrays = iotbx.cif.reader(
file_path=file_name).build_miller_arrays(
base_array_info=base_array_info)
if (len(all_miller_arrays) == 0):
raise Sorry(
"No data arrays were found in this CIF file. Please make " +
"sure that the file contains reflection data, rather than the refined "
+ "model.")
column_labels = set()
if (extend_flags):
map_to_asu = True
def get_label(miller_array):
label = None
for l in miller_array.info().labels:
if ('_meas' in l):
if miller_array.is_xray_amplitude_array():
label = "FOBS"
elif miller_array.is_xray_intensity_array():
label = "IOBS"
elif l.endswith(".phase_meas"):
label = "PHIM"
break
elif ("_calc" in l):
if miller_array.is_xray_amplitude_array():
label = "FC"
elif miller_array.is_xray_intensity_array():
label = "ICALC"
elif l.endswith(".F_calc"):
label = "FC"
elif l.endswith(".phase_calc"):
label = "PHIC"
break
elif miller_array.anomalous_flag():
if miller_array.is_xray_amplitude_array():
label = "F"
elif miller_array.is_xray_intensity_array():
label = "I"
break
elif 'status' in l or '_free' in l:
label = output_r_free_label
break
elif miller_array.is_hendrickson_lattman_array():
label = "HL"
elif (miller_array.is_complex_array()):
if (l.endswith("DELFWT")):
label = "DELFWT"
break
elif (l.endswith("FWT")):
label = "FWT"
break
elif (miller_array.is_real_array()):
if ("pdbx_anom_difference" in l):
label = "DANO"
break
elif (l.endswith(".fom")):
label = "FOM"
break
return label
for (data_name, miller_arrays) in all_miller_arrays.iteritems():
for ma in miller_arrays.values():
other_symmetry = crystal_symmetry
try:
crystal_symmetry = other_symmetry.join_symmetry(
other_symmetry=ma.crystal_symmetry(), force=True)
except AssertionError, e:
str_e = str(e)
from cStringIO import StringIO
s = StringIO()
if "Space group is incompatible with unit cell parameters." in str_e:
other_symmetry.show_summary(f=s)
ma.crystal_symmetry().show_summary(f=s)
str_e += "\n%s" % (s.getvalue())
raise Sorry(str_e)
else:
raise
def run(args):
import libtbx
from libtbx import easy_pickle
from dials.util import log
from dials.util.options import OptionParser
parser = OptionParser(
#usage=usage,
phil=phil_scope,
read_reflections=True,
read_datablocks=False,
read_experiments=True,
check_format=False,
#epilog=help_message
)
params, options, args = parser.parse_args(show_diff_phil=False,
return_unhandled=True)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
if params.seed is not None:
import random
flex.set_random_seed(params.seed)
random.seed(params.seed)
if params.save_plot and not params.animate:
import matplotlib
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use('Agg') # use a non-interactive backend
datasets_input = []
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) or len(reflections):
if len(reflections) == 1:
reflections_input = reflections[0]
reflections = []
for i in range(len(experiments)):
reflections.append(
reflections_input.select(reflections_input['id'] == i))
if len(experiments) > len(reflections):
flattened_reflections = []
for refl in reflections:
for i in range(0, flex.max(refl['id']) + 1):
sel = refl['id'] == i
flattened_reflections.append(refl.select(sel))
reflections = flattened_reflections
assert len(experiments) == len(reflections)
i_refl = 0
for i_expt in enumerate(experiments):
refl = reflections[i_refl]
for expt, refl in zip(experiments, reflections):
crystal_symmetry = crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group())
if 0 and 'intensity.prf.value' in refl:
sel = refl.get_flags(refl.flags.integrated_prf)
assert sel.count(True) > 0
refl = refl.select(sel)
data = refl['intensity.prf.value']
variances = refl['intensity.prf.variance']
else:
assert 'intensity.sum.value' in refl
sel = refl.get_flags(refl.flags.integrated_sum)
assert sel.count(True) > 0
refl = refl.select(sel)
data = refl['intensity.sum.value']
variances = refl['intensity.sum.variance']
# FIXME probably need to do some filtering of intensities similar to that
# done in export_mtz
miller_indices = refl['miller_index']
assert variances.all_gt(0)
sigmas = flex.sqrt(variances)
miller_set = miller.set(crystal_symmetry,
miller_indices,
anomalous_flag=False)
intensities = miller.array(miller_set, data=data, sigmas=sigmas)
intensities.set_observation_type_xray_intensity()
intensities.set_info(
miller.array_info(source='DIALS', source_type='pickle'))
datasets_input.append(intensities)
files = args
for file_name in files:
try:
data = easy_pickle.load(file_name)
intensities = data['observations'][0]
intensities.set_info(
miller.array_info(source=file_name, source_type='pickle'))
intensities = intensities.customized_copy(
anomalous_flag=False).set_info(intensities.info())
batches = None
except Exception:
reader = any_reflection_file(file_name)
assert reader.file_type() == 'ccp4_mtz'
as_miller_arrays = reader.as_miller_arrays(merge_equivalents=False)
intensities = [
ma for ma in as_miller_arrays
if ma.info().labels == ['I', 'SIGI']
][0]
batches = [
ma for ma in as_miller_arrays if ma.info().labels == ['BATCH']
]
if len(batches):
batches = batches[0]
else:
batches = None
mtz_object = reader.file_content()
intensities = intensities.customized_copy(
anomalous_flag=False,
indices=mtz_object.extract_original_index_miller_indices(
)).set_info(intensities.info())
intensities.set_observation_type_xray_intensity()
datasets_input.append(intensities)
if len(datasets_input) == 0:
raise Sorry('No valid reflection files provided on command line')
datasets = []
for intensities in datasets_input:
if params.batch is not None:
assert batches is not None
bmin, bmax = params.batch
assert bmax >= bmin
sel = (batches.data() >= bmin) & (batches.data() <= bmax)
assert sel.count(True) > 0
intensities = intensities.select(sel)
if params.min_i_mean_over_sigma_mean is not None and (
params.d_min is libtbx.Auto or params.d_min is not None):
from xia2.Modules import Resolutionizer
rparams = Resolutionizer.phil_defaults.extract().resolutionizer
rparams.nbins = 20
resolutionizer = Resolutionizer.resolutionizer(
intensities, None, rparams)
i_mean_over_sigma_mean = 4
d_min = resolutionizer.resolution_i_mean_over_sigma_mean(
i_mean_over_sigma_mean)
if params.d_min is libtbx.Auto:
intensities = intensities.resolution_filter(
d_min=d_min).set_info(intensities.info())
if params.verbose:
logger.info('Selecting reflections with d > %.2f' % d_min)
elif d_min > params.d_min:
logger.info('Rejecting dataset %s as d_min too low (%.2f)' %
(file_name, d_min))
continue
else:
logger.info('Estimated d_min for %s: %.2f' %
(file_name, d_min))
elif params.d_min not in (None, libtbx.Auto):
intensities = intensities.resolution_filter(
d_min=params.d_min).set_info(intensities.info())
if params.normalisation == 'kernel':
from mmtbx.scaling import absolute_scaling
normalisation = absolute_scaling.kernel_normalisation(
intensities, auto_kernel=True)
intensities = normalisation.normalised_miller.deep_copy()
cb_op_to_primitive = intensities.change_of_basis_op_to_primitive_setting(
)
intensities = intensities.change_basis(cb_op_to_primitive)
if params.mode == 'full' or params.space_group is not None:
if params.space_group is not None:
space_group_info = params.space_group.primitive_setting()
if not space_group_info.group().is_compatible_unit_cell(
intensities.unit_cell()):
logger.info(
'Skipping data set - incompatible space group and unit cell: %s, %s'
% (space_group_info, intensities.unit_cell()))
continue
else:
space_group_info = sgtbx.space_group_info('P1')
intensities = intensities.customized_copy(
space_group_info=space_group_info)
datasets.append(intensities)
crystal_symmetries = [d.crystal_symmetry().niggli_cell() for d in datasets]
lattice_ids = range(len(datasets))
from xfel.clustering.cluster import Cluster
from xfel.clustering.cluster_groups import unit_cell_info
ucs = Cluster.from_crystal_symmetries(crystal_symmetries,
lattice_ids=lattice_ids)
threshold = 1000
if params.save_plot:
from matplotlib import pyplot as plt
fig = plt.figure("Andrews-Bernstein distance dendogram",
figsize=(12, 8))
ax = plt.gca()
else:
ax = None
clusters, _ = ucs.ab_cluster(params.unit_cell_clustering.threshold,
log=params.unit_cell_clustering.log,
write_file_lists=False,
schnell=False,
doplot=params.save_plot,
ax=ax)
if params.save_plot:
plt.tight_layout()
plt.savefig('%scluster_unit_cell.png' % params.plot_prefix)
plt.close(fig)
logger.info(unit_cell_info(clusters))
largest_cluster = None
largest_cluster_lattice_ids = None
for cluster in clusters:
cluster_lattice_ids = [m.lattice_id for m in cluster.members]
if largest_cluster_lattice_ids is None:
largest_cluster_lattice_ids = cluster_lattice_ids
elif len(cluster_lattice_ids) > len(largest_cluster_lattice_ids):
largest_cluster_lattice_ids = cluster_lattice_ids
dataset_selection = largest_cluster_lattice_ids
if len(dataset_selection) < len(datasets):
logger.info('Selecting subset of data for cosym analysis: %s' %
str(dataset_selection))
datasets = [datasets[i] for i in dataset_selection]
# per-dataset change of basis operator to ensure all consistent
change_of_basis_ops = []
for i, dataset in enumerate(datasets):
metric_subgroups = sgtbx.lattice_symmetry.metric_subgroups(dataset,
max_delta=5)
subgroup = metric_subgroups.result_groups[0]
cb_op_inp_best = subgroup['cb_op_inp_best']
datasets[i] = dataset.change_basis(cb_op_inp_best)
change_of_basis_ops.append(cb_op_inp_best)
cb_op_ref_min = datasets[0].change_of_basis_op_to_niggli_cell()
for i, dataset in enumerate(datasets):
if params.space_group is None:
datasets[i] = dataset.change_basis(cb_op_ref_min).customized_copy(
space_group_info=sgtbx.space_group_info('P1'))
else:
datasets[i] = dataset.change_basis(cb_op_ref_min)
datasets[i] = datasets[i].customized_copy(
crystal_symmetry=crystal.symmetry(
unit_cell=datasets[i].unit_cell(),
space_group_info=params.space_group.primitive_setting(),
assert_is_compatible_unit_cell=False))
datasets[i] = datasets[i].merge_equivalents().array()
change_of_basis_ops[i] = cb_op_ref_min * change_of_basis_ops[i]
result = analyse_datasets(datasets, params)
space_groups = {}
reindexing_ops = {}
for dataset_id in result.reindexing_ops.iterkeys():
if 0 in result.reindexing_ops[dataset_id]:
cb_op = result.reindexing_ops[dataset_id][0]
reindexing_ops.setdefault(cb_op, [])
reindexing_ops[cb_op].append(dataset_id)
if dataset_id in result.space_groups:
space_groups.setdefault(result.space_groups[dataset_id], [])
space_groups[result.space_groups[dataset_id]].append(dataset_id)
logger.info('Space groups:')
for sg, datasets in space_groups.iteritems():
logger.info(str(sg.info().reference_setting()))
logger.info(datasets)
logger.info('Reindexing operators:')
for cb_op, datasets in reindexing_ops.iteritems():
logger.info(cb_op)
logger.info(datasets)
if (len(experiments) and len(reflections)
and params.output.reflections is not None
and params.output.experiments is not None):
import copy
from dxtbx.model import ExperimentList
from dxtbx.serialize import dump
reindexed_experiments = ExperimentList()
reindexed_reflections = flex.reflection_table()
expt_id = 0
for cb_op, dataset_ids in reindexing_ops.iteritems():
cb_op = sgtbx.change_of_basis_op(cb_op)
for dataset_id in dataset_ids:
expt = experiments[dataset_selection[dataset_id]]
refl = reflections[dataset_selection[dataset_id]]
reindexed_expt = copy.deepcopy(expt)
refl_reindexed = copy.deepcopy(refl)
cb_op_this = cb_op * change_of_basis_ops[dataset_id]
reindexed_expt.crystal = reindexed_expt.crystal.change_basis(
cb_op_this)
refl_reindexed['miller_index'] = cb_op_this.apply(
refl_reindexed['miller_index'])
reindexed_experiments.append(reindexed_expt)
refl_reindexed['id'] = flex.int(refl_reindexed.size(), expt_id)
reindexed_reflections.extend(refl_reindexed)
expt_id += 1
logger.info('Saving reindexed experiments to %s' %
params.output.experiments)
dump.experiment_list(reindexed_experiments, params.output.experiments)
logger.info('Saving reindexed reflections to %s' %
params.output.reflections)
reindexed_reflections.as_pickle(params.output.reflections)
elif params.output.suffix is not None:
for cb_op, dataset_ids in reindexing_ops.iteritems():
cb_op = sgtbx.change_of_basis_op(cb_op)
for dataset_id in dataset_ids:
file_name = files[dataset_selection[dataset_id]]
basename = os.path.basename(file_name)
out_name = os.path.splitext(
basename)[0] + params.output.suffix + '_' + str(
dataset_selection[dataset_id]) + ".mtz"
reader = any_reflection_file(file_name)
assert reader.file_type() == 'ccp4_mtz'
mtz_object = reader.file_content()
cb_op_this = cb_op * change_of_basis_ops[dataset_id]
if not cb_op_this.is_identity_op():
logger.info('reindexing %s (%s)' %
(file_name, cb_op_this.as_xyz()))
mtz_object.change_basis_in_place(cb_op_this)
mtz_object.write(out_name)
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None,
assume_shelx_observation_type_is=None,
enforce_positive_sigmas=False,
):
"""
Convert the contents of the reflection file into a list of
:py:class:`cctbx.miller.array` objects, each of which may contain multiple
columns of data from the underlying file. By default this will
automatically merge redundant observations to obtain a unique set under
symmetry.
:param crystal_symmetry: :py:class:`cctbx.crystal.symmetry` object
(defaults to using internally specified symmetry, if any)
:param force_symmetry: TODO
:param merge_equivalents: merge redundant obervations (default=True)
:param base_array_info: :py:class:`cctbx.miller.array_info` object
containing basic information to be propagated to the arrays
:param assume_shelx_observation_type_is: if specified, instead of raising
an exception if the SHELX file type is not known from the file name plus
data type tag, the function will force the specified data type.
"""
assert (assume_shelx_observation_type_is in
[None, "amplitudes", "intensities"])
if (self._file_type is None):
return []
info_source = self._file_name
if (info_source.startswith("./") or info_source.startswith(".\\")):
info_source = info_source[2:]
if (base_array_info is None):
base_array_info = miller.array_info(
source=info_source,
source_type=self._file_type)
if (self._file_type == "cctbx.miller.array"):
result = []
for miller_array in self._file_content:
info = miller_array.info()
if (info is None or not isinstance(info, miller.array_info)):
info = base_array_info
info.source = info_source
info.crystal_symmetry_from_file = crystal.symmetry(
unit_cell=miller_array.unit_cell(),
space_group_info=miller_array.space_group_info(),
raise_sorry_if_incompatible_unit_cell=True)
result.append(miller_array.customized_copy(
crystal_symmetry=miller_array.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry,
raise_sorry_if_incompatible_unit_cell=True))
.set_info(info)
.set_observation_type(miller_array.observation_type()))
return result
if (( crystal_symmetry is None
or crystal_symmetry.unit_cell() is None)
and self._observation_type == 'hklf+ins/res'
):
name, ext = os.path.splitext(self._file_name)
if ext != '.hkl': # it may be compressed: name.hkl.gz
name, ext = os.path.splitext(name)
for shelx_file_name in ('%s.ins' % name, '%s.res' % name):
try:
shelx_file = open(shelx_file_name)
break
except IOError:
continue
else:
raise Sorry("Can't open files %s.ins or %s.res"
"required by the option hklf+ins/res" % ((name,)*2))
crystal_symmetry = crystal_symmetry_from_ins.extract_from(
file=shelx_file)
shelx_file.seek(0)
remaining = shelx_file.read()
shelx_file.close()
m = re.search("^HKLF\s*(\d)", remaining, re.X|re.M|re.S)
if m is None:
raise Sorry("%s does not contain the mandatory HKLF instruction"
% shelx_file.name)
if m.group(1) == "4":
self._observation_type = "intensities"
elif m.group(1) == "3":
self._observation_type = "amplitudes"
else:
raise Sorry("HKLF %s not supported" % m.group(1))
result = self._file_content.as_miller_arrays(
crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry,
merge_equivalents=merge_equivalents,
base_array_info=base_array_info,
)
if (self.file_type() == "shelx_hklf"):
if ((self._observation_type == "intensities") or
(assume_shelx_observation_type_is == "intensities")) :
result[0].set_info(result[0].info().customized_copy(
labels=["Iobs", "SigIobs"]))
result[0].set_observation_type_xray_intensity()
elif ((self._observation_type == "amplitudes") or
(assume_shelx_observation_type_is == "amplitudes")) :
result[0].set_info(result[0].info().customized_copy(
labels=["Fobs", "SigFobs"]))
result[0].set_observation_type_xray_amplitude()
else:
raise Sorry("Unresolved amplitude/intensity ambiguity: %s\n"
" SHELX reflection files may contain amplitudes or intensities.\n"
" Please append =amplitudes\n"
" or =hklf3\n"
" or =intensities\n"
" or =hklf4\n"
" to the file name argument or parameter to resolve the"
" ambiguity.\n"
" If a corresponding .ins file is available, look for the"
" HKLF codeword.\n"
" Alternatively, run the phenix.reflection_statistics"
" command twice,\n"
" once with =amplitudes and once with =intensities. Inspect"
" the <I^2>/(<I>)^2\n"
" statistics. For acentric structures the values should"
" fluctuate around\n"
" 2.0, for centric structures around 3.0. If the statistics"
" are not conclusive\n"
" it will be best to recover the original reflection data, such"
" as SCALEPACK,\n"
" SCALA MTZ, XDS, or d*TREK files." % self._file_name)
# discard reflections where sigma <= 0
# XXX note that this will happen after data merging, so for unmerged data
# it is better to specify merge_equivalents=False!
if (enforce_positive_sigmas) :
result_ = []
for array in result :
result_.append(array.enforce_positive_sigmas())
result = result_
return result
def filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
return_batches=False,
):
"""Create a list of filtered arrays from experiments and reflections.
A partiality threshold can be set, and if outlier_rejection_after_filter
is True, and intensity.scale values are not present, then a round of
outlier rejection will take place.
Raises:
ValueError: if no datasets remain after filtering.
"""
miller_arrays = []
ids_to_del = []
if return_batches:
assert all(expt.scan is not None for expt in experiments)
batch_offsets = [expt.scan.get_batch_offset() for expt in experiments]
reflections = assign_batches_to_reflections(reflections, batch_offsets)
batch_arrays = []
for idx, (expt, refl) in enumerate(zip(experiments, reflections)):
crystal_symmetry = crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group(),
)
# want to use scale intensities if present, else sum + prf (if available)
if "intensity.scale.value" in refl:
intensity_choice = ["scale"]
intensity_to_use = "intensity.scale"
else:
assert "intensity.sum.value" in refl
intensity_to_use = "intensity.sum"
intensity_choice = ["sum"]
if "intensity.prf.value" in refl:
intensity_choice.append("profile")
intensity_to_use = "intensity.prf"
try:
logger.info("Filtering reflections for dataset %s" % idx)
refl = filter_reflection_table(
refl,
intensity_choice,
min_isigi=-5,
filter_ice_rings=False,
combine_partials=True,
partiality_threshold=partiality_threshold,
)
except ValueError:
logger.info(
"Dataset %s removed as no reflections left after filtering",
idx)
ids_to_del.append(idx)
else:
# If scale was chosen - will return scale or have raised ValueError
# If prf or sum, possible was no prf but want to continue.
try:
refl["intensity"] = refl[intensity_to_use + ".value"]
refl["variance"] = refl[intensity_to_use + ".variance"]
except KeyError: # catch case where prf were removed.
refl["intensity"] = refl["intensity.sum.value"]
refl["variance"] = refl["intensity.sum.variance"]
if outlier_rejection_after_filter and intensity_to_use != "intensity.scale":
refl = reject_outliers(refl, expt, method="simple", zmax=12.0)
refl = refl.select(
~refl.get_flags(refl.flags.outlier_in_scaling))
miller_set = miller.set(crystal_symmetry,
refl["miller_index"],
anomalous_flag=False)
intensities = miller_set.array(data=refl["intensity"],
sigmas=flex.sqrt(refl["variance"]))
intensities.set_observation_type_xray_intensity()
intensities.set_info(
miller.array_info(source="DIALS", source_type="pickle"))
miller_arrays.append(intensities)
if return_batches:
batch_arrays.append(
miller_set.array(data=refl["batch"]).set_info(
intensities.info()))
if not miller_arrays:
raise ValueError(
"""No datasets remain after pre-filtering. Please check input data.
The datasets may not contain any full reflections; the command line
option partiality_threshold can be lowered to include partials.""")
for id_ in ids_to_del[::-1]:
del experiments[id_]
del reflections[id_]
if return_batches:
return miller_arrays, batch_arrays
return miller_arrays
class intensities_scaler(object):
"""
Author : Uervirojnangkoorn, M.
Created : 7/13/2014
Merge equivalent reflections and report intensity and refinement statistics.
"""
def __init__(self):
"""
Constructor
"""
self.CONST_SE_MIN_WEIGHT = 0.17
self.CONST_SE_MAX_WEIGHT = 1.0
self.CONST_SIG_I_FACTOR = 1.5
def write_stat_pickle(self, iparams, stat_dict):
fname = iparams.run_no + '/pickle.stat'
if os.path.isfile(fname):
pickle_stat = pickle.load(open(fname, "rb"))
for key in stat_dict.keys():
if key in pickle_stat.keys():
pickle_stat[key].append(stat_dict[key][0])
else:
pickle_stat[key] = stat_dict[key]
pickle.dump(pickle_stat, open(fname, "wb"))
else:
pickle.dump(stat_dict, open(fname, "wb"))
def read_stat_pickle(self, iparams):
fname = iparams.run_no + '/pickle.stat'
if os.path.isfile(fname):
pickle_stat = pickle.load(open(fname, "rb"))
for key in pickle_stat.keys():
data = pickle_stat[key]
print "key:", key, " size:", len(data)
for d in data:
print d
def calc_avg_I_cpp(self, prep_output, iparams, avg_mode):
group_no, group_id_list, miller_index, miller_indices_ori, I, sigI, G, B, p_set, rs_set, wavelength_set, sin_theta_over_lambda_sq, SE, uc_mean, wavelength_mean, pickle_filename_set, txt_out = prep_output
from prime import Average_Mode, averaging_engine
if avg_mode == 'average': avg_mode_cpp = Average_Mode.Average
elif avg_mode == 'weighted': avg_mode_cpp = Average_Mode.Weighted
elif avg_mode == 'final': avg_mode_cpp = Average_Mode.Final
else: raise Sorry("Bad averaging mode selected: %s" % avg_mode)
sigma_max = iparams.sigma_rejection
engine = averaging_engine(group_no, group_id_list, miller_index,
miller_indices_ori, I, sigI, G, B, p_set,
rs_set, wavelength_set,
sin_theta_over_lambda_sq, SE,
pickle_filename_set)
engine.avg_mode = avg_mode_cpp
engine.sigma_max = sigma_max
engine.flag_volume_correction = iparams.flag_volume_correction
engine.n_rejection_cycle = iparams.n_rejection_cycle
engine.flag_output_verbose = iparams.flag_output_verbose
results = engine.calc_avg_I()
mdh = merge_data_handler(
results.miller_index, results.I_avg, results.sigI_avg,
(results.r_meas_top, results.r_meas_btm, results.multiplicity),
(results.I_avg_even, results.I_avg_odd, results.I_avg_even_h,
results.I_avg_odd_h, results.I_avg_even_k, results.I_avg_odd_k,
results.I_avg_even_l, results.I_avg_odd_l), uc_mean,
wavelength_mean)
return mdh, results.txt_obs_out, results.txt_reject_out
def calc_mean_unit_cell(self, results):
uc_array = [
list(pres.uc_params) for pres in results if pres is not None
]
return np.mean(uc_array, 0), np.median(uc_array,
0), np.std(uc_array, 0)
def calc_mean_postref_parameters(self, results):
params_array = [[pres.G, pres.B, pres.ry, pres.rz, pres.re, pres.r0, \
pres.voigt_nu, pres.rotx, pres.roty, pres.R_final, pres.R_xy_final, pres.SE] \
for pres in results if (pres is not None and not math.isnan(pres.G) and not math.isnan(pres.B) \
and not math.isnan(pres.ry) and not math.isnan(pres.rz) and not math.isnan(pres.re) and not math.isnan(pres.r0) \
and not math.isnan(pres.voigt_nu) and not math.isnan(pres.rotx) and not math.isnan(pres.roty) \
and not math.isnan(pres.R_final) and not math.isnan(pres.R_xy_final) and not math.isnan(pres.SE))]
return np.mean(params_array, 0), np.median(params_array,
0), np.std(params_array, 0)
def prepare_output(self, results, iparams, avg_mode):
if avg_mode == 'average':
cc_thres = 0
else:
cc_thres = iparams.frame_accept_min_cc
std_filter = iparams.sigma_rejection
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(
results)
G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean
G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med
G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std
#prepare data for merging
miller_indices_all = flex.miller_index()
miller_indices_ori_all = flex.miller_index()
I_all = flex.double()
sigI_all = flex.double()
G_all = flex.double()
B_all = flex.double()
p_all = flex.double()
rx_all = flex.double()
rs_all = flex.double()
rh_all = flex.double()
SE_all = flex.double()
sin_sq_all = flex.double()
wavelength_all = flex.double()
detector_distance_set = flex.double()
R_init_all = flex.double()
R_final_all = flex.double()
R_xy_init_all = flex.double()
R_xy_final_all = flex.double()
pickle_filename_all = flex.std_string()
filtered_results = []
cn_good_frame, cn_bad_frame_SE, cn_bad_frame_uc, cn_bad_frame_cc, cn_bad_frame_G, cn_bad_frame_re = (
0, 0, 0, 0, 0, 0)
crystal_orientation_dict = {}
for pres in results:
if pres is not None:
pickle_filepath = pres.pickle_filename.split('/')
img_filename = pickle_filepath[len(pickle_filepath) - 1]
flag_pres_ok = True
#check SE, CC, UC, G, B, gamma_e
if math.isnan(pres.G):
flag_pres_ok = False
if math.isnan(pres.SE) or np.isinf(pres.SE):
flag_pres_ok = False
if flag_pres_ok and SE_std > 0:
if abs(pres.SE - SE_med) / SE_std > std_filter:
flag_pres_ok = False
cn_bad_frame_SE += 1
if flag_pres_ok and pres.CC_final < cc_thres:
flag_pres_ok = False
cn_bad_frame_cc += 1
if flag_pres_ok:
if G_std > 0:
if abs(pres.G - G_med) / G_std > std_filter:
flag_pres_ok = False
cn_bad_frame_G += 1
if flag_pres_ok:
if re_std > 0:
if abs(pres.re - re_med) / re_std > std_filter:
flag_pres_ok = False
cn_bad_frame_re += 1
if flag_pres_ok and not good_unit_cell(
pres.uc_params, iparams, iparams.merge.uc_tolerance):
flag_pres_ok = False
cn_bad_frame_uc += 1
data_size = pres.observations.size()
if flag_pres_ok:
cn_good_frame += 1
filtered_results.append(pres)
R_init_all.append(pres.R_init)
R_final_all.append(pres.R_final)
R_xy_init_all.append(pres.R_xy_init)
R_xy_final_all.append(pres.R_xy_final)
miller_indices_all.extend(pres.observations.indices())
miller_indices_ori_all.extend(
pres.observations_original.indices())
I_all.extend(pres.observations.data())
sigI_all.extend(pres.observations.sigmas())
G_all.extend(flex.double([pres.G] * data_size))
B_all.extend(flex.double([pres.B] * data_size))
p_all.extend(pres.partiality)
rs_all.extend(pres.rs_set)
rh_all.extend(pres.rh_set)
sin_sq_all.extend(
pres.observations.two_theta(wavelength=pres.wavelength)
.sin_theta_over_lambda_sq().data())
SE_all.extend(flex.double([pres.SE] * data_size))
wavelength_all.extend(
flex.double([pres.wavelength] * data_size))
detector_distance_set.append(pres.detector_distance_mm)
pickle_filename_all.extend(
flex.std_string([pres.pickle_filename] * data_size))
crystal_orientation_dict[
pres.pickle_filename] = pres.crystal_orientation
#plot stats
self.plot_stats(filtered_results, iparams)
#write out updated crystal orientation as a pickle file
if not iparams.flag_hush:
pickle.dump(crystal_orientation_dict,
open(iparams.run_no + '/' + "crystal.o", "wb"),
pickle.HIGHEST_PROTOCOL)
#calculate average unit cell
uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(filtered_results)
unit_cell_mean = unit_cell(tuple(uc_mean))
#recalculate stats for pr parameters
pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(
filtered_results)
G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean
G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med
G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std
#from all observations merge them
crystal_symmetry = crystal.symmetry(
unit_cell=tuple(uc_mean),
space_group_symbol=iparams.target_space_group)
miller_set_all = miller.set(crystal_symmetry=crystal_symmetry,
indices=miller_indices_all,
anomalous_flag=target_anomalous_flag)
miller_array_all = miller_set_all.array(
data=I_all, sigmas=sigI_all).set_observation_type_xray_intensity()
#sort reflections according to asymmetric-unit symmetry hkl
perm = miller_array_all.sort_permutation(by_value="packed_indices")
miller_indices_all_sort = miller_array_all.indices().select(perm)
miller_indices_ori_all_sort = miller_indices_ori_all.select(perm)
I_obs_all_sort = miller_array_all.data().select(perm)
sigI_obs_all_sort = miller_array_all.sigmas().select(perm)
G_all_sort = G_all.select(perm)
B_all_sort = B_all.select(perm)
p_all_sort = p_all.select(perm)
rs_all_sort = rs_all.select(perm)
wavelength_all_sort = wavelength_all.select(perm)
sin_sq_all_sort = sin_sq_all.select(perm)
SE_all_sort = SE_all.select(perm)
pickle_filename_all_sort = pickle_filename_all.select(perm)
miller_array_uniq = miller_array_all.merge_equivalents().array(
).complete_array(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max)
matches_uniq = miller.match_multi_indices(
miller_indices_unique=miller_array_uniq.indices(),
miller_indices=miller_indices_all_sort)
pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()])
pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()])
group_id_list = flex.int(
[pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))])
tally = Counter()
for elem in group_id_list:
tally[elem] += 1
cn_group = len(tally)
#preparte txt out stat
txt_out = 'Summary of refinement and merging\n'
txt_out += ' No. good frames: %12.0f\n' % (cn_good_frame)
txt_out += ' No. bad cc frames: %12.0f\n' % (cn_bad_frame_cc)
txt_out += ' No. bad G frames) : %12.0f\n' % (cn_bad_frame_G)
txt_out += ' No. bad unit cell frames: %12.0f\n' % (cn_bad_frame_uc)
txt_out += ' No. bad gamma_e frames: %12.0f\n' % (cn_bad_frame_re)
txt_out += ' No. bad SE: %12.0f\n' % (cn_bad_frame_SE)
txt_out += ' No. observations: %12.0f\n' % (
len(I_obs_all_sort))
txt_out += 'Mean target value (BEFORE: Mean Median (Std.))\n'
txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_init_all), np.median(R_init_all), np.std(R_init_all))
txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_xy_init_all), np.median(R_xy_init_all),
np.std(R_xy_init_all))
txt_out += 'Mean target value (AFTER: Mean Median (Std.))\n'
txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_final_all), np.median(R_final_all), np.std(R_final_all))
txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_xy_final_all), np.median(R_xy_final_all),
np.std(R_xy_final_all))
txt_out += ' SE: %12.2f %12.2f (%9.2f)\n' % (
SE_mean, SE_med, SE_std)
txt_out += ' G: %12.3e %12.3e (%9.2e)\n' % (
G_mean, G_med, G_std)
txt_out += ' B: %12.2f %12.2f (%9.2f)\n' % (
B_mean, B_med, B_std)
txt_out += ' Rot.x: %12.2f %12.2f (%9.2f)\n' % (
rotx_mean * 180 / math.pi, rotx_med * 180 / math.pi,
rotx_std * 180 / math.pi)
txt_out += ' Rot.y: %12.2f %12.2f (%9.2f)\n' % (
roty_mean * 180 / math.pi, roty_med * 180 / math.pi,
roty_std * 180 / math.pi)
txt_out += ' gamma_y: %12.5f %12.5f (%9.5f)\n' % (
ry_mean, ry_med, ry_std)
txt_out += ' gamma_z: %12.5f %12.5f (%9.5f)\n' % (
rz_mean, rz_med, rz_std)
txt_out += ' gamma_0: %12.5f %12.5f (%9.5f)\n' % (
r0_mean, r0_med, r0_std)
txt_out += ' gamma_e: %12.5f %12.5f (%9.5f)\n' % (
re_mean, re_med, re_std)
txt_out += ' voigt_nu: %12.5f %12.5f (%9.5f)\n' % (
voigt_nu_mean, voigt_nu_med, voigt_nu_std)
txt_out += ' unit cell\n'
txt_out += ' a: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[0], uc_med[0], uc_std[0])
txt_out += ' b: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[1], uc_med[1], uc_std[1])
txt_out += ' c: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[2], uc_med[2], uc_std[2])
txt_out += ' alpha: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[3], uc_med[3], uc_std[3])
txt_out += ' beta: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[4], uc_med[4], uc_std[4])
txt_out += ' gamma: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[5], uc_med[5], uc_std[5])
txt_out += 'Parmeters from integration (not-refined)\n'
txt_out += ' Wavelength: %12.5f %12.5f (%9.5f)\n' % (
np.mean(wavelength_all), np.median(wavelength_all),
np.std(wavelength_all))
txt_out += ' Detector distance: %12.5f %12.5f (%9.5f)\n' % (
np.mean(detector_distance_set), np.median(detector_distance_set),
np.std(detector_distance_set))
txt_out += '* (standard deviation)\n'
#write out stat. pickle
if not iparams.flag_hush:
stat_dict = {"n_frames_good": [cn_good_frame], \
"n_frames_bad_cc": [cn_bad_frame_cc], \
"n_frames_bad_G": [cn_bad_frame_G], \
"n_frames_bad_uc": [cn_bad_frame_uc], \
"n_frames_bad_gamma_e": [cn_bad_frame_re], \
"n_frames_bad_SE": [cn_bad_frame_SE], \
"n_observations": [len(I_obs_all_sort)], \
"R_start": [np.mean(R_init_all)], \
"R_end": [np.mean(R_final_all)], \
"R_xy_start": [np.mean(R_xy_init_all)], \
"R_xy_end": [np.mean(R_xy_final_all)], \
"mean_gamma_y": [ry_mean], \
"std_gamma_y": [ry_std], \
"mean_gamma_z": [rz_mean], \
"std_gamma_z": [rz_std], \
"mean_gamma_0": [r0_mean], \
"std_gamma_0": [r0_std], \
"mean_gamma_e": [re_mean], \
"std_gamma_e": [re_std], \
"mean_voigt_nu": [voigt_nu_mean], \
"std_voigt_nu": [voigt_nu_std], \
"mean_a": [uc_mean[0]], \
"std_a": [uc_std[0]], \
"mean_b": [uc_mean[1]], \
"std_b": [uc_std[1]], \
"mean_c": [uc_mean[2]], \
"std_c": [uc_std[2]], \
"mean_alpha": [uc_mean[3]], \
"std_alpha": [uc_std[3]], \
"mean_beta": [uc_mean[4]], \
"std_beta": [uc_std[4]], \
"mean_gamma": [uc_mean[5]], \
"std_gamma": [uc_std[5]]}
self.write_stat_pickle(iparams, stat_dict)
return cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, \
I_obs_all_sort, sigI_obs_all_sort,G_all_sort, B_all_sort, \
p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, uc_mean, \
np.mean(wavelength_all), pickle_filename_all_sort, txt_out
def write_output(self, mdh, iparams, output_mtz_file_prefix, avg_mode):
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
uc_mean = mdh.uc_mean
wavelength_mean = mdh.wavelength_mean
#output mtz file and report binning stat
miller_set_merge = crystal.symmetry(
unit_cell=unit_cell(tuple(uc_mean)),
space_group_symbol=iparams.target_space_group).build_miller_set(
anomalous_flag=target_anomalous_flag,
d_min=iparams.merge.d_min)
mdh.generate_miller_array_from_miller_set(miller_set_merge,
target_anomalous_flag)
miller_array_complete = miller_set_merge.array()
fake_data = flex.double([1.0] * len(miller_array_complete.indices()))
miller_array_template_asu = miller_array_complete.customized_copy(data=fake_data, \
sigmas=fake_data).resolution_filter(d_min=iparams.merge.d_min, \
d_max=iparams.merge.d_max)
n_refl_all = mdh.get_size()
#do another resolution filter here
i_sel_res = mdh.miller_array_merge.resolution_filter_selection(
d_min=iparams.merge.d_min, d_max=iparams.merge.d_max)
mdh.reduce_by_selection(i_sel_res)
n_refl_out_resolutions = n_refl_all - mdh.get_size()
#remove outliers
sequences = flex.int(range(mdh.get_size()))
good_sequences = []
for i_rejection in range(iparams.n_rejection_cycle):
binner_merge = mdh.miller_array_merge.setup_binner(n_bins=200)
for i_bin in range(1, 201):
i_binner = (binner_merge.bin_indices() == i_bin)
I_obs_bin = mdh.miller_array_merge.data().select(i_binner)
sequences_bin = sequences.select(i_binner)
if len(I_obs_bin) > 0:
I_obs_bin = mdh.miller_array_merge.data().select(i_binner)
try:
i_filter = flex.abs(
(I_obs_bin - np.median(I_obs_bin)) /
np.std(I_obs_bin)) < 10
except Exception, e:
print "Warning: outlier rejection by bins failed because of floating point."
print e
i_filter = flex.bool([True] * len(I_obs_bin))
good_sequences.extend(list(sequences_bin.select(i_filter)))
mdh.reduce_by_selection(flex.size_t(good_sequences))
n_refl_outliers = n_refl_all - n_refl_out_resolutions - mdh.get_size()
#get iso if given.
mxh = mx_handler()
flag_hklisoin_found, miller_array_iso = mxh.get_miller_array_from_reflection_file(
iparams.hklisoin)
#write output files
if output_mtz_file_prefix != '':
#write as mtz file
miller_array_merge_unique = mdh.miller_array_merge.merge_equivalents(
).array()
info = miller.array_info(wavelength=wavelength_mean)
miller_array_merge_unique.set_info(info)
mtz_dataset_merge = miller_array_merge_unique.as_mtz_dataset(
column_root_label="IOBS")
mtz_dataset_merge.mtz_object().write(
file_name=output_mtz_file_prefix + '_merge.mtz')
#write as cns file
f_cns = open(output_mtz_file_prefix + '_merge.hkl', 'w')
miller_array_merge_unique.export_as_cns_hkl(file_object=f_cns)
f_cns.close()
if iparams.flag_hush:
cc12, n_refl_cc12 = mdh.get_cc12()
cciso, n_refl_cciso = mdh.get_cciso(miller_array_iso)
cc_anom_acentric, n_refl_anom_acentric = mdh.get_cc_anom()
txt_out = 'Warning: flag_hush is set to True. Continue without writing merging statistic tables.\n'
txt_out += 'Bin Resolution Range Completeness <N_obs> |Rmerge Rsplit CC1/2 N_ind |CCiso N_ind|CCanoma N_ind| <I/sigI> <I> <sigI> <I**2>\n'
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
txt_out += ' TOTAL %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \
%((mdh.get_size()/miller_array_template_asu.size())*100, \
mdh.get_size(), miller_array_template_asu.size(),\
mdh.get_multiplicity(), mdh.get_r_meas()*100, mdh.get_r_split()*100, \
cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \
cc_anom_acentric, n_refl_anom_acentric, \
mdh.get_mean_IoversigI(), mdh.get_mean_I(), mdh.get_mean_sigI(), mdh.get_second_moment())
else:
#calculate isotropic B-factor
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = mdh.miller_array_merge.as_amplitude_array()
observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f,
asu_contents,
e_statistics=True)
B_merged = wp.wilson_b
except Exception, e:
B_merged = 0
print "Warning: b-factor calculation in mod_util failed. Reset b-factor to 0"
print e
#report binning stats
txt_out = '\n'
txt_out += 'Isotropic B-factor: %7.2f\n' % (B_merged)
txt_out += 'No. of reflections\n'
txt_out += ' all: %7.0f\n' % (n_refl_all)
txt_out += ' outside resolution: %7.0f\n' % (
n_refl_out_resolutions)
txt_out += ' outliers: %7.0f\n' % (n_refl_outliers)
txt_out += ' total left: %7.0f\n' % (mdh.get_size())
txt_out += 'Summary for ' + output_mtz_file_prefix + '_merge.mtz\n'
txt_out += 'Bin Resolution Range Completeness <N_obs> |Rmerge Rsplit CC1/2 N_ind |CCiso N_ind|CCanoma N_ind| <I/sigI> <I> <sigI> <I**2>\n'
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
#for stat pickle
sp_res, sp_complete, sp_n_obs, sp_cc12, sp_cc12_anom, sp_rmerge, sp_i_o_sigi, sp_isqr = (
[], [], [], [], [], [], [], [])
#binning
binner_template_asu = miller_array_template_asu.setup_binner(
n_bins=iparams.n_bins)
binner_template_asu_indices = binner_template_asu.bin_indices()
#for stats on axis cones
mdh_astar = deepcopy(mdh)
mdh_bstar = deepcopy(mdh)
mdh_cstar = deepcopy(mdh)
mdh_astar.reduce_to_cone_on_axis((1, 0, 0),
iparams.percent_cone_fraction)
mdh_bstar.reduce_to_cone_on_axis((0, 1, 0),
iparams.percent_cone_fraction)
mdh_cstar.reduce_to_cone_on_axis((0, 0, 1),
iparams.percent_cone_fraction)
#prepare text out for axis cones
txt_out_cone = 'Summary of CC1/2 on three crystal axes\n'
txt_out_cone += 'Bin Resolution Range CC1/2 <I> N_refl \n'
txt_out_cone += ' a* b* c* | a* b* c* | a* b* c* \n'
txt_out_cone += '---------------------------------------------------------------------------------------------------------\n'
for i in range(1, iparams.n_bins + 1):
i_binner = (binner_template_asu_indices == i)
miller_indices_template_bin = miller_array_template_asu.indices(
).select(i_binner)
#for all reflections
mdh_bin = deepcopy(mdh)
mdh_bin.reduce_by_miller_index(miller_indices_template_bin)
cc12, n_refl_cc12 = mdh_bin.get_cc12()
cciso, n_refl_cciso = mdh_bin.get_cciso(miller_array_iso)
cc_anom_acentric, n_refl_anom_acentric = mdh_bin.get_cc_anom()
completeness = (mdh_bin.get_size() /
len(miller_indices_template_bin)) * 100
multiplicity = mdh_bin.get_multiplicity()
txt_out += '%02d %7.2f - %7.2f %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \
%(i, binner_template_asu.bin_d_range(i)[0], binner_template_asu.bin_d_range(i)[1], \
completeness, \
mdh_bin.get_size(), len(miller_indices_template_bin),\
multiplicity, mdh_bin.get_r_meas()*100, mdh_bin.get_r_split()*100, \
cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \
cc_anom_acentric, n_refl_anom_acentric, \
mdh_bin.get_mean_IoversigI(), mdh_bin.get_mean_I(), mdh_bin.get_mean_sigI(), mdh_bin.get_second_moment())
#for reflections on cones
mdh_astar_bin = deepcopy(mdh_astar)
mdh_astar_bin.reduce_by_miller_index(
miller_indices_template_bin)
cc12_astar, n_refl_cc12_astar = mdh_astar_bin.get_cc12()
mdh_bstar_bin = deepcopy(mdh_bstar)
mdh_bstar_bin.reduce_by_miller_index(
miller_indices_template_bin)
cc12_bstar, n_refl_cc12_bstar = mdh_bstar_bin.get_cc12()
mdh_cstar_bin = deepcopy(mdh_cstar)
mdh_cstar_bin.reduce_by_miller_index(
miller_indices_template_bin)
cc12_cstar, n_refl_cc12_cstar = mdh_cstar_bin.get_cc12()
txt_out_cone += '%02d %7.2f - %7.2f %7.2f %7.2f %7.2f %10.1f %10.1f %10.1f %6.0f %6.0f %6.0f\n' \
%(i, binner_template_asu.bin_d_range(i)[0], binner_template_asu.bin_d_range(i)[1], \
cc12_astar*100, cc12_bstar*100, cc12_cstar*100, \
mdh_astar_bin.get_mean_I(), mdh_bstar_bin.get_mean_I(), mdh_cstar_bin.get_mean_I(), \
n_refl_cc12_astar, n_refl_cc12_bstar, n_refl_cc12_cstar)
#for stat pickle
sp_res.append(binner_template_asu.bin_d_range(i)[1])
sp_complete.append(completeness)
sp_n_obs.append(multiplicity)
sp_cc12.append(cc12)
sp_cc12_anom.append(cc_anom_acentric)
sp_rmerge.append(mdh_bin.get_r_meas() * 100)
sp_i_o_sigi.append(mdh_bin.get_mean_IoversigI())
sp_isqr.append(mdh.get_second_moment())
#txt out total for all reflections
cc12, n_refl_cc12 = mdh.get_cc12()
cciso, n_refl_cciso = mdh.get_cciso(miller_array_iso)
cc_anom_acentric, n_refl_anom_acentric = mdh.get_cc_anom()
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
txt_out += ' TOTAL %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \
%((mdh.get_size()/miller_array_template_asu.size())*100, \
mdh.get_size(), miller_array_template_asu.size(),\
mdh.get_multiplicity(), mdh.get_r_meas()*100, mdh.get_r_split()*100, \
cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \
cc_anom_acentric, n_refl_anom_acentric, \
mdh.get_mean_IoversigI(), mdh.get_mean_I(), mdh.get_mean_sigI(), mdh.get_second_moment())
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
txt_out += '\n'
#txt out total for reflections on cones
cc12_astar, n_refl_cc12_astar = mdh_astar.get_cc12()
cc12_bstar, n_refl_cc12_bstar = mdh_bstar.get_cc12()
cc12_cstar, n_refl_cc12_cstar = mdh_cstar.get_cc12()
txt_out_cone += '----------------------------------------------------------------------------------------------------------\n'
txt_out_cone += ' total %7.2f %7.2f %7.2f %10.1f %10.1f %10.1f %6.0f %6.0f %6.0f\n' \
%(cc12_astar*100, cc12_bstar*100, cc12_cstar*100, \
mdh_astar.get_mean_I(), mdh_bstar.get_mean_I(), mdh_cstar.get_mean_I(), \
n_refl_cc12_astar, n_refl_cc12_bstar, n_refl_cc12_cstar)
txt_out_cone += '----------------------------------------------------------------------------------------------------------\n'
txt_out_cone += '\n'
#save data for stat. pickle in stat_dict
stat_dict = {"binned_resolution": [sp_res], \
"binned_completeness": [sp_complete], \
"binned_n_obs": [sp_n_obs], \
"binned_cc12": [sp_cc12], \
"binned_cc12_anom": [sp_cc12_anom], \
"binned_rmerge": [sp_rmerge], \
"binned_i_o_sigi": [sp_i_o_sigi], \
"binned_isqr": [sp_isqr], \
"total_res_max": [mdh.miller_array_merge.d_max_min()[0]], \
"total_res_min": [mdh.miller_array_merge.d_max_min()[1]], \
"total_completeness": [(mdh.get_size()/miller_array_template_asu.size())*100], \
"total_n_obs": [mdh.get_multiplicity()], \
"total_cc12": [mdh.get_cc12()[0]*100], \
"total_rmerge": [mdh.get_r_meas()*100], \
"total_i_o_sigi": [mdh.get_mean_IoversigI()], \
"space_group_info": [mdh.miller_array_merge.space_group_info()], \
}
self.write_stat_pickle(iparams, stat_dict)
txt_out += txt_out_cone
for orig, recycled in zip((ma1, ma2, ma3, ma4), recycled_arrays):
assert orig.size() == recycled.size()
recycled = recycled.customized_copy(
anomalous_flag=orig.anomalous_flag())
orig, recycled = orig.common_sets(recycled)
assert orig.indices().all_eq(recycled.indices())
assert approx_equal(orig.data(), recycled.data(), eps=1e-5)
#
cif_model = reader(input_string=r3adrsf,
builder=cif.builders.cif_model_builder()).model()
cs = cif.builders.crystal_symmetry_builder(
cif_model["r3adrsf"]).crystal_symmetry
ma_builder = cif.builders.miller_array_builder(
cif_model['r3adrAsf'],
base_array_info=miller.array_info(crystal_symmetry_from_file=cs))
miller_arrays = ma_builder.arrays().values()
assert len(miller_arrays) == 4
mas_as_cif_block = cif.miller_arrays_as_cif_block(
miller_arrays[0].map_to_asu(),
column_names=miller_arrays[0].info().labels,
format="corecif")
for array in miller_arrays[1:]:
labels = array.info().labels
if len(labels) > 1:
for label in labels:
if label.startswith("wavelength_id"):
labels.remove(label)
mas_as_cif_block.add_miller_array(array=array.map_to_asu(),
column_names=array.info().labels)
s = StringIO()
def scaled_data_as_miller_array(reflection_table_list,
experiments,
best_unit_cell=None,
anomalous_flag=False):
"""Get a scaled miller array from an experiment and reflection table."""
if len(reflection_table_list) > 1:
joint_table = flex.reflection_table()
for reflection_table in reflection_table_list:
# better to just create many miller arrays and join them?
refl_for_joint_table = flex.reflection_table()
for col in [
"miller_index",
"intensity.scale.value",
"inverse_scale_factor",
"intensity.scale.variance",
]:
refl_for_joint_table[col] = reflection_table[col]
good_refl_sel = ~reflection_table.get_flags(
reflection_table.flags.bad_for_scaling, all=False)
refl_for_joint_table = refl_for_joint_table.select(good_refl_sel)
joint_table.extend(refl_for_joint_table)
else:
reflection_table = reflection_table_list[0]
good_refl_sel = ~reflection_table.get_flags(
reflection_table.flags.bad_for_scaling, all=False)
joint_table = reflection_table.select(good_refl_sel)
# Filter out negative scale factors to avoid merging statistics errors.
# These are not removed from the output data, as it is likely one would
# want to do further analysis e.g. delta cc1/2 and rescaling, to exclude
# certain data and get better scale factors for all reflections.
pos_scales = joint_table["inverse_scale_factor"] > 0
if pos_scales.count(False) > 0:
logger.info(
"""There are %s reflections with non-positive scale factors which
will not be used for calculating merging statistics""",
pos_scales.count(False),
)
joint_table = joint_table.select(pos_scales)
if best_unit_cell is None:
best_unit_cell = determine_best_unit_cell(experiments)
miller_set = miller.set(
crystal_symmetry=crystal.symmetry(
unit_cell=best_unit_cell,
space_group=experiments[0].crystal.get_space_group(),
assert_is_compatible_unit_cell=False,
),
indices=joint_table["miller_index"],
anomalous_flag=anomalous_flag,
)
i_obs = miller.array(
miller_set,
data=joint_table["intensity.scale.value"] /
joint_table["inverse_scale_factor"],
)
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas((joint_table["intensity.scale.variance"]**0.5) /
joint_table["inverse_scale_factor"])
i_obs.set_info(
miller.array_info(source="DIALS", source_type="reflection_tables"))
return i_obs
def write_output(self, mdh, iparams, output_mtz_file_prefix, avg_mode):
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
uc_mean = mdh.uc_mean
wavelength_mean = mdh.wavelength_mean
#output mtz file and report binning stat
miller_set_merge = crystal.symmetry(
unit_cell=unit_cell(tuple(uc_mean)),
space_group_symbol=iparams.target_space_group).build_miller_set(
anomalous_flag=target_anomalous_flag,
d_min=iparams.merge.d_min)
mdh.generate_miller_array_from_miller_set(miller_set_merge,
target_anomalous_flag)
miller_array_complete = miller_set_merge.array()
fake_data = flex.double([1.0] * len(miller_array_complete.indices()))
miller_array_template_asu = miller_array_complete.customized_copy(data=fake_data, \
sigmas=fake_data).resolution_filter(d_min=iparams.merge.d_min, \
d_max=iparams.merge.d_max)
n_refl_all = mdh.get_size()
#do another resolution filter here
i_sel_res = mdh.miller_array_merge.resolution_filter_selection(
d_min=iparams.merge.d_min, d_max=iparams.merge.d_max)
mdh.reduce_by_selection(i_sel_res)
n_refl_out_resolutions = n_refl_all - mdh.get_size()
#remove outliers
sequences = flex.int(range(mdh.get_size()))
good_sequences = []
for i_rejection in range(iparams.n_rejection_cycle):
binner_merge = mdh.miller_array_merge.setup_binner(n_bins=200)
for i_bin in range(1, 201):
i_binner = (binner_merge.bin_indices() == i_bin)
I_obs_bin = mdh.miller_array_merge.data().select(i_binner)
sequences_bin = sequences.select(i_binner)
if len(I_obs_bin) > 0:
I_obs_bin = mdh.miller_array_merge.data().select(i_binner)
try:
i_filter = flex.abs(
(I_obs_bin - np.median(I_obs_bin)) /
np.std(I_obs_bin)) < 10
except Exception as e:
print "Warning: outlier rejection by bins failed because of floating point."
print e
i_filter = flex.bool([True] * len(I_obs_bin))
good_sequences.extend(list(sequences_bin.select(i_filter)))
mdh.reduce_by_selection(flex.size_t(good_sequences))
n_refl_outliers = n_refl_all - n_refl_out_resolutions - mdh.get_size()
#get iso if given.
mxh = mx_handler()
flag_hklisoin_found, miller_array_iso = mxh.get_miller_array_from_reflection_file(
iparams.hklisoin)
#write output files
if output_mtz_file_prefix != '':
#write as mtz file
miller_array_merge_unique = mdh.miller_array_merge.merge_equivalents(
).array()
info = miller.array_info(wavelength=wavelength_mean)
miller_array_merge_unique.set_info(info)
mtz_dataset_merge = miller_array_merge_unique.as_mtz_dataset(
column_root_label="IOBS")
mtz_dataset_merge.mtz_object().write(
file_name=output_mtz_file_prefix + '_merge.mtz')
#write as cns file
f_cns = open(output_mtz_file_prefix + '_merge.hkl', 'w')
miller_array_merge_unique.export_as_cns_hkl(file_object=f_cns)
f_cns.close()
#calculate merging stat table
if True:
#calculate isotropic B-factor
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = mdh.miller_array_merge.as_amplitude_array()
observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f,
asu_contents,
e_statistics=True)
B_merged = wp.wilson_b
except Exception as e:
B_merged = 0
print "Warning: b-factor calculation in mod_util failed. Reset b-factor to 0"
print e
#report binning stats
txt_out = '\n'
txt_out += 'Isotropic B-factor: %7.2f\n' % (B_merged)
txt_out += 'No. of reflections\n'
txt_out += ' all: %7.0f\n' % (n_refl_all)
txt_out += ' outside resolution: %7.0f\n' % (
n_refl_out_resolutions)
txt_out += ' outliers: %7.0f\n' % (n_refl_outliers)
txt_out += ' total left: %7.0f\n' % (mdh.get_size())
txt_out += 'Summary for ' + output_mtz_file_prefix + '_merge.mtz\n'
txt_out += 'Bin Resolution Range Completeness <N_obs> |Rmerge Rsplit CC1/2 N_ind |CCiso N_ind|CCanoma N_ind| <I/sigI> <I> <sigI> <I**2>\n'
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
#for stat pickle
sp_res, sp_complete, sp_n_obs, sp_cc12, sp_cc12_anom, sp_rmerge, sp_i_o_sigi, sp_isqr = (
[], [], [], [], [], [], [], [])
#binning
binner_template_asu = miller_array_template_asu.setup_binner(
n_bins=iparams.n_bins)
binner_template_asu_indices = binner_template_asu.bin_indices()
#for stats on axis cones
mdh_astar = deepcopy(mdh)
mdh_bstar = deepcopy(mdh)
mdh_cstar = deepcopy(mdh)
mdh_astar.reduce_to_cone_on_axis((1, 0, 0),
iparams.percent_cone_fraction)
mdh_bstar.reduce_to_cone_on_axis((0, 1, 0),
iparams.percent_cone_fraction)
mdh_cstar.reduce_to_cone_on_axis((0, 0, 1),
iparams.percent_cone_fraction)
#prepare text out for axis cones
txt_out_cone = 'Summary of CC1/2 on three crystal axes\n'
txt_out_cone += 'Bin Resolution Range CC1/2 <I> N_refl \n'
txt_out_cone += ' a* b* c* | a* b* c* | a* b* c* \n'
txt_out_cone += '---------------------------------------------------------------------------------------------------------\n'
for i in range(1, iparams.n_bins + 1):
i_binner = (binner_template_asu_indices == i)
miller_indices_template_bin = miller_array_template_asu.indices(
).select(i_binner)
#for all reflections
mdh_bin = deepcopy(mdh)
mdh_bin.reduce_by_miller_index(miller_indices_template_bin)
cc12, n_refl_cc12 = mdh_bin.get_cc12()
cciso, n_refl_cciso = mdh_bin.get_cciso(miller_array_iso)
cc_anom_acentric, n_refl_anom_acentric = mdh_bin.get_cc_anom()
completeness = (mdh_bin.get_size() /
len(miller_indices_template_bin)) * 100
multiplicity = mdh_bin.get_multiplicity()
txt_out += '%02d %7.2f - %7.2f %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \
%(i, binner_template_asu.bin_d_range(i)[0], binner_template_asu.bin_d_range(i)[1], \
completeness, \
mdh_bin.get_size(), len(miller_indices_template_bin),\
multiplicity, mdh_bin.get_r_meas()*100, mdh_bin.get_r_split()*100, \
cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \
cc_anom_acentric, n_refl_anom_acentric, \
mdh_bin.get_mean_IoversigI(), mdh_bin.get_mean_I(), mdh_bin.get_mean_sigI(), mdh_bin.get_second_moment())
#for reflections on cones
mdh_astar_bin = deepcopy(mdh_astar)
mdh_astar_bin.reduce_by_miller_index(
miller_indices_template_bin)
cc12_astar, n_refl_cc12_astar = mdh_astar_bin.get_cc12()
mdh_bstar_bin = deepcopy(mdh_bstar)
mdh_bstar_bin.reduce_by_miller_index(
miller_indices_template_bin)
cc12_bstar, n_refl_cc12_bstar = mdh_bstar_bin.get_cc12()
mdh_cstar_bin = deepcopy(mdh_cstar)
mdh_cstar_bin.reduce_by_miller_index(
miller_indices_template_bin)
cc12_cstar, n_refl_cc12_cstar = mdh_cstar_bin.get_cc12()
txt_out_cone += '%02d %7.2f - %7.2f %7.2f %7.2f %7.2f %10.1f %10.1f %10.1f %6.0f %6.0f %6.0f\n' \
%(i, binner_template_asu.bin_d_range(i)[0], binner_template_asu.bin_d_range(i)[1], \
cc12_astar*100, cc12_bstar*100, cc12_cstar*100, \
mdh_astar_bin.get_mean_I(), mdh_bstar_bin.get_mean_I(), mdh_cstar_bin.get_mean_I(), \
n_refl_cc12_astar, n_refl_cc12_bstar, n_refl_cc12_cstar)
#for stat pickle
sp_res.append(binner_template_asu.bin_d_range(i)[1])
sp_complete.append(completeness)
sp_n_obs.append(multiplicity)
sp_cc12.append(cc12)
sp_cc12_anom.append(cc_anom_acentric)
sp_rmerge.append(mdh_bin.get_r_meas() * 100)
sp_i_o_sigi.append(mdh_bin.get_mean_IoversigI())
sp_isqr.append(mdh.get_second_moment())
#txt out total for all reflections
cc12, n_refl_cc12 = mdh.get_cc12()
cciso, n_refl_cciso = mdh.get_cciso(miller_array_iso)
cc_anom_acentric, n_refl_anom_acentric = mdh.get_cc_anom()
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
txt_out += ' TOTAL %5.1f %6.0f / %6.0f %7.2f %7.2f %7.2f %7.2f %6.0f %7.2f %6.0f %7.2f %6.0f %8.2f %10.1f %8.1f %6.2f\n' \
%((mdh.get_size()/miller_array_template_asu.size())*100, \
mdh.get_size(), miller_array_template_asu.size(),\
mdh.get_multiplicity(), mdh.get_r_meas()*100, mdh.get_r_split()*100, \
cc12*100, n_refl_cc12, cciso*100, n_refl_cciso, \
cc_anom_acentric, n_refl_anom_acentric, \
mdh.get_mean_IoversigI(), mdh.get_mean_I(), mdh.get_mean_sigI(), mdh.get_second_moment())
txt_out += '--------------------------------------------------------------------------------------------------------------------------------------------------\n'
txt_out += '\n'
#txt out total for reflections on cones
cc12_astar, n_refl_cc12_astar = mdh_astar.get_cc12()
cc12_bstar, n_refl_cc12_bstar = mdh_bstar.get_cc12()
cc12_cstar, n_refl_cc12_cstar = mdh_cstar.get_cc12()
txt_out_cone += '----------------------------------------------------------------------------------------------------------\n'
txt_out_cone += ' total %7.2f %7.2f %7.2f %10.1f %10.1f %10.1f %6.0f %6.0f %6.0f\n' \
%(cc12_astar*100, cc12_bstar*100, cc12_cstar*100, \
mdh_astar.get_mean_I(), mdh_bstar.get_mean_I(), mdh_cstar.get_mean_I(), \
n_refl_cc12_astar, n_refl_cc12_bstar, n_refl_cc12_cstar)
txt_out_cone += '----------------------------------------------------------------------------------------------------------\n'
txt_out_cone += '\n'
txt_out_table1 = "Table1 (" + avg_mode + ")\n"
txt_out_table1 += " Space group: " + str(
mdh.miller_array_merge.space_group_info()) + "\n"
txt_out_table1 += " Cell dimensions: %6.2f, %6.2f, %6.2f, %6.2f, %6.2f, %6.2f\n" % tuple(
mdh.uc_mean)
txt_out_table1 += " Resolution (A): %6.2f - %6.2f (%6.2f - %6.2f)\n" % (
mdh.miller_array_merge.d_max_min()[0],
mdh.miller_array_merge.d_max_min()[1], sp_res[-2], sp_res[-1])
txt_out_table1 += " Rmerge: %6.2f (%6.2f)\n" % (
mdh.get_r_meas() * 100, sp_rmerge[-1])
txt_out_table1 += " CC1/2: %6.2f (%6.2f)\n" % (mdh.get_cc12()[0] *
100, sp_cc12[-1])
txt_out_table1 += " I/sigI: %6.2f (%6.2f)\n" % (
mdh.get_mean_IoversigI(), sp_i_o_sigi[-1])
txt_out_table1 += " Completeness (%%): %6.2f (%6.2f)\n" % (
(mdh.get_size() / miller_array_template_asu.size()) * 100,
sp_complete[-1])
txt_out_table1 += " Redundancy: %6.2f (%6.2f)\n" % (
mdh.get_multiplicity(), sp_n_obs[-1])
#save data for stat. pickle in stat_dict
if not iparams.flag_hush:
stat_dict = {"binned_resolution": [sp_res], \
"binned_completeness": [sp_complete], \
"binned_n_obs": [sp_n_obs], \
"binned_cc12": [sp_cc12], \
"binned_cc12_anom": [sp_cc12_anom], \
"binned_rmerge": [sp_rmerge], \
"binned_i_o_sigi": [sp_i_o_sigi], \
"binned_isqr": [sp_isqr], \
"total_res_max": [mdh.miller_array_merge.d_max_min()[0]], \
"total_res_min": [mdh.miller_array_merge.d_max_min()[1]], \
"total_completeness": [(mdh.get_size()/miller_array_template_asu.size())*100], \
"total_n_obs": [mdh.get_multiplicity()], \
"total_cc12": [mdh.get_cc12()[0]*100], \
"total_rmerge": [mdh.get_r_meas()*100], \
"total_i_o_sigi": [mdh.get_mean_IoversigI()], \
"space_group_info": [mdh.miller_array_merge.space_group_info()], \
}
self.write_stat_pickle(iparams, stat_dict)
txt_out += txt_out_cone + txt_out_table1
return mdh, txt_out
def write(self, experiments, reflections):
"""
Write the experiments and reflections to file
"""
# if mmmcif filename is auto, then choose scaled.cif or integrated.cif
if self.params.mmcif.hklout in (None, Auto, "auto"):
if ("intensity.scale.value"
in reflections) and ("intensity.scale.variance"
in reflections):
filename = "scaled.cif"
logger.info(
"Data appears to be scaled, setting mmcif.hklout = 'scaled_unmerged.cif'"
)
else:
filename = "integrated.cif"
logger.info(
"Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'"
)
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated,
all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
### Assumes you want to apply the lp and dqe corrections to sum and prf
### Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_computing.data_reduction"] = (
"%s (Winter, G. et al., 2018)" % dials_version)
cif_block[
"_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97."
# Hard coding X-ray
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity).lower()
# FIXME finish metadata addition - detector and source details needed
# http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html
## Add source information;
# _diffrn_source.pdbx_wavelength_list = (list of wavelengths)
# _diffrn_source.source = (general class of source e.g. synchrotron)
# _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04)
wls = []
epochs = []
for exp in experiments:
wls.append(round(exp.beam.get_wavelength(), 5))
epochs.append(exp.scan.get_epochs()[0])
unique_wls = set(wls)
cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join(
str(w) for w in unique_wls)
## Add detector information;
# _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc)
# _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd)
# _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M)
# One date is required, so if multiple just use the first date.
min_epoch = min(epochs)
date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch))
cif_block["_diffrn_detector.pdbx_collection_date"] = date_str
# Write the crystal information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
))
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group()))
cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha,
beta, gamma, latt_type))
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
))
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row((
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
))
cif_block.add_loop(cif_loop)
# Make a dict of unit_cell parameters
unit_cell_parameters = {}
if crystal.num_scan_points > 1:
for i in range(crystal.num_scan_points):
a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point(
i).parameters()
unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma)
else:
unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma)
### _pdbx_diffrn_image_proc has been removed from the dictionary extension.
### Keeping this section commented out as it may be added back in some
### form in future
#
# Write the image data
# scan = experiments[0].scan
# z0 = scan.get_image_range()[0]
#
# cif_loop = iotbx.cif.model.loop(
# header=("_pdbx_diffrn_image_proc.image_id",
# "_pdbx_diffrn_image_proc.crystal_id",
# "_pdbx_diffrn_image_proc.image_number",
# "_pdbx_diffrn_image_proc.phi_value",
# "_pdbx_diffrn_image_proc.wavelength",
# "_pdbx_diffrn_image_proc.cell_length_a",
# "_pdbx_diffrn_image_proc.cell_length_b",
# "_pdbx_diffrn_image_proc.cell_length_c",
# "_pdbx_diffrn_image_proc.cell_angle_alpha",
# "_pdbx_diffrn_image_proc.cell_angle_beta",
# "_pdbx_diffrn_image_proc.cell_angle_gamma"))
# for i in range(len(scan)):
# z = z0 + i
# if crystal.num_scan_points > 1:
# a, b, c, alpha, beta, gamma = unit_cell_parameters[i]
# else:
# a, b, c, alpha, beta, gamma = unit_cell_parameters[0]
# # phi is the angle at the image centre
# phi = scan.get_angle_from_image_index(z + 0.5, deg=True)
# cif_loop.add_row((i+1, 1, z, phi, wavelength,
# a, b, c, alpha, beta, gamma))
# cif_block.add_loop(cif_loop)
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
headernames = {
"scales": "_pdbx_diffrn_unmerged_refln.scale_value",
"intensity.scale.value":
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"intensity.scale.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum",
"intensity.sum.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf",
"intensity.prf.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection",
"partiality": "_pdbx_diffrn_unmerged_refln.partiality",
}
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = flex.sqrt(
reflections["intensity.scale.variance"])
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"])
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = flex.sqrt(
reflections["intensity.sum.variance"])
variables_present.extend(
["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = flex.sqrt(
reflections["intensity.prf.variance"])
variables_present.extend(
["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (headernames[name], )
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set,
data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS",
source_type="reflection_tables"))
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
)
cif_block.update(result.as_cif_block())
cif_loop = iotbx.cif.model.loop(header=header)
for i, r in enumerate(reflections.rows()):
refl_id = i + 1
scan_id = r["id"] + 1
_, _, _, _, z0, z1 = r["bbox"]
h, k, l = r["miller_index"]
variable_values = tuple((r[name]) for name in variables_present)
cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) +
variable_values)
cif_block.add_loop(cif_loop)
# Add the block
self._cif["dials"] = cif_block
# Print to file
with open(filename, "w") as fh:
self._cif.show(out=fh)
# Log
logger.info("Wrote reflections to %s" % filename)
def extract(file_name,
crystal_symmetry,
wavelength_id,
crystal_id,
show_details_if_error,
output_r_free_label,
merge_non_unique_under_symmetry,
map_to_asu,
remove_systematic_absences,
all_miller_arrays=None,
incompatible_flags_to_work_set=False,
ignore_bad_sigmas=False,
extend_flags=False,
return_as_miller_arrays=False,
log=sys.stdout):
import iotbx.cif
from cctbx import miller
if all_miller_arrays is None:
base_array_info = miller.array_info(
crystal_symmetry_from_file=crystal_symmetry)
all_miller_arrays = iotbx.cif.reader(
file_path=file_name).build_miller_arrays(
base_array_info=base_array_info)
if (len(all_miller_arrays) == 0):
raise Sorry(
"No data arrays were found in this CIF file. Please make " +
"sure that the file contains reflection data, rather than the refined "
+ "model.")
column_labels = set()
if (extend_flags):
map_to_asu = True
# TODO: is all_mille_arrays a dict ? If not change back
for (data_name, miller_arrays) in six.iteritems(all_miller_arrays):
for ma in miller_arrays.values():
other_symmetry = crystal_symmetry
try:
crystal_symmetry = other_symmetry.join_symmetry(
other_symmetry=ma.crystal_symmetry(), force=True)
except AssertionError as e:
str_e = str(e)
from six.moves import cStringIO as StringIO
s = StringIO()
if "Space group is incompatible with unit cell parameters." in str_e:
other_symmetry.show_summary(f=s)
ma.crystal_symmetry().show_summary(f=s)
str_e += "\n%s" % (s.getvalue())
raise Sorry(str_e)
else:
raise
if (crystal_symmetry.unit_cell() is None
or crystal_symmetry.space_group_info() is None):
raise Sorry(
"Crystal symmetry is not defined. Please use the --symmetry option."
)
mtz_object = iotbx.mtz.object() \
.set_title(title="phenix.cif_as_mtz") \
.set_space_group_info(space_group_info=crystal_symmetry.space_group_info())
unit_cell = crystal_symmetry.unit_cell()
mtz_crystals = {}
mtz_object.set_hkl_base(unit_cell=unit_cell)
from iotbx.reflection_file_utils import cif_status_flags_as_int_r_free_flags
# generate list of all reflections (for checking R-free flags)
from iotbx.reflection_file_utils import make_joined_set
all_arrays = []
for (data_name, miller_arrays) in six.iteritems(all_miller_arrays):
for ma in miller_arrays.values():
all_arrays.append(ma)
complete_set = make_joined_set(all_arrays)
if return_as_miller_arrays:
miller_array_list = []
current_i = -1
uc = None
for i, (data_name,
miller_arrays) in enumerate(six.iteritems(all_miller_arrays)):
for ma in miller_arrays.values():
#ma = ma.customized_copy(
# crystal_symmetry=crystal_symmetry).set_info(ma.info())
if ma._space_group_info is None:
ma._space_group_info = crystal_symmetry.space_group_info()
labels = ma.info().labels
label = get_label(miller_array=ma,
output_r_free_label=output_r_free_label)
if label is None:
print("Can't determine output label for %s - skipping." % \
ma.info().label_string(), file=log)
continue
elif label.startswith(output_r_free_label):
ma, _ = cif_status_flags_as_int_r_free_flags(
ma, test_flag_value="f")
if isinstance(ma.data(), flex.double):
data_int = ma.data().iround()
assert data_int.as_double().all_eq(ma.data())
ma = ma.customized_copy(data=data_int).set_info(ma.info())
elif (
(ma.is_xray_amplitude_array() or ma.is_xray_intensity_array())
and isinstance(ma.data(), flex.int)):
ma = ma.customized_copy(data=ma.data().as_double()).set_info(
ma.info())
crys_id = 0
for l in labels:
if 'crystal_id' in l:
crys_id = int(l.split('=')[-1])
break
if crys_id > 0 and crystal_id is None:
label += "%i" % crys_id
if crystal_id is not None and crys_id > 0 and crys_id != crystal_id:
continue
if ma.unit_cell(
) is not None: # use symmetry file on the command line if it's None
unit_cell = ma.unit_cell()
if crys_id not in mtz_crystals or \
(i > current_i and unit_cell is not None and uc is not None and unit_cell.parameters() != uc.parameters()):
# Ensure new mtz crystals are created if miller_array objects have different unit cells
# Can happen if there are more datasets in the same cif file, like MAD datasets
uc = unit_cell
current_i = i
# Use unique project and crystal names so that MtzGet() in cmtzlib.c picks up individual unit cells
mtz_crystals[crys_id] = (mtz_object.add_crystal(
name="crystal_%i" % i,
project_name="project_%i" % i,
unit_cell=uc), {})
crystal, datasets = mtz_crystals[crys_id]
w_id = 0
for l in labels:
if 'wavelength_id' in l:
w_id = int(l.split('=')[-1])
break
if wavelength_id is not None and w_id > 0 and w_id != wavelength_id:
continue
if w_id > 1 and wavelength_id is None:
if (label in column_labels):
label += "%i" % w_id
#print "label is", label
if w_id not in datasets:
wavelength = ma.info().wavelength
if (wavelength is None):
wavelength = 0
datasets[w_id] = crystal.add_dataset(name="dataset",
wavelength=wavelength)
dataset = datasets[w_id]
# if all sigmas for an array are set to zero either raise an error, or set sigmas to None
if ma.sigmas() is not None and (ma.sigmas()
== 0).count(False) == 0:
if ignore_bad_sigmas:
print("Warning: bad sigmas, setting sigmas to None.",
file=log)
ma.set_sigmas(None)
else:
raise Sorry("""Bad sigmas: all sigmas are equal to zero.
Add --ignore_bad_sigmas to command arguments to leave out sigmas from mtz file."""
)
if not ma.is_unique_set_under_symmetry():
if merge_non_unique_under_symmetry:
print("Warning: merging non-unique data", file=log)
if (label.startswith(output_r_free_label)
and incompatible_flags_to_work_set):
merging = ma.merge_equivalents(
incompatible_flags_replacement=0)
if merging.n_incompatible_flags > 0:
print("Warning: %i reflections were placed in the working set " \
"because of incompatible flags between equivalents." %(
merging.n_incompatible_flags), file=log)
else:
try:
merging = ma.merge_equivalents()
except Sorry as e:
if ("merge_equivalents_exact: incompatible"
in str(e)):
raise Sorry(
str(e) + " for %s" % ma.info().labels[-1] +
"\n" +
"Add --incompatible_flags_to_work_set to command line "
"arguments to place incompatible flags to working set."
)
raise
ma = merging.array().customized_copy(
crystal_symmetry=ma).set_info(ma.info())
elif return_as_miller_arrays: # allow non-unique set
pass
else:
n_all = ma.indices().size()
sel_unique = ma.unique_under_symmetry_selection()
sel_dup = ~flex.bool(n_all, sel_unique)
n_duplicate = sel_dup.count(True)
n_uus = sel_unique.size()
msg = (
"Miller indices not unique under symmetry: " + file_name + \
"(%d redundant indices out of %d)" % (n_all-n_uus, n_all) +
"Add --merge to command arguments to force merging data.")
if (show_details_if_error):
print(msg)
ma.show_comprehensive_summary(prefix=" ")
ma.map_to_asu().sort().show_array(prefix=" ")
raise Sorry(msg)
if (map_to_asu):
ma = ma.map_to_asu().set_info(ma.info())
if (remove_systematic_absences):
ma = ma.remove_systematic_absences()
if (label.startswith(output_r_free_label)
and complete_set is not None):
n_missing = len(complete_set.lone_set(other=ma).indices())
if (n_missing > 0):
if (extend_flags):
from cctbx import r_free_utils
# determine flag values
fvals = list(set(ma.data()))
print("fvals", fvals)
fval = None
if (len(fvals) == 1):
fval = fvals[0]
elif (len(fvals) == 2):
f1 = (ma.data()
== fvals[0]).count(True) / ma.data().size()
f2 = (ma.data()
== fvals[1]).count(True) / ma.data().size()
if (f1 < f2): fval = fvals[0]
else: fval = fvals[1]
elif (len(fvals) == 0):
fval = None
else:
fval = 0
if (not fval in fvals):
raise Sorry(
"Cannot determine free-R flag value.")
#
if (fval is not None):
ma = r_free_utils.extend_flags(
r_free_flags=ma,
test_flag_value=fval,
array_label=label,
complete_set=complete_set,
preserve_input_values=True,
allow_uniform_flags=True,
log=sys.stdout)
else:
ma = None
else:
libtbx.warn((
"%d reflections do not have R-free flags in the " +
"array '%s' - this may " +
"cause problems if you try to use the MTZ file for refinement "
+
"or map calculation. We recommend that you extend the flags "
+
"to cover all reflections (--extend_flags on the command line)."
) % (n_missing, label))
# Get rid of fake (0,0,0) reflection in some CIFs
if (ma is not None):
ma = ma.select_indices(indices=flex.miller_index(
((0, 0, 0), )),
negate=True).set_info(ma.info())
if return_as_miller_arrays:
miller_array_list.append(ma)
continue # don't make a dataset
dec = None
if ("FWT" in label):
dec = iotbx.mtz.ccp4_label_decorator()
column_types = None
if ("PHI" in label or "PHWT" in label) and (ma.is_real_array()):
column_types = "P"
elif (label.startswith("DANO") and ma.is_real_array()):
if (ma.sigmas() is not None):
column_types = "DQ"
else:
column_types = "D"
label_base = label
i = 1
while label in column_labels:
label = label_base + "-%i" % (i)
i += 1
if (ma is not None):
column_labels.add(label)
if ("FWT-1" in label): dec = None
dataset.add_miller_array(ma,
column_root_label=label,
label_decorator=dec,
column_types=column_types)
if return_as_miller_arrays:
return miller_array_list
else:
return mtz_object
def exercise_miller_arrays_as_cif_block():
from iotbx.cif import reader
cif_model = reader(input_string=cif_miller_array,
builder=cif.builders.cif_model_builder()).model()
ma_builder = cif.builders.miller_array_builder(cif_model['global'])
ma1 = ma_builder.arrays()['_refln_F_squared_meas']
mas_as_cif_block = cif.miller_arrays_as_cif_block(ma1,
array_type='meas',
format="corecif")
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1 - 1j] * ma1.size())),
array_type='calc')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1 - 2j] * ma1.size())),
column_names=['_refln_A_calc', '_refln_B_calc'])
for key in ('_refln_F_squared_meas', '_refln_F_squared_sigma',
'_refln_F_calc', '_refln_phase_calc', '_refln_A_calc',
'_refln_A_calc'):
assert (key in mas_as_cif_block.cif_block.keys()), key
#
mas_as_cif_block = cif.miller_arrays_as_cif_block(ma1,
array_type='meas',
format="mmcif")
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1 - 1j] * ma1.size())),
array_type='calc')
for key in ('_refln.F_squared_meas', '_refln.F_squared_sigma',
'_refln.F_calc', '_refln.phase_calc',
'_space_group_symop.operation_xyz', '_cell.length_a',
'_refln.index_h'):
assert key in mas_as_cif_block.cif_block.keys()
#
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1,
column_names=[
'_diffrn_refln_intensity_net', '_diffrn_refln_intensity_sigma'
],
miller_index_prefix='_diffrn_refln')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.std_string(ma1.size(), 'om')),
column_name='_diffrn_refln_intensity_u')
for key in ('_diffrn_refln_intensity_net', '_diffrn_refln_intensity_sigma',
'_diffrn_refln_intensity_u'):
assert key in list(mas_as_cif_block.cif_block.keys())
#
try:
reader(input_string=cif_global)
except CifParserError as e:
pass
else:
raise Exception_expected
cif_model = reader(input_string=cif_global, strict=False).model()
assert not show_diff(
str(cif_model), """\
data_1
_c 3
_d 4
""")
# exercise adding miller arrays with non-matching indices
cs = crystal.symmetry(unit_cell=uctbx.unit_cell((10, 10, 10, 90, 90, 90)),
space_group_info=sgtbx.space_group_info(symbol="P1"))
mi = flex.miller_index(((1, 0, 0), (1, 2, 3), (2, 3, 4)))
ms1 = miller.set(cs, mi)
ma1 = miller.array(ms1, data=flex.double((1, 2, 3)))
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1, column_name="_refln.F_meas_au")
ms2 = miller.set(cs, mi[:2])
ma2 = miller.array(ms2, data=flex.complex_double([1 - 2j] * ms2.size()))
mas_as_cif_block.add_miller_array(ma2,
column_names=("_refln.F_calc_au",
"_refln.phase_calc")),
ms3 = miller.set(cs, flex.miller_index(((1, 0, 0), (5, 6, 7), (2, 3, 4))))
ma3 = miller.array(ms3, data=flex.double((4, 5, 6)))
mas_as_cif_block.add_miller_array(ma3, column_name="_refln.F_squared_meas")
ms4 = miller.set(
cs,
flex.miller_index(
((1, 2, 3), (5, 6, 7), (1, 1, 1), (1, 0, 0), (2, 3, 4))))
ma4 = ms4.d_spacings()
mas_as_cif_block.add_miller_array(ma4, column_name="_refln.d_spacing")
# extract arrays from cif block and make sure we get back what we started with
arrays = cif.builders.miller_array_builder(
mas_as_cif_block.cif_block).arrays()
recycled_arrays = (arrays['_refln.F_meas_au'], arrays['_refln.F_calc_au'],
arrays['_refln.F_squared_meas'],
arrays['_refln.d_spacing'])
for orig, recycled in zip((ma1, ma2, ma3, ma4), recycled_arrays):
assert orig.size() == recycled.size()
recycled = recycled.customized_copy(
anomalous_flag=orig.anomalous_flag())
orig, recycled = orig.common_sets(recycled)
assert orig.indices().all_eq(recycled.indices())
assert approx_equal(orig.data(), recycled.data(), eps=1e-5)
#
cif_model = reader(input_string=r3adrsf,
builder=cif.builders.cif_model_builder()).model()
cs = cif.builders.crystal_symmetry_builder(
cif_model["r3adrsf"]).crystal_symmetry
ma_builder = cif.builders.miller_array_builder(
cif_model['r3adrAsf'],
base_array_info=miller.array_info(crystal_symmetry_from_file=cs))
miller_arrays = list(ma_builder.arrays().values())
assert len(miller_arrays) == 4
mas_as_cif_block = cif.miller_arrays_as_cif_block(
miller_arrays[0].map_to_asu(),
column_names=miller_arrays[0].info().labels,
format="corecif")
for array in miller_arrays[1:]:
labels = array.info().labels
if len(labels) > 1:
for label in labels:
if label.startswith("wavelength_id"):
labels.remove(label)
mas_as_cif_block.add_miller_array(array=array.map_to_asu(),
column_names=array.info().labels)
s = StringIO()
print(mas_as_cif_block.refln_loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_refln_index_h
_refln_index_k
_refln_index_l
_refln.crystal_id
_refln.scale_group_code
_refln.wavelength_id
_refln.pdbx_I_plus
_refln.pdbx_I_plus_sigma
_refln.pdbx_I_minus
_refln.pdbx_I_minus_sigma
-87 5 46 1 1 3 40.2 40.4 6.7 63.9
-87 5 45 1 1 3 47.8 29.7 35.1 30.5
-87 5 44 1 1 3 18.1 33.2 0.5 34.6
-87 5 43 1 1 3 6.1 45.4 12.9 51.6
-87 5 42 1 1 3 -6.6 45.6 -15.5 55.8
-87 7 37 1 1 3 6.3 43.4 ? ?
-87 7 36 1 1 3 -67.2 55.4 ? ?
-88 2 44 1 1 3 0 -1 35 38.5
-88 2 43 1 1 3 0 -1 57.4 41.5
-88 4 45 1 1 3 -1 46.1 -9.1 45.6
-88 4 44 1 1 3 -19.8 49.2 0.3 34.7
-88 6 44 1 1 3 -1.8 34.8 ? ?
""")
def _extract_flags(self, data_description = "R-free flags"):
r_free_flags, test_flag_value = None, None
params = self.parameters.r_free_flags
# Extract
if(not self.parameters.r_free_flags.generate):
try:
r_free_flags, test_flag_value = \
self.reflection_file_server.get_r_free_flags(
file_name = params.file_name,
label = params.label,
test_flag_value = params.test_flag_value,
disable_suitability_test = params.disable_suitability_test,
parameter_scope = "")
except reflection_file_utils.Sorry_No_array_of_the_required_type as e:
if(self.parameters.r_free_flags.generate is not None):
if(not self.keep_going):
self.err.append(explain_how_to_generate_array_of_r_free_flags())
self.err.append("Please try again.")
return None
r_free_flags, test_flag_value = None, None
else:
params.file_name = r_free_flags.info().source
params.label = r_free_flags.info().label_string()
params.test_flag_value = test_flag_value
msg = miller_array_symmetry_safety_check(
miller_array = r_free_flags,
data_description = data_description,
working_point_group = self.working_point_group)
if(msg is not None and not self.keep_going):
self.err.append(msg)
info = r_free_flags.info()
try:
processed = r_free_flags.regularize()
except RuntimeError as e:
self.err.append("Bad free-r flags:\n %s"%str(e))
return None
if (self.force_non_anomalous):
processed = processed.average_bijvoet_mates()
r_free_flags = processed.set_info(info)
# Generate or stop
if(r_free_flags is None):
if ((params.fraction is None) or
(params.lattice_symmetry_max_delta is None) or
(params.use_lattice_symmetry is None)):
msg = """
No R-free flags are available, but one or more parameters required to generate
new flags is undefined.
"""
self.err.append(msg)
return None
print("Generating a new array of R-free flags.", file=self.log)
print(file=self.log)
r_free_flags = self.f_obs.generate_r_free_flags(
fraction = params.fraction,
max_free = params.max_free,
lattice_symmetry_max_delta = params.lattice_symmetry_max_delta,
use_lattice_symmetry = params.use_lattice_symmetry,
use_dataman_shells = params.use_dataman_shells,
n_shells = params.n_shells
).set_info(miller.array_info(labels = ["R-free-flags"]))
params.label = r_free_flags.info().label_string()
params.test_flag_value = 1
# check if anomalous pairs are sound
if(r_free_flags is not None):
r_free_flags.deep_copy().as_non_anomalous_array()
# make sure flags match anomalous flag of data
if(self.raw_data.anomalous_flag() and not r_free_flags.anomalous_flag()):
info = r_free_flags.info()
observation_type = r_free_flags.observation_type()
r_free_flags = r_free_flags.generate_bijvoet_mates()
r_free_flags.set_observation_type(observation_type)
r_free_flags.set_info(info)
return r_free_flags
def make_cif_block(self, experiments, reflections):
"""Write the data to a cif block"""
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated,
all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
# Assumes you want to apply the lp and dqe corrections to sum and prf
# Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.revision_id"] = 1
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_entry.id"] = "DIALS"
# add software loop
mmcif_software_header = (
"_software.pdbx_ordinal",
"_software.citation_id",
"_software.name", # as defined at [1]
"_software.version",
"_software.type",
"_software.classification",
"_software.description",
)
mmcif_citations_header = (
"_citation.id",
"_citation.journal_abbrev",
"_citation.journal_volume",
"_citation.journal_issue",
"_citation.page_first",
"_citation.page_last",
"_citation.year",
"_citation.title",
)
software_loop = iotbx.cif.model.loop(header=mmcif_software_header)
citations_loop = iotbx.cif.model.loop(header=mmcif_citations_header)
software_loop.add_row((
1,
1,
"DIALS",
dials_version,
"package",
"data processing",
"Data processing and integration within the DIALS software package",
))
citations_loop.add_row((
1,
"Acta Cryst. D",
74,
2,
85,
97,
2018,
"DIALS: implementation and evaluation of a new integration package",
))
if "scale" in self.params.intensity:
software_loop.add_row((
2,
2,
"DIALS",
dials_version,
"program",
"data scaling",
"Data scaling and merging within the DIALS software package",
))
citations_loop.add_row((
2,
"Acta Cryst. D",
76,
4,
385,
399,
2020,
"Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling",
))
cif_block.add_loop(software_loop)
cif_block.add_loop(citations_loop)
# Hard coding X-ray
if self.params.mmcif.pdb_version == "v5_next":
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity).lower()
# FIXME finish metadata addition - detector and source details needed
# http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html
# Add source information;
# _diffrn_source.pdbx_wavelength_list = (list of wavelengths)
# _diffrn_source.source = (general class of source e.g. synchrotron)
# _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04)
wls = []
epochs = []
for exp in experiments:
wls.append(round(exp.beam.get_wavelength(), 5))
epochs.append(exp.scan.get_epochs()[0])
unique_wls = set(wls)
cif_block["_exptl_crystal.id"] = 1 # links to crystal_id
cif_block["_diffrn.id"] = 1 # links to diffrn_id
cif_block["_diffrn.crystal_id"] = 1
cif_block["_diffrn_source.diffrn_id"] = 1
cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join(
str(w) for w in unique_wls)
# Add detector information;
# _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc)
# _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd)
# _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M)
# One date is required, so if multiple just use the first date.
min_epoch = min(epochs)
date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch))
cif_block["_diffrn_detector.diffrn_id"] = 1
cif_block["_diffrn_detector.pdbx_collection_date"] = date_str
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
extra_items = {
"scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"),
"intensity.scale.value": (
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"%8.3f",
),
"intensity.scale.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"%8.3f",
),
"intensity.sum.value": (
"_pdbx_diffrn_unmerged_refln.intensity_sum",
"%8.3f",
),
"intensity.sum.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"%8.3f",
),
"intensity.prf.value": (
"_pdbx_diffrn_unmerged_refln.intensity_prf",
"%8.3f",
),
"intensity.prf.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"%8.3f",
),
"angle":
("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"),
"partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"),
}
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = flex.sqrt(
reflections["intensity.scale.variance"])
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"])
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = flex.sqrt(
reflections["intensity.sum.variance"])
variables_present.extend(
["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = flex.sqrt(
reflections["intensity.prf.variance"])
variables_present.extend(
["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (extra_items[name][0], )
self._fmt += " " + extra_items[name][1]
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set,
data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS",
source_type="reflection_tables"))
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
assert_is_not_unique_set_under_symmetry=False,
)
merged_block = iotbx.cif.model.block()
merged_block["_reflns.pdbx_ordinal"] = 1
merged_block["_reflns.pdbx_diffrn_id"] = 1
merged_block["_reflns.entry_id"] = "DIALS"
merged_data = result.as_cif_block()
merged_block.update(merged_data)
cif_block.update(merged_block)
# Write the crystal information
# if v5, that's all so return
if self.params.mmcif.pdb_version == "v5":
return cif_block
# continue if v5_next
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
))
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group()))
cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha,
beta, gamma, latt_type))
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
))
expid_to_scan_id = {
exp.identifier: i + 1
for i, exp in enumerate(experiments)
}
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row((
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
))
cif_block.add_loop(cif_loop)
_, _, _, _, z0, z1 = reflections["bbox"].parts()
h, k, l = [
hkl.iround()
for hkl in reflections["miller_index"].as_vec3_double().parts()
]
# make scan id consistent with header as defined above
scan_id = flex.int(reflections.size(), 0)
for id_ in reflections.experiment_identifiers().keys():
expid = reflections.experiment_identifiers()[id_]
sel = reflections["id"] == id_
scan_id.set_selected(sel, expid_to_scan_id[expid])
loop_values = [
flex.size_t_range(1,
len(reflections) + 1),
scan_id,
z0,
z1,
h,
k,
l,
] + [reflections[name] for name in variables_present]
cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values)))
cif_block.add_loop(cif_loop)
return cif_block
def get_solve_map (self) :
map_coeffs = self._convert_amplitudes_and_phases(f_label="FP",
phi_label="PHIB", fom_label="FOM", weighted=True)
from cctbx.miller import array_info
info = array_info(labels=["FWT","PHWT"])
return map_coeffs.set_info(info)
def extract_flags(self, data_description="R-free flags"):
r_free_flags, test_flag_value = None, None
params = self.parameters.r_free_flags
# Extract
if (not self.parameters.r_free_flags.generate):
try:
r_free_flags, test_flag_value = \
self.reflection_file_server.get_r_free_flags(
file_name = params.file_name,
label = params.label,
test_flag_value = params.test_flag_value,
disable_suitability_test = params.disable_suitability_test,
parameter_scope = self.flags_parameter_scope)
except reflection_file_utils.Sorry_No_array_of_the_required_type as e:
if (self.parameters.r_free_flags.generate is not None):
explain_how_to_generate_array_of_r_free_flags(
log=self.log,
flags_parameter_scope=self.flags_parameter_scope)
if (self.keep_going): return None
raise Sorry("Please try again.")
r_free_flags, test_flag_value = None, None
else:
params.file_name = r_free_flags.info().source
params.label = r_free_flags.info().label_string()
params.test_flag_value = test_flag_value
if ([self.working_point_group,
self.symmetry_safety_check].count(None) == 0):
miller_array_symmetry_safety_check(
miller_array=r_free_flags,
data_description=data_description,
working_point_group=self.working_point_group,
symmetry_safety_check=self.symmetry_safety_check,
log=self.log)
info = r_free_flags.info()
try:
processed = r_free_flags.regularize()
except RuntimeError as e:
raise Sorry("Bad free-r flags:\n %s" % str(e))
if (self.force_non_anomalous):
processed = processed.average_bijvoet_mates()
r_free_flags = processed.set_info(info)
# Generate or stop
if (r_free_flags is None):
if ((params.fraction is None)
or (params.lattice_symmetry_max_delta is None)
or (params.use_lattice_symmetry is None)):
raise Sorry(
"No R-free flags are available, but one or more " +
"parameters required to generate new flags is undefined.")
print("Generating a new array of R-free flags.", file=self.log)
print(file=self.log)
libtbx.call_back(
message="warn",
data=
"PHENIX will generate a new array of R-free flags. Please " +
"check to make sure that the input data do not already contain "
+
"an R-free set; if one is present, you should cancel this job and "
+
"disable generation of new flags. If the program you are running "
+
"outputs an MTZ file, you should be sure to use that file in all "
+ "future refinements.")
r_free_flags = self.f_obs.generate_r_free_flags(
fraction=params.fraction,
max_free=params.max_free,
lattice_symmetry_max_delta=params.lattice_symmetry_max_delta,
use_lattice_symmetry=params.use_lattice_symmetry,
use_dataman_shells=params.use_dataman_shells,
n_shells=params.n_shells).set_info(
miller.array_info(labels=["R-free-flags"]))
params.label = r_free_flags.info().label_string()
params.test_flag_value = 1
# check if anomalous pairs are sound
if (r_free_flags is not None):
r_free_flags.deep_copy().as_non_anomalous_array()
# make sure flags match anomalous flag of data
if (self.raw_data.anomalous_flag()
and not r_free_flags.anomalous_flag()):
info = r_free_flags.info()
observation_type = r_free_flags.observation_type()
r_free_flags = r_free_flags.generate_bijvoet_mates()
r_free_flags.set_observation_type(observation_type)
r_free_flags.set_info(info)
return r_free_flags