def test_remove_absences(inplace, hkl_index):
"""Test DataSet.remove_absences()"""
params = (34., 45., 98., 90., 90., 90.)
cell = gemmi.UnitCell(*params)
sg_1 = gemmi.SpaceGroup(1)
sg_19 = gemmi.SpaceGroup(19)
Hall = rs.utils.generate_reciprocal_asu(cell, sg_1, 5., anomalous=False)
h, k, l = Hall.T
absent = rs.utils.is_absent(Hall, sg_19)
ds = rs.DataSet({
'H': h,
'K': k,
'L': l,
'I': np.ones(len(h)),
},
spacegroup=sg_19,
cell=cell).infer_mtz_dtypes()
if hkl_index:
ds.set_index(['H', 'K', 'L'], inplace=True)
ds_test = ds.remove_absences(inplace=inplace)
ds_true = ds[~ds.label_absences().ABSENT]
assert len(ds_test) == len(Hall) - absent.sum()
assert np.array_equal(ds_test.get_hkls(), ds_true.get_hkls())
if inplace:
assert id(ds_test) == id(ds)
else:
assert id(ds_test) != id(ds)
def test_find_spacegroup(self):
self.assertEqual(gemmi.SpaceGroup('P21212').hm, 'P 21 21 2')
self.assertEqual(gemmi.find_spacegroup_by_name('P21').hm, 'P 1 21 1')
self.assertEqual(gemmi.find_spacegroup_by_name('P 2').hm, 'P 1 2 1')
def check_xhm(name, xhm):
self.assertEqual(gemmi.SpaceGroup(name).xhm(), xhm)
check_xhm('R 3 2', 'R 3 2:H')
check_xhm('R 3 2:h', 'R 3 2:H')
check_xhm('R32:H', 'R 3 2:H')
check_xhm('H32', 'R 3 2:H')
check_xhm('R 3 2:R', 'R 3 2:R')
check_xhm('P6', 'P 6')
check_xhm('P 6', 'P 6')
check_xhm('P65', 'P 65')
check_xhm('I1211', 'I 1 21 1')
check_xhm('Aem2', 'A b m 2')
check_xhm('C c c e', 'C c c a:1')
check_xhm('i2', 'I 1 2 1')
check_xhm('I 41/A', 'I 41/a:1')
check_xhm('I -4 2 D', 'I -4 2 d')
check_xhm('P 1 21/c 1', 'P 1 21/c 1')
check_xhm('P 21 21 2 A', 'P 21212(a)')
check_xhm('B 2', 'B 1 1 2')
self.assertRaises(ValueError, gemmi.SpaceGroup, 'i3')
self.assertEqual(gemmi.find_spacegroup_by_number(5).hm, 'C 1 2 1')
self.assertEqual(gemmi.SpaceGroup(4005).hm, 'I 1 2 1')
self.assertIsNone(gemmi.find_spacegroup_by_name('abc'))
def test_reflection_properties(self):
sg = gemmi.SpaceGroup('I 1 2 1')
gops = sg.operations()
self.assertTrue(gops.is_reflection_centric([3,0,3]))
self.assertFalse(gops.is_reflection_centric([3,3,3]))
self.assertEqual(gops.epsilon_factor([3,0,3]), 2)
self.assertEqual(gops.epsilon_factor([0,3,0]), 4)
self.assertFalse(gops.is_systematically_absent([1,2,3]))
self.assertTrue(gops.is_systematically_absent([1,2,4]))
sg = gemmi.SpaceGroup('F 4 3 2')
gops = sg.operations()
self.assertEqual(gops.epsilon_factor([2,0,0]), 16)
self.assertEqual(gops.epsilon_factor([3,3,3]), 12)
self.assertEqual(gops.epsilon_factor_without_centering([2,0,0]), 4)
def sg_symbol_to_sg_no(symbol='Fm-3m'):
symbol = str(symbol)
symbol_origin = symbol
sg_no = 1
try:
sg_no = gemmi.SpaceGroup(symbol).number
except:
pass
if sg_no == 0 and symbol[-2:] == ':2':
symbol = symbol.replace(':2', '')
sg_no = gemmi.SpaceGroup(symbol).number
if sg_no == 0:
print(symbol_origin, 'not recognised, set to P-1.')
sg_no = 2
return sg_no
def from_parser(cls, parser):
dmin = 0. if parser.dmin is None else parser.dmin
pe_keys = parser.positional_encoding_keys
if pe_keys is not None:
pe_keys = pe_keys.split(',')
spacegroups = None
if parser.spacegroups is not None:
spacegroups = [
gemmi.SpaceGroup(i) for i in parser.spacegroups.split(",")
]
if len(spacegroups) == 1:
spacegroups = [
spacegroups[0] for i in range(len(parser.reflection_files))
]
elif len(spacegroups) != len(parser.reflection_files):
raise ValueError(
"Multiple values provided for --spacegroups=, but the number of provided values does not match the number of reflection files. "
"Either provide a single spacegroup or one per reflection file as a comma-separated list. "
)
return cls(
parser.intensity_key,
None, #<-- uncertainty key has to match {SIG,Sig}intensity_key
parser.image_key,
parser.metadata_keys.split(','),
parser.separate_files,
parser.anomalous,
dmin,
parser.isigi_cutoff,
pe_keys,
parser.positional_encoding_frequencies,
spacegroups,
)
def test_read_hkl(IOtest_hkl, spacegroup, cell):
"""
Test rs.read_precognition() with a .hkl file
"""
result = rs.read_precognition(IOtest_hkl, spacegroup=spacegroup, cell=cell)
# Check main DataSet features
assert isinstance(result, rs.DataSet)
assert isinstance(result["F"], rs.DataSeries)
assert result.columns.to_list() == ["F", "SigF"]
assert list(result.index.names) == ["H", "K", "L"]
# Check _metadata
assert result._index_dtypes == {"H": "HKL", "K": "HKL", "L": "HKL"}
if spacegroup:
assert result.spacegroup.xhm() == gemmi.SpaceGroup(spacegroup).xhm()
else:
assert result.spacegroup is None
if cell:
assert result.cell.a == cell[0]
assert result.cell.b == cell[1]
assert result.cell.c == cell[2]
assert result.cell.alpha == cell[3]
assert result.cell.beta == cell[4]
assert result.cell.gamma == cell[5]
else:
assert result.cell is None
def test_phase_restrictions(sgtbx_by_xhm):
"""
Test rs.utils.phase_restrictions against reference data generated
from sgtbx.
"""
xhm = sgtbx_by_xhm[0]
reference = sgtbx_by_xhm[1]
# Convert reference phase restrictions to canonical output
H = reference[['h', 'k', 'l']].to_numpy()
sg = gemmi.SpaceGroup(xhm)
ref_restrictions = []
for h, entry in zip(H, reference["phase_restrictions"].to_list()):
if entry == "None" or rs.utils.is_absent([h], sg)[0]:
ref_restrictions.append([])
else:
phases = np.array(entry.split(","), dtype=float)
phases = rs.utils.canonicalize_phases(np.rad2deg(phases))
phases.sort()
ref_restrictions.append(phases)
restrictions = rs.utils.get_phase_restrictions(H, sg)
assert len(ref_restrictions) == len(restrictions)
for ref, test in zip(ref_restrictions, restrictions):
if ref is []:
assert ref == test
else:
assert np.allclose(np.sin(np.deg2rad(ref)),
np.sin(np.deg2rad(np.array(test))))
def test_short_name(self):
for (longer, shorter) in [('P 21 2 21', 'P21221'),
('P 1 2 1', 'P2'),
('P 1', 'P1'),
('R 3 2:R', 'R32'),
('R 3 2:H', 'H32')]:
self.assertEqual(gemmi.SpaceGroup(longer).short_name(), shorter)
def test_hkl_to_observed(sgtbx_by_xhm):
xhm = sgtbx_by_xhm[0]
reference = sgtbx_by_xhm[1]
H = reference[['h', 'k', 'l']].to_numpy()
sg = gemmi.SpaceGroup(xhm)
Hasu, isym = rs.utils.hkl_to_asu(H, sg)
H_observed = rs.utils.hkl_to_observed(Hasu, isym, sg)
assert np.array_equal(H, H_observed)
def sg_symbol_to_sg_no(symbol='Fm-3m'):
"""Find the spacegroup number of a spacegroup symbol.
if error, return 1 (P1)."""
symbol = str(symbol)
symbol_origin = symbol
sg_no = 1
try:
sg_no = gemmi.SpaceGroup(symbol).number
except:
pass
if sg_no == 0 and symbol[-2:] == ':2':
symbol = symbol.replace(':2', '')
sg_no = gemmi.SpaceGroup(symbol).number
elif sg_no == 0 and symbol[-2:] != ':2':
print(symbol_origin, 'not recognised, set to "P1".')
sg_no = 1
return sg_no
def spacegroup(self, val):
# GH#18: Type-checking for supported input types
if isinstance(val, gemmi.SpaceGroup) or (val is None):
self._spacegroup = val
elif isinstance(val, (str, int)):
self._spacegroup = gemmi.SpaceGroup(val)
else:
raise ValueError(
f"Cannot construct gemmi.SpaceGroup from value: {val}")
def cell_and_spacegroups():
data = [
((10., 20., 30., 90., 80., 75.), 'P 1'),
((30., 50., 80., 90., 100., 90.), 'P 1 21 1'),
((10., 20., 30., 90., 90., 90.), 'P 21 21 21'),
((89., 89., 105., 90., 90., 120.), 'P 31 2 1'),
((30., 30., 30., 90., 90., 120.), 'R 32'),
]
return [(gemmi.UnitCell(*i), gemmi.SpaceGroup(j)) for i, j in data]
def test_is_centric(sgtbx_by_xhm, sg_type):
"""
Test rs.utils.is_centric using reference data generated from sgtbx.
"""
xhm = sgtbx_by_xhm[0]
reference = sgtbx_by_xhm[1]
# Drop [0, 0, 0] from test data
reference = reference.drop(reference.query("h==0 and k==0 and l==0").index)
H = reference[['h', 'k', 'l']].to_numpy()
ref_centric = reference['is_centric'].to_numpy()
if sg_type is gemmi.SpaceGroup:
sg = gemmi.SpaceGroup(xhm)
elif sg_type is gemmi.GroupOps:
sg = gemmi.SpaceGroup(xhm).operations()
centric = rs.utils.is_centric(H, sg)
assert np.array_equal(centric, ref_centric)
def test_pickling(self):
try:
import cPickle as pickle # Use cPickle on Python 2.7
except ImportError:
import pickle
sg = gemmi.SpaceGroup("P 31 2 1")
pkl_string = pickle.dumps(sg, protocol=pickle.HIGHEST_PROTOCOL)
result = pickle.loads(pkl_string)
self.assertTrue(isinstance(result, gemmi.SpaceGroup))
self.assertEqual(sg.xhm(), result.xhm())
def test_spacegroup(data_fmodel, spacegroup):
if spacegroup != 1.2 and spacegroup != "R 3:blah":
data_fmodel.spacegroup = spacegroup
if isinstance(spacegroup, (str, int)):
assert data_fmodel.spacegroup.xhm() == gemmi.SpaceGroup(
spacegroup).xhm()
else:
assert data_fmodel.spacegroup == spacegroup
else:
with pytest.raises(ValueError):
data_fmodel.spacegroup = spacegroup
def test_in_asu(sgtbx_by_xhm):
"""
Test rs.utils.in_asu using reference data generated from sgtbx
"""
xhm = sgtbx_by_xhm[0]
reference = sgtbx_by_xhm[1]
H = reference[['h', 'k', 'l']].to_numpy()
sg = gemmi.SpaceGroup(xhm)
in_asu = rs.utils.in_asu(H, sg)
ref_in_asu = reference['in_asu'].to_numpy()
assert np.array_equal(in_asu, ref_in_asu)
def test_systematic_absences(sgtbx_by_xhm):
"""
Test rs.utils.hkl_is_absent() using reference data generated from
sgtbx
"""
xhm = sgtbx_by_xhm[0]
reference = sgtbx_by_xhm[1]
H = reference[['h', 'k', 'l']].to_numpy()
groupops = gemmi.SpaceGroup(xhm)
absent = rs.utils.is_absent(H, groupops)
reference_absent = reference['is_absent'].to_numpy()
assert np.array_equal(absent, reference_absent)
def test_reduction(self):
cell = gemmi.UnitCell(687.9, 687.9, 1933.3, 90.0, 90.0, 90.0)
sg = gemmi.SpaceGroup('I 4 2 2')
gv = gemmi.GruberVector(cell, sg)
gv.niggli_reduce()
self.assertTrue(gv.is_niggli())
self.assertTrue(gv.is_buerger())
self.assertTrue(gv.is_normalized())
p = cell.a
q = 1082.134662368783
t = 108.5325886
par = (p * p, p * p, q * q, -p * p, -p * p, 0)
assert_almost_equal_seq(self, gv.parameters, par)
self.assertTrue(gv.cell_parameters(), (p, p, q, t, t, 0))
def test_pickling(self):
import os
try:
import cPickle as pickle # Use cPickle on Python 2.7
except ImportError:
import pickle
sg = gemmi.SpaceGroup("P 31 2 1")
with open("temp.pkl", "wb") as temp:
pickle.dump(sg, temp)
with open("temp.pkl", "rb") as temp:
result = pickle.load(temp)
self.assertTrue(isinstance(result, gemmi.SpaceGroup))
self.assertEqual(sg.xhm(), result.xhm())
os.remove("temp.pkl")
def test_write_ccp4_map(realmap):
"""Test rs.io.write_ccp4_map()"""
mapfile = tempfile.NamedTemporaryFile(suffix=".map")
sg = gemmi.SpaceGroup(1)
cell = gemmi.UnitCell(10., 20., 30., 90., 90., 90.)
if not isinstance(realmap, np.ndarray) or not (realmap.ndim == 3):
with pytest.raises(ValueError):
rs.io.write_ccp4_map(realmap, mapfile.name, cell, sg)
mapfile.close()
return
rs.io.write_ccp4_map(realmap, mapfile.name, cell, sg)
assert exists(mapfile.name)
mapfile.close()
def test_hkl_to_observed_phase(data_fmodel_P1):
"""Test DataSet.hkl_to_observed() handling of phase"""
data_fmodel_P1.spacegroup = gemmi.SpaceGroup(96)
asu = data_fmodel_P1.hkl_to_asu()
result = asu.hkl_to_observed()
# Check phases have been canonicalized
assert (result["PHIFMODEL"] >= -180.).all()
assert (result["PHIFMODEL"] <= 180.).all()
# Compare as complex structure factors
original = rs.utils.to_structurefactor(data_fmodel_P1.FMODEL,
data_fmodel_P1.PHIFMODEL)
new = rs.utils.to_structurefactor(result.FMODEL, result.PHIFMODEL)
assert np.allclose(new, original)
def __init__(self, s0, cell, R, lam_min, lam_max, dmin, spacegroup='1'):
"""
Parameters
----------
s0 : array
a 3 vector indicating the direction of the incoming beam wavevector.
This can be any length, it will be unit normalized in the constructor.
cell : iterable or gemmi.UnitCell
A tuple or list of unit cell params (a, b, c, alpha, beta, gamma) or a gemmi.UnitCell object
R : array
A 3x3 rotation matrix corresponding to the crystal orientation for the frame.
lam_min : float
The lower end of the wavelength range of the beam.
lam_max : float
The upper end of the wavelength range of the beam.
dmin : float
The maximum resolution of the model
spacegroup : gemmi.SpaceGroup (optional)
Anything that the gemmi.SpaceGroup constructor understands.
"""
if not isinstance(cell, gemmi.UnitCell):
cell = gemmi.UnitCell(*cell)
self.cell = cell
if not isinstance(spacegroup, gemmi.SpaceGroup):
spacegroup = gemmi.SpaceGroup(spacegroup)
self.spacegroup = spacegroup
self.R = R
self.lam_min = lam_min
self.lam_max = lam_max
self.dmin = dmin
self.B = np.array(self.cell.fractionalization_matrix).T
# self.s{0,1} are dynamically masked by their outlier status
self.s0 = s0 / np.linalg.norm(s0)
# Initialize the full reciprocal grid
hmax, kmax, lmax = self.cell.get_hkl_limits(dmin)
Hall = np.mgrid[-hmax:hmax + 1:1., -kmax:kmax + 1:1.,
-lmax:lmax + 1:1., ].reshape((3, -1)).T
Hall = Hall[np.any(Hall != 0, axis=1)]
d = cell.calculate_d_array(Hall)
Hall = Hall[d >= dmin]
self.Hall = Hall
def expand_to_p1(self):
"""
Generates all symmetrically equivalent reflections. The spacegroup
symmetry is set to P1.
Returns
-------
DataSet
"""
if not self.merged:
raise ValueError(
"This function is only applicable for merged DataSets")
groupops = self.spacegroup.operations()
p1 = rs.concat([self.apply_symop(op) for op in groupops])
p1.spacegroup = gemmi.SpaceGroup(1)
p1.reset_index(inplace=True)
p1.drop_duplicates(subset=["H", "K", "L"], inplace=True)
p1.set_index(["H", "K", "L"], inplace=True)
return p1
def test_read_csv(IOtest_mtz, sep, spacegroup, cell, merged, infer_mtz_dtypes):
"""Test rs.read_csv()"""
csvfile = tempfile.NamedTemporaryFile(suffix=".csv")
expected = rs.read_mtz(IOtest_mtz)
expected.to_csv(csvfile.name, sep=sep)
result = rs.read_csv(csvfile.name,
spacegroup=spacegroup,
cell=cell,
merged=merged,
infer_mtz_dtypes=infer_mtz_dtypes,
sep=sep)
print(result)
result.set_index(["H", "K", "L"], inplace=True)
csvfile.close()
if spacegroup:
if isinstance(spacegroup, gemmi.SpaceGroup):
assert result.spacegroup.xhm() == spacegroup.xhm()
else:
assert result.spacegroup.xhm() == gemmi.SpaceGroup(
spacegroup).xhm()
else:
assert result.spacegroup is None
if cell:
if isinstance(cell, gemmi.UnitCell):
assert result.cell.a == cell.a
else:
assert result.cell.a == cell[0]
else:
assert result.cell is None
assert result.merged == merged
if infer_mtz_dtypes:
assert_frame_equal(result, expected)
else:
assert_frame_equal(result,
expected,
check_dtype=False,
check_index_type=False)
def test_read_ii(IOtest_ii, IOtest_log, spacegroup, cell, log):
"""
Test rs.read_precognition() with a .ii file
"""
# Hacky way to parametrize a pytest fixture
if log == "log":
log = IOtest_log
result = rs.read_precognition(IOtest_ii,
spacegroup=spacegroup,
cell=cell,
logfile=log)
# Check main DataSet features
assert isinstance(result, rs.DataSet)
assert isinstance(result["I"], rs.DataSeries)
assert len(result.columns) == 7
assert list(result.index.names) == ["H", "K", "L"]
# Check _metadata
assert result._index_dtypes == {"H": "HKL", "K": "HKL", "L": "HKL"}
if spacegroup:
assert result.spacegroup.xhm() == gemmi.SpaceGroup(spacegroup).xhm()
else:
assert result.spacegroup is None
if log:
assert result.cell.a == 34.4660
assert result.cell.b == 45.6000
assert result.cell.c == 99.5850
assert result.cell.alpha == 90.0
assert result.cell.beta == 90.0
assert result.cell.gamma == 90.0
elif cell:
assert result.cell.a == cell[0]
assert result.cell.b == cell[1]
assert result.cell.c == cell[2]
assert result.cell.alpha == cell[3]
assert result.cell.beta == cell[4]
assert result.cell.gamma == cell[5]
else:
assert result.cell is None
def __setstate__(self, state):
data = state["data"]
self.grid = gemmi.FloatGrid(
data.shape[0],
data.shape[1],
data.shape[2],
)
spacegroup = state["spacegroup"]
self.grid.spacegroup = gemmi.SpaceGroup(spacegroup)
unit_cell = state["unit_cell"]
self.grid.unit_cell = gemmi.UnitCell(
unit_cell[0],
unit_cell[1],
unit_cell[2],
unit_cell[3],
unit_cell[4],
unit_cell[5],
)
for index, val in np.ndenumerate(data):
self.grid.set_value(index[0], index[1], index[2], data[index])
def expand_to_p1(self):
"""
Generates all symmetrically equivalent reflections. The spacegroup
symmetry is set to P1.
Returns
-------
DataSet
"""
if not self.merged:
raise ValueError(
"This function is only applicable for merged DataSets")
if not in_asu(self.get_hkls(), spacegroup=self.spacegroup).all():
raise ValueError(
"This function is only applicable for reflection data in the reciprocal ASU and anomalous data in a two-column (unstacked) format"
)
p1 = rs.DataSet(spacegroup=self.spacegroup, cell=self.cell)
# Get all symops, in ascending order by ISYM
groupops = self.spacegroup.operations()
allops = [op for op in groupops for op in (op, op.negated())]
# Apply each symop and drop duplicates with higher ISYM
for isym, op in enumerate(allops, 1):
ds = self.copy()
ds["M/ISYM"] = isym
ds["M/ISYM"] = ds["M/ISYM"].astype("M/ISYM")
p1 = p1.append(ds.hkl_to_observed(m_isym="M/ISYM"))
p1.drop_duplicates(subset=["H", "K", "L"], inplace=True)
# Restrict to p1 ASU
p1.spacegroup = gemmi.SpaceGroup(1)
p1 = p1.loc[in_asu(p1.get_hkls(), spacegroup=p1.spacegroup)]
return p1
unit_cell2 = cryst2.get_unit_cell()
refls2 = reflection_table.from_file(refl_file2)
# Remove reflections not used in refinement
refls2 = refls2.select(refls2.get_flags(refls2.flags.used_in_refinement))
print('generating DIALS dataframes')
dials_df1 = rs.DataSet(
{
'X': refls1['xyzobs.px.value'].parts()[0].as_numpy_array(),
'Y': refls1['xyzobs.px.value'].parts()[1].as_numpy_array(),
'Wavelength': refls1['Wavelength'].as_numpy_array(),
'BATCH': refls1['imageset_id'].as_numpy_array(),
},
cell=gemmi.UnitCell(*unit_cell1.parameters()),
spacegroup=gemmi.SpaceGroup(cryst1.get_space_group().type(
).universal_hermann_mauguin_symbol())).infer_mtz_dtypes()
dials_df2 = rs.DataSet(
{
'X': refls2['xyzobs.px.value'].parts()[0].as_numpy_array(),
'Y': refls2['xyzobs.px.value'].parts()[1].as_numpy_array(),
'Wavelength': refls2['Wavelength'].as_numpy_array(),
'BATCH': refls2['imageset_id'].as_numpy_array(),
},
cell=gemmi.UnitCell(*unit_cell2.parameters()),
spacegroup=gemmi.SpaceGroup(cryst2.get_space_group().type(
).universal_hermann_mauguin_symbol())).infer_mtz_dtypes()
print('initializing metrics')
nspots = np.zeros(len(elist2))
nmatch = np.zeros(len(elist2))
import pytest
import reciprocalspaceship as rs
import gemmi
@pytest.mark.parametrize("check_isomorphous", [True, False])
@pytest.mark.parametrize("sg", [gemmi.SpaceGroup(19), gemmi.SpaceGroup(96)])
@pytest.mark.parametrize("ignore_index", [True, False])
def test_concat(data_fmodel, check_isomorphous, sg, ignore_index):
"""
Test whether attributes of DataSet are preserved through calls to
pd.concat()
"""
other = data_fmodel.copy(deep=True)
other.spacegroup = sg
if check_isomorphous and sg.number == 19:
with pytest.raises(ValueError):
result = rs.concat([data_fmodel, other],
ignore_index=ignore_index,
check_isomorphous=check_isomorphous)
else:
result = rs.concat([data_fmodel, other],
ignore_index=ignore_index,
check_isomorphous=check_isomorphous)
assert isinstance(result, rs.DataSet)
assert len(result) == len(data_fmodel) * 2
if ignore_index:
assert result._index_dtypes == {}
for attr in data_fmodel._metadata:
if attr == "_index_dtypes":
continue
def fact(name):
return gemmi.SpaceGroup(name).operations().find_grid_factors()
