def test_hklmapping_roundtrip_phase(mtz_by_spacegroup): """ Test roundtrip of DataSet.hkl_to_asu() and DataSet.hkl_to_observed() preserve phases """ ref = rs.read_mtz(mtz_by_spacegroup) expected = rs.read_mtz(mtz_by_spacegroup[:-4] + "_p1.mtz") expected["sf"] = expected.to_structurefactor("FMODEL", "PHIFMODEL") expected.spacegroup = ref.spacegroup # Roundtrip temp = expected.hkl_to_asu() result = temp.hkl_to_observed() # Check indices assert_index_equal(result.index, expected.index) # Confirm phase changes are applied by comparing as complex structure factors expected_sf = expected.loc[result.index].to_structurefactor("FMODEL", "PHIFMODEL") result_sf = result.to_structurefactor("FMODEL", "PHIFMODEL") assert np.allclose(np.abs(result_sf), np.abs(expected_sf)) assert np.allclose(result_sf, expected_sf) # Confirm phase changes were applied to complex structure factors in DataSet assert np.allclose(np.abs(result["sf"]), np.abs(expected.loc[result.index, "sf"])) assert np.allclose(result["sf"], expected.loc[result.index, "sf"])
def test_hkl_to_asu(mtz_by_spacegroup, inplace, reset_index, anomalous): """Test DataSet.hkl_to_asu() for common spacegroups""" expected = rs.read_mtz(mtz_by_spacegroup) p1 = rs.read_mtz(mtz_by_spacegroup[:-4] + "_p1.mtz") p1.spacegroup = expected.spacegroup # Add complex structure factors p1["sf"] = p1.to_structurefactor("FMODEL", "PHIFMODEL") expected["sf"] = expected.to_structurefactor("FMODEL", "PHIFMODEL") if reset_index: p1.reset_index(inplace=True) result = p1.hkl_to_asu(inplace=inplace, anomalous=anomalous) if reset_index: result.set_index(["H", "K", "L"], inplace=True) # Confirm inplace if inplace: assert id(result) == id(p1) else: assert id(result) != id(p1) # Confirm centric reflections are always in +ASU assert len(expected.centrics.index.difference(result.centrics.index)) == 0 assert len(result.centrics.index.difference(expected.centrics.index)) == 0 # If anomalous=True, acentric reflections were mapped to the Friedel-minus ASU. # To test these reflections against `expected` we will map them back to the # Friedel-plus ASU # Note: # - `result` no longer has a unique MultiIndex after this if anomalous: result.reset_index(inplace=True) acentric = ~result.label_centrics()["CENTRIC"] friedel_minus = result["M/ISYM"] % 2 == 0 result[friedel_minus & acentric] = result[friedel_minus & acentric].apply_symop( "-x,-y,-z" ) result.set_index(["H", "K", "L"], inplace=True) assert len(result.index.difference(expected.index)) == 0 assert len(expected.index.difference(result.index)) == 0 # Confirm structure factor amplitudes are always unchanged assert np.allclose(expected.loc[result.index, "FMODEL"].to_numpy(), result["FMODEL"].to_numpy()) # Confirm phase changes are applied by comparing complex structure factors expected_sf = expected.loc[result.index].to_structurefactor("FMODEL", "PHIFMODEL") result_sf = result.to_structurefactor("FMODEL", "PHIFMODEL") assert np.allclose(result_sf, expected_sf) # Confirm phase changes were applied to complex structure factors in DataSet assert np.allclose(np.abs(result["sf"]), np.abs(expected.loc[result.index, "sf"])) assert np.allclose(result["sf"], expected.loc[result.index, "sf"])
def test_expand_to_p1(mtz_by_spacegroup): """Test DataSet.expand_to_p1() for common spacegroups""" x = rs.read_mtz(mtz_by_spacegroup) expected = rs.read_mtz(mtz_by_spacegroup[:-4] + '_p1.mtz') expected.sort_index(inplace=True) result = x.expand_to_p1() result.sort_index(inplace=True) expected_sf = expected.to_structurefactor("FMODEL", "PHIFMODEL") result_sf = result.to_structurefactor("FMODEL", "PHIFMODEL") assert_index_equal(result.index, expected.index) assert np.allclose(result_sf.to_numpy(), expected_sf.to_numpy(), rtol=1e-4)
def test_hkl_to_asu(mtz_by_spacegroup, inplace, reset_index, anomalous): """Test DataSet.hkl_to_asu() for common spacegroups""" x = rs.read_mtz(mtz_by_spacegroup) y = rs.read_mtz(mtz_by_spacegroup[:-4] + "_p1.mtz") y.spacegroup = x.spacegroup if reset_index: y.reset_index(inplace=True) yasu = y.hkl_to_asu(inplace=inplace, anomalous=anomalous) if reset_index: yasu.set_index(["H", "K", "L"], inplace=True) # Confirm centric reflections are always in +ASU expected_centric = x.loc[x.label_centrics()["CENTRIC"]] result_centric = yasu.loc[yasu.label_centrics()["CENTRIC"]] assert len(expected_centric.index.difference(result_centric.index)) == 0 assert len(result_centric.index.difference(expected_centric.index)) == 0 # If anomalous=True, confirm acentric reflections were in +/- ASU if anomalous: yasu.reset_index(inplace=True) acentric = ~yasu.label_centrics()["CENTRIC"] friedel_minus = yasu["M/ISYM"] % 2 == 0 yasu[friedel_minus & acentric] = yasu[friedel_minus & acentric].apply_symop("-x,-y,-z") yasu.set_index(["H", "K", "L"], inplace=True) assert len(yasu.index.difference(x.index)) == 0 assert len(x.index.difference(yasu.index)) == 0 # Confirm structure factor amplitudes are always unchanged Fx = x.loc[yasu.index, "FMODEL"].values.astype(float) Fyasu = yasu["FMODEL"].values.astype(float) assert np.allclose(Fx, Fyasu) # Confirm phase changes are applied Phx = x.loc[yasu.index, "PHIFMODEL"].values.astype(float) Phyasu = yasu["PHIFMODEL"].values.astype(float) Sx = Fx * np.exp(1j * np.deg2rad(Phx)) Syasu = Fyasu * np.exp(1j * np.deg2rad(Phyasu)) assert np.allclose(Sx, Syasu, rtol=1e-3) # Confirm inplace if inplace: assert id(yasu) == id(y) else: assert id(yasu) != id(y)
def load_dataset(datapath): """ Load dataset at given datapath. Datapath is expected to be a list of directories to follow. """ inFN = abspath(join(dirname(__file__), datapath)) return rs.read_mtz(inFN)
def test_write(self): datadir = join(abspath(dirname(__file__)), '../data/fmodel') data = rs.read_mtz(join(datadir, '9LYZ.mtz')) temp = tempfile.NamedTemporaryFile(suffix=".mtz") # Missing cell should raise AttributeError data_missingcell = data.copy() data_missingcell.cell = None with self.assertRaises(AttributeError): data_missingcell.write_mtz(temp.name) # Missing spacegroup should raise AttributeError data_missingsg = data.copy() data_missingsg.spacegroup = None with self.assertRaises(AttributeError): data_missingsg.write_mtz(temp.name) # Writing MTZ should produce a file data.write_mtz(temp.name) self.assertTrue(exists(temp.name)) temp.close() # Having a non-MTZType should raise AttributeError, unless flag # is set temp = tempfile.NamedTemporaryFile(suffix=".mtz") data["nonMTZ"] = 1 with self.assertRaises(ValueError): data.write_mtz(temp.name) data.write_mtz(temp.name, skip_problem_mtztypes=True) self.assertTrue(exists(temp.name)) temp.close() return
def test_copy_rfree(self): datadir = join(abspath(dirname(__file__)), '../data/fmodel') data = rs.read_mtz(join(datadir, '9LYZ.mtz')) data_rfree = rs.utils.add_rfree(data, inplace=False) # Test copy of R-free to copy of data rfree = rs.utils.copy_rfree(data, data_rfree, inplace=False) self.assertFalse(id(data) == id(rfree)) self.assertFalse("R-free-flags" in data.columns) self.assertTrue("R-free-flags" in rfree.columns) self.assertTrue( np.array_equal(rfree["R-free-flags"].values, data_rfree["R-free-flags"].values)) # Test copy of R-free inplace rfree = rs.utils.copy_rfree(data, data_rfree, inplace=True) self.assertTrue(id(data) == id(rfree)) self.assertTrue("R-free-flags" in data.columns) self.assertTrue("R-free-flags" in rfree.columns) self.assertTrue( np.array_equal(rfree["R-free-flags"].values, data_rfree["R-free-flags"].values)) return
def main(): # Parse commandline arguments parser = parse_arguments() args = parser.parse_args() if len(args.mtz) == 1: mtz = rs.read_mtz(args.mtz[0]) summarize(mtz, args.precision) else: mtzs = dict(zip(args.mtz, map(rs.read_mtz, args.mtz))) for key, value in mtzs.items(): print(f"MTZ file: {key}\n") summarize(value, args.precision) print(f"{'-'*50}") # Begin IPython shell if args.embed: from IPython import embed bold = '\033[1m' end = '\033[0m' if "mtzs" in locals(): header = f"rs.DataSets stored in {bold}mtzs{end} dictionary" else: header = f"rs.DataSet stored as {bold}mtz{end}" print() embed(colors='neutral', header=header) return
def ref_hewl(): """ Load phenix.french_wilson output for data_hewl """ datapath = ["..", "data", "algorithms", "phenix_fw_ref.mtz"] inFN = abspath(join(dirname(__file__), *datapath)) mtz = rs.read_mtz(inFN) return mtz
def hewl_unmerged(): """ Load HEWL SSAD data from APS 24IDC, scaled unmerged data from AIMLESS """ datapath = ["..", "data", "algorithms", "HEWL_unmerged.mtz"] inFN = abspath(join(dirname(__file__), *datapath)) mtz = rs.read_mtz(inFN) return mtz
def base_test_separate(flags, filenames): with TemporaryDirectory() as td: out = td + '/out' command = flags + f" --separate-files {' '.join(filenames)} {out}" from careless.parser import parser parser = parser.parse_args(command.split()) run_careless(parser) for i,in_file in enumerate(filenames): out_file = out + f"_{i}.mtz" assert exists(out_file) in_ds = rs.read_mtz(in_file) out_ds = rs.read_mtz(out_file) assert in_ds.spacegroup == out_ds.spacegroup if parser.dmin is not None: assert out_ds.compute_dHKL().dHKL.min() >= parser.dmin if parser.anomalous: assert 'F(+)' in out_ds
def load_dataset(datapath, as_gemmi=False): """ Load dataset at given datapath. Datapath is expected to be a list of directories to follow. """ inFN = abspath(join(dirname(__file__), *datapath)) if as_gemmi: return gemmi.read_mtz_file(inFN) else: return rs.read_mtz(inFN)
def test_to_reciprocalgrid_float(mtz_by_spacegroup, sample_rate, p1): """ Test DataSet.to_reciprocalgrid() against gemmi for float data """ if p1: dataset = rs.read_mtz(mtz_by_spacegroup[:-4] + "_p1.mtz") else: dataset = rs.read_mtz(mtz_by_spacegroup) gemmimtz = dataset.to_gemmi() # Note: Use P1 data to determine proper gridsize testp1 = dataset.expand_to_p1() testp1.spacegroup = dataset.spacegroup gridsize = testp1.to_gemmi().get_size_for_hkl(sample_rate=sample_rate) gemmigrid = gemmimtz.get_value_on_grid("FMODEL", size=gridsize) expected = np.array(gemmigrid) result = dataset.to_reciprocalgrid("FMODEL", gridsize) assert np.allclose(result, expected)
def test_read_unmerged_2m_isym(data_unmerged): """Test rs.read_mtz() with unmerged data containing 2 M/ISYM columns""" data_unmerged["EXTRA"] = 1 data_unmerged["EXTRA"] = data_unmerged["EXTRA"].astype("M/ISYM") temp = tempfile.NamedTemporaryFile(suffix=".mtz") data_unmerged.write_mtz(temp.name) with pytest.raises(ValueError): fails = rs.read_mtz(temp.name) # Clean up temp.close()
def test_to_reciprocalgrid_complex(mtz_by_spacegroup, sample_rate, p1): """ Test DataSet.to_reciprocalgrid() against gemmi for complex data """ if p1: dataset = rs.read_mtz(mtz_by_spacegroup[:-4] + "_p1.mtz") else: dataset = rs.read_mtz(mtz_by_spacegroup) gemmimtz = dataset.to_gemmi() # Note: Use P1 data to determine proper gridsize testp1 = dataset.expand_to_p1() testp1.spacegroup = dataset.spacegroup gridsize = testp1.to_gemmi().get_size_for_hkl(sample_rate=sample_rate) gemmigrid = gemmimtz.get_f_phi_on_grid("FMODEL", "PHIFMODEL", size=gridsize) expected = np.array(gemmigrid) dataset["sf"] = dataset.to_structurefactor("FMODEL", "PHIFMODEL") result = dataset.to_reciprocalgrid("sf", gridsize) assert np.allclose(result, expected, rtol=1e-4)
def test_expand_to_p1_with_p1(mtz_by_spacegroup): """DataSet.expand_to_p1() should not affect P1 data""" expected = rs.read_mtz(mtz_by_spacegroup[:-4] + "_p1.mtz") result = expected.expand_to_p1() result.sort_index(inplace=True) expected.sort_index(inplace=True) expected_sf = expected.to_structurefactor("FMODEL", "PHIFMODEL") result_sf = result.to_structurefactor("FMODEL", "PHIFMODEL") assert_index_equal(result.index, expected.index) assert np.allclose(result_sf.to_numpy(), expected_sf.to_numpy(), rtol=1e-4)
def test_roundtrip(self): datadir = join(abspath(dirname(__file__)), '../data/fmodel') data = rs.read_mtz(join(datadir, '9LYZ.mtz')) # Write data, read data, write data again... shouldn't change temp = tempfile.NamedTemporaryFile(suffix=".mtz") data.write_mtz(temp.name) data2 = rs.read_mtz(temp.name) temp2 = tempfile.NamedTemporaryFile(suffix=".mtz") data2.write_mtz(temp2.name) self.assertTrue(data.equals(data2)) self.assertEqual(data.spacegroup.number, data2.spacegroup.number) self.assertEqual(data.cell.a, data2.cell.a) self.assertTrue(filecmp.cmp(temp.name, temp2.name)) # Clean up temp.close() temp2.close() return
def test_acentrics(mtz_by_spacegroup, cache): """Test DataSet.acentrics against DataSet.label_centrics""" ds = rs.read_mtz(mtz_by_spacegroup) if cache: ds.label_centrics(inplace=True) expected = ds.loc[~ds.CENTRIC] result = ds.acentrics assert "CENTRIC" in result else: expected = ds.loc[~ds.label_centrics()["CENTRIC"]] result = ds.acentrics assert "CENTRIC" not in result assert_frame_equal(result, expected)
def base_test_together(flags, filenames): with TemporaryDirectory() as td: out = td + '/out' command = flags + f" {' '.join(filenames)} {out}" from careless.parser import parser parser = parser.parse_args(command.split()) run_careless(parser) out_file = out + f"_0.mtz" assert exists(out_file) out_ds = rs.read_mtz(out_file) if parser.dmin is not None: assert out_ds.compute_dHKL().dHKL.min() >= parser.dmin if parser.anomalous: assert 'F(+)' in out_ds
def test_read(self): datadir = join(abspath(dirname(__file__)), '../data/fmodel') data = rs.read_mtz(join(datadir, '9LYZ.mtz')) # Confirm columns, indices, and metadata self.assertEqual(data.spacegroup.number, 96) self.assertEqual(data.columns.to_list(), ["FMODEL", "PHIFMODEL"]) self.assertEqual(list(data.index.names), ["H", "K", "L"]) self.assertIsInstance(data.spacegroup, gemmi.SpaceGroup) self.assertIsInstance(data.cell, gemmi.UnitCell) self.assertIsInstance(data, rs.DataSet) self.assertIsInstance(data["FMODEL"], rs.DataSeries) return
def test_add_rfree(self): datadir = join(abspath(dirname(__file__)), '../data/fmodel') data = rs.read_mtz(join(datadir, '9LYZ.mtz')) # Should copy data rfree = rs.utils.add_rfree(data) self.assertFalse(id(data) == id(rfree)) self.assertFalse("R-free-flags" in data.columns) self.assertTrue("R-free-flags" in rfree.columns) # Test inplace option rfree = rs.utils.add_rfree(data, inplace=True) self.assertTrue(id(data) == id(rfree)) self.assertTrue("R-free-flags" in data.columns) self.assertTrue("R-free-flags" in rfree.columns) return
def test_apply_symop_roundtrip(mtz_by_spacegroup): """ Test DataSet.apply_symop() using fmodel datasets. This test will apply one of the symmetry operations, and confirm that hkl_to_asu() returns it to the same HKL, with the same phase """ dataset = rs.read_mtz(mtz_by_spacegroup) for op in dataset.spacegroup.operations(): applied = dataset.apply_symop(op) back = applied.hkl_to_asu() assert np.array_equal(back.FMODEL.to_numpy(), dataset.FMODEL.to_numpy()) assert np.array_equal(back.get_hkls(), dataset.get_hkls()) original = rs.utils.to_structurefactor(dataset.FMODEL, dataset.PHIFMODEL) back = rs.utils.to_structurefactor(back.FMODEL, back.PHIFMODEL) assert np.isclose(original, back, rtol=1e-3).all()
def test_roundtrip(data_unmerged, label_centrics): """ Test roundtrip of rs.read_mtz() and DataSet.write_mtz() with unmerged data """ if label_centrics: data_unmerged.label_centrics(inplace=True) temp = tempfile.NamedTemporaryFile(suffix=".mtz") temp2 = tempfile.NamedTemporaryFile(suffix=".mtz") data_unmerged.write_mtz(temp.name) data2 = rs.read_mtz(temp.name) data2.write_mtz(temp2.name) assert filecmp.cmp(temp.name, temp2.name) assert_frame_equal(data_unmerged, data2) assert data_unmerged.merged == data2.merged # Clean up temp.close() temp2.close()
def test_read_csv(IOtest_mtz, sep, spacegroup, cell, merged, infer_mtz_dtypes): """Test rs.read_csv()""" csvfile = tempfile.NamedTemporaryFile(suffix=".csv") expected = rs.read_mtz(IOtest_mtz) expected.to_csv(csvfile.name, sep=sep) result = rs.read_csv(csvfile.name, spacegroup=spacegroup, cell=cell, merged=merged, infer_mtz_dtypes=infer_mtz_dtypes, sep=sep) print(result) result.set_index(["H", "K", "L"], inplace=True) csvfile.close() if spacegroup: if isinstance(spacegroup, gemmi.SpaceGroup): assert result.spacegroup.xhm() == spacegroup.xhm() else: assert result.spacegroup.xhm() == gemmi.SpaceGroup( spacegroup).xhm() else: assert result.spacegroup is None if cell: if isinstance(cell, gemmi.UnitCell): assert result.cell.a == cell.a else: assert result.cell.a == cell[0] else: assert result.cell is None assert result.merged == merged if infer_mtz_dtypes: assert_frame_equal(result, expected) else: assert_frame_equal(result, expected, check_dtype=False, check_index_type=False)
bins) * multiplicity elif mean_intensity_method == "anisotropic": Sigma = mean_intensity_by_miller_index(I / multiplicity, ds.get_hkls(), bw) * multiplicity # Initialize outputs ds[outputI] = 0. ds[outputSigI] = 0. mean_I, std_I, mean_F, std_F = _french_wilson_posterior_quad( ds[intensity_key].to_numpy(), ds[sigma_key].to_numpy(), Sigma, ds.CENTRIC.to_numpy()) # Convert dtypes of columns to MTZDtypes ds[outputI] = rs.DataSeries(mean_I, index=ds.index, dtype="Intensity") ds[outputSigI] = rs.DataSeries(std_I, index=ds.index, dtype="Stddev") ds[outputF] = rs.DataSeries(mean_F, index=ds.index, dtype="SFAmplitude") ds[outputSigF] = rs.DataSeries(std_F, index=ds.index, dtype="Stddev") return ds if __name__ == "__main__": # pragma: no cover import reciprocalspaceship as rs from sys import argv ds = rs.read_mtz(argv[1]).dropna() ds = ds.stack_anomalous() ds = scale_merged_intensities(ds, "IMEAN", "SIGIMEAN") from IPython import embed embed(colors='Linux')
from matplotlib import pyplot as plt import reciprocalspaceship as rs from argparse import ArgumentParser desc = """ Make a simple plot of scaling results. """ parser = ArgumentParser(desc) parser.add_argument('mtz_file', help="An mtz file output from `uvscale`.") parser = parser.parse_args() inFN = parser.mtz_file mtz = rs.read_mtz(inFN).compute_dHKL() mtz.sort_values('dHKL', inplace=True) plt.fill_between(mtz.dHKL**-2, mtz.SqrtSigma - mtz.SigSqrtSigma, mtz.SqrtSigma + mtz.SigSqrtSigma, color='b', alpha=0.1, label="$\sqrt{\Sigma} \pm (std. dev)$" ) plt.plot(mtz.dHKL**-2, mtz.SqrtSigma, '--r', label="$\sqrt{\Sigma}$" ) plt.errorbar(mtz.dHKL**-2, mtz.F, yerr=mtz.SigF, ls='none', color='k', label="$F$" ) plt.xlabel("$D_{h}^{-2}\ (\AA^{-2})$")
def from_mtz_files(cls, filenames, formatter): return cls.from_datasets((rs.read_mtz(i) for i in filenames), formatter)
def load(filename): if filename.endswith('.mtz'): return rs.read_mtz(filename) elif filename.endswith('.stream'): return rs.read_crystfel(filename)
#offFN = "/home/kmdalton/xtal/gfp/300fs_wo144/integration/xtals/abismal/checkpoint_0_300.mtz" #onFN = "/home/kmdalton/xtal/gfp/300fs_wo144/integration/xtals/abismal/checkpoint_1_300.mtz" #phase_FN = "/home/kmdalton/xtal/gfp/models/GFP_SSRL_refine_54_final_P21.mtz" offFN = "/home/kmdalton/opt/careless-examples/pyp/merge/pyp_0.mtz" onFN = "/home/kmdalton/opt/careless-examples/pyp/merge/pyp_1.mtz" phase_FN = "/home/kmdalton/opt/careless-examples/pyp/phenix/PYP_refine_1.mtz" outFN = "test.mtz" nproc = 1 chain_length = 3_000 burnin = 3_000 ds = rs.read_mtz(onFN).join( rs.read_mtz(offFN).expand_to_p1().expand_anomalous(), rsuffix='off', lsuffix='on', check_isomorphous=False).dropna() ds = ds.compute_multiplicity().compute_dHKL().label_centrics() Fon, SigFon, Foff, SigFoff = ds[[ 'Flocon', 'Fscaleon', 'Flocoff', 'Fscaleoff', ]].to_numpy('float').T def truncnorm_quadrature(Fon, SigFon,
from careless.utils.device import disable_gpu status = disable_gpu() assert status dmin = 5. #Use less memory and go faster the test files only go out to 4 angstroms anyway base_dir = dirname(abspath(__file__)) mtz_filenames = [ base_dir + '/pyp_off.mtz', base_dir + '/pyp_2ms.mtz', ] mtz_data = [] for f in mtz_filenames: m = rs.read_mtz(f) mtz_data.append(m[m.compute_dHKL().dHKL >= dmin]) reference_filename = base_dir + '/pyp_reference.mtz' for f in mtz_filenames: assert exists(f) assert exists(reference_filename) reference_data = rs.read_mtz(reference_filename) reference_data = reference_data[reference_data.compute_dHKL().dHKL >= dmin] @pytest.mark.parametrize("merger_class", [MonoMerger, PolyMerger]) @pytest.mark.parametrize("anomalous", [False, True]) def test_Constructor(merger_class, anomalous):