def test_scale_handle_bad_dataset(dials_data, tmp_path): """Set command line parameters such that one dataset does not meet the criteria for inclusion in scaling. Check that this is excluded and the scaling job completes without failure.""" location = dials_data("multi_crystal_proteinase_k", pathlib=True) command = [ "dials.scale", "reflection_selection.method=intensity_ranges", "Isigma_range=90.0,1000", ] for i in range(1, 6): command.append(location / f"experiments_{i}.json") command.append(location / f"reflections_{i}.pickle") result = procrunner.run(command, working_directory=tmp_path) assert not result.returncode and not result.stderr reflections = flex.reflection_table.from_file(tmp_path / "scaled.refl") expts = load.experiment_list(tmp_path / "scaled.expt", check_format=False) assert len(expts) == 4 assert len(reflections.experiment_identifiers()) == 4
def test_import_integrate_hkl(dials_data, tmpdir): result = procrunner.run( [ "dials.import_xds", "input.method=reflections", dials_data("centroid_test_data").join("INTEGRATE.HKL"), dials_data("centroid_test_data").join("experiments.json"), ], working_directory=tmpdir, ) assert not result.returncode and not result.stderr table = flex.reflection_table.from_file(tmpdir / "integrate_hkl.refl") assert "miller_index" in table assert "id" in table assert "panel" in table assert "xyzcal.px" in table assert "xyzobs.px.value" in table assert "intensity.cor.value" in table assert "intensity.cor.variance" in table assert len(table) == 174911
def test_find_spots_with_xfel_stills(dials_regression, tmp_path): # now with XFEL stills result = procrunner.run( [ "dials.find_spots", "nproc=1", os.path.join( dials_regression, "spotfinding_test_data", "idx-s00-20131106040302615.cbf", ), "output.reflections=spotfinder.refl", "algorithm=dispersion", ], working_directory=tmp_path, ) assert not result.returncode and not result.stderr assert (tmp_path / "spotfinder.refl").is_file() reflections = flex.reflection_table.from_file(tmp_path / "spotfinder.refl") assert len(reflections) == 2643
def test_find_spots_from_images_override_maximum(dials_data, tmp_path): result = procrunner.run( [ "dials.find_spots", "nproc=1", "maximum_trusted_value=100", "output.reflections=spotfinder.refl", "output.shoeboxes=True", "algorithm=dispersion", ] + list( dials_data("centroid_test_data", pathlib=True).glob("centroid*.cbf")), working_directory=tmp_path, ) assert not result.returncode and not result.stderr assert (tmp_path / "spotfinder.refl").is_file() reflections = flex.reflection_table.from_file(tmp_path / "spotfinder.refl") sbox = reflections["shoebox"] for s in sbox: assert flex.max(s.data) <= 100
def test_find_spots_with_resolution_filter(dials_data, tmp_path): result = procrunner.run( [ "dials.find_spots", "nproc=1", "output.reflections=spotfinder.refl", "output.shoeboxes=False", "algorithm=dispersion", "filter.d_min=2", "filter.d_max=15", ] + list( dials_data("centroid_test_data", pathlib=True).glob("centroid*.cbf")), working_directory=tmp_path, ) assert not result.returncode and not result.stderr assert (tmp_path / "spotfinder.refl").is_file() reflections = flex.reflection_table.from_file(tmp_path / "spotfinder.refl") assert len(reflections) in range(467, 469) assert "shoebox" not in reflections
def test_find_spots_from_images(dials_data, tmp_path): result = procrunner.run( [ "dials.find_spots", "nproc=1", "output.reflections=spotfinder.refl", "output.shoeboxes=True", "algorithm=dispersion", ] + list( dials_data("centroid_test_data", pathlib=True).glob("centroid*.cbf")), working_directory=tmp_path, ) assert not result.returncode and not result.stderr assert (tmp_path / "spotfinder.refl").is_file() reflections = flex.reflection_table.from_file(tmp_path / "spotfinder.refl") _check_expected_results(reflections) # No identifiers set if just running on images and not outputting experiments assert not reflections.experiment_identifiers().values(), list( reflections.experiment_identifiers().values())
def test_mmcif(compress, hklout, dials_data, tmp_path): # Call dials.export after integration command = [ "dials.export", "format=mmcif", dials_data("centroid_test_data", pathlib=True) / "experiments.json", dials_data("centroid_test_data", pathlib=True) / "integrated.pickle", ] if hklout is not None: command.append(f"mmcif.hklout={hklout}") else: hklout = "integrated.cif" if compress is not None: command.append(f"mmcif.compress={compress}") hklin = hklout + "." + compress else: hklin = hklout result = procrunner.run(command, working_directory=tmp_path) assert not result.returncode and not result.stderr assert (tmp_path / hklin).is_file()
def test_xds_ccp4a(regression_test, dials_data, tmpdir, ccp4, xds): command_line = [ "xia2", "pipeline=3di", "nproc=1", "njob=2", "mode=parallel", "trust_beam_centre=True", "scaler=ccp4a", dials_data("fumarase").strpath, ] result = procrunner.run(command_line, working_directory=tmpdir.strpath) success, issues = xia2.Test.regression.check_result( "mad_example.ccp4a", result, tmpdir, ccp4, xds, expected_data_files=expected_data_files, ) assert success, issues
def test_find_spots_from_images(dials_data, tmpdir): result = procrunner.run( [ "dials.find_spots", "output.reflections=spotfinder.refl", "output.shoeboxes=True", "algorithm=dispersion", ] + [ f.strpath for f in dials_data("centroid_test_data").listdir("centroid*.cbf") ], working_directory=tmpdir.strpath, ) assert not result.returncode and not result.stderr assert tmpdir.join("spotfinder.refl").check(file=1) reflections = flex.reflection_table.from_file(tmpdir / "spotfinder.refl") _check_expected_results(reflections) # No identifiers set if just running on images and not outputting experiments assert not reflections.experiment_identifiers().values(), list( reflections.experiment_identifiers().values())
def run(self): assert hasattr(self, "recwrap"), "No recipewrapper object found" params = self.recwrap.recipe_step["job_parameters"] command = params["wrapped_commands"] self.modify_command(command, params) logger.info("Command: %s", " ".join(command)) result = procrunner.run(command) logger.info("Command successful, took %.1f seconds", result["runtime"]) if "filename" in params: utils.record_result(self, params["filename"], "Result") self.additional_file_actions(params["filename"], not result["exitcode"]) if "logname" in params: utils.record_result(self, params["logname"], "Log") return not result["exitcode"]
def test_find_spots_with_resolution_filter(dials_data, tmpdir): result = procrunner.run( [ "dials.find_spots", "output.reflections=spotfinder.refl", "output.shoeboxes=False", "algorithm=dispersion", "filter.d_min=2", "filter.d_max=15", ] + [ f.strpath for f in dials_data("centroid_test_data").listdir("centroid*.cbf") ], working_directory=tmpdir.strpath, ) assert not result.returncode and not result.stderr assert tmpdir.join("spotfinder.refl").check(file=1) reflections = flex.reflection_table.from_file(tmpdir / "spotfinder.refl") assert len(reflections) in range(467, 469) assert "shoebox" not in reflections
def test_write_xinfo_insulin_with_missing_image(insulin_with_missing_image, run_in_tmpdir): result = procrunner.run( [ "xia2.setup", f"image={insulin_with_missing_image.parent.joinpath('insulin_1_001.img')}", ], environment_override={"CCP4": run_in_tmpdir}, ) assert not result.returncode assert not result.stderr xinfo = run_in_tmpdir.join("automatic.xinfo") assert xinfo.check(file=1) x = XInfo(xinfo) assert len(x.get_crystals()["DEFAULT"]["sweeps"]) == 2 assert x.get_crystals()["DEFAULT"]["sweeps"]["SWEEP1"]["start_end"] == [ 1, 22 ] assert x.get_crystals()["DEFAULT"]["sweeps"]["SWEEP2"]["start_end"] == [ 24, 45 ]
def test_mmcif(compress, hklout, dials_data, tmpdir): # Call dials.export after integration command = [ "dials.export", "format=mmcif", dials_data("centroid_test_data").join("experiments.json").strpath, dials_data("centroid_test_data").join("integrated.pickle").strpath, ] if hklout is not None: command.append(f"mmcif.hklout={hklout}") else: hklout = "integrated.cif" if compress is not None: command.append(f"mmcif.compress={compress}") hklin = hklout + "." + compress else: hklin = hklout result = procrunner.run(command, working_directory=tmpdir.strpath) assert not result.returncode and not result.stderr assert tmpdir.join(hklin).check(file=1)
def test_import_spot_xds_with_filtering(dials_data, tmpdir): result = procrunner.run( [ "dials.import_xds", "input.method=reflections", dials_data("centroid_test_data").join("SPOT.XDS"), "remove_invalid=True", ], working_directory=tmpdir, ) assert not result.returncode and not result.stderr with tmpdir.join("spot_xds.refl").open("rb") as fh: table = pickle.load(fh) assert "miller_index" in table assert "id" in table assert "panel" in table assert "xyzobs.px.value" in table assert "intensity.sum.value" in table assert len(table) == 664
def test_with_mask(dials_data, tmpdir): image_files = sorted( dials_data("centroid_test_data", pathlib=True).glob("centroid*.cbf")) mask_filename = dials_data("centroid_test_data").join("mask.pickle") result = procrunner.run( [ "dials.import", "mask=" + mask_filename.strpath, "output.experiments=experiments_with_mask.expt", ] + image_files, working_directory=tmpdir, ) assert not result.returncode and not result.stderr assert tmpdir.join("experiments_with_mask.expt").check(file=1) experiments = load.experiment_list( tmpdir.join("experiments_with_mask.expt").strpath) assert experiments[0].identifier != "" assert (experiments[0].imageset.external_lookup.mask.filename == mask_filename.strpath)
def test_can_import_multiple_sequences(dials_data, tmpdir): # Find the image files image_files = dials_data("centroid_test_data").listdir("centroid*.cbf", sort=True) del image_files[4] # Delete filename to force two sequences result = procrunner.run( [ "dials.import", "output.experiments=experiments_multiple_sequences.expt" ] + image_files, working_directory=tmpdir, ) assert not result.returncode and not result.stderr assert tmpdir.join("experiments_multiple_sequences.expt").check(file=1) experiments = load.experiment_list( tmpdir.join("experiments_multiple_sequences.expt").strpath) assert len(experiments) == 2 for experiment in experiments: assert experiment.identifier != ""
def _command_runner( command, output_directory=None, store_command=None, store_output=None, **kwargs ): """Run a command and write its output to a defined location""" print(" ".join(str(e) for e in command)) if output_directory and store_command is None: store_command = output_directory / f"{command[0]}.cmd" if output_directory and store_output is None: store_output = output_directory / f"{command[0]}.log" if store_command: store_command.parent.mkdir(parents=True, exist_ok=True) store_command.write_text(" ".join(str(e) for e in command)) start = time.perf_counter() result = procrunner.run( command, environment_override={"DIALS_NOBANNER": "1"}, **kwargs ) print(f"running command took {time.perf_counter() - start:.1f} seconds\n") assert not result.returncode, "Command execution failed" if store_output: store_output.parent.mkdir(parents=True, exist_ok=True) store_output.write_bytes(result.stdout)
def test_symmetry_with_absences(dials_data, tmpdir, option): """Simple test to check that dials.symmetry, with absences, completes""" cmd = [ "dials.symmetry", dials_data("l_cysteine_dials_output") / "20_integrated_experiments.json", dials_data("l_cysteine_dials_output") / "20_integrated.pickle", dials_data("l_cysteine_dials_output") / "25_integrated_experiments.json", dials_data("l_cysteine_dials_output") / "25_integrated.pickle", ] if option: cmd.append(option) result = procrunner.run(cmd, working_directory=tmpdir) assert not result.returncode and not result.stderr assert tmpdir.join("symmetrized.refl").check() assert tmpdir.join("symmetrized.expt").check() expts = load.experiment_list( tmpdir.join("symmetrized.expt").strpath, check_format=False ) assert str(expts[0].crystal.get_space_group().info()) == "P 21 21 21"
def test_modify_geometry(dials_regression, tmpdir): orig_expt_json = os.path.join( dials_regression, "experiment_test_data/kappa_experiments.json") new_expt_json = tmpdir.join("modified.expt") result = procrunner.run( ["dials.modify_geometry", orig_expt_json, "angles=10,20,30"], working_directory=tmpdir, ) assert not result.returncode and not result.stderr assert os.path.exists(orig_expt_json), orig_expt_json orig_expt = load.experiment_list(orig_expt_json, check_format=False) assert new_expt_json.check() new_expt = load.experiment_list(new_expt_json.strpath, check_format=False) orig_gonio = orig_expt.goniometers()[0] new_gonio = new_expt.goniometers()[0] assert orig_gonio.get_angles() == pytest.approx([0, 180, 0]) assert new_gonio.get_angles() == pytest.approx([10, 20, 30])
def test_find_spots_with_xfel_stills(dials_regression, tmpdir): # now with XFEL stills result = procrunner.run( [ "dials.find_spots", os.path.join( dials_regression, "spotfinding_test_data", "idx-s00-20131106040302615.cbf", ), "output.reflections=spotfinder.refl", "algorithm=dispersion", ], working_directory=tmpdir.strpath, ) assert not result.returncode and not result.stderr assert tmpdir.join("spotfinder.refl").check(file=1) with tmpdir.join("spotfinder.refl").open("rb") as f: reflections = pickle.load(f) assert len(reflections) == 2643
def test_default_sort_on_miller_index(dials_data, tmpdir): result = procrunner.run( [ "dev.dials.sort_reflections", dials_data("centroid_test_data").join("integrated.pickle").strpath, "output=sorted3.refl", ], working_directory=tmpdir.strpath, ) assert not result.returncode and not result.stderr assert tmpdir.join("sorted3.refl").check(file=1) from dials.array_family import flex data = flex.reflection_table.from_file(tmpdir.join("sorted3.refl").strpath) mi1 = data["miller_index"] orig = flex.reflection_table.from_file( dials_data("centroid_test_data").join("integrated.pickle").strpath ) mi2 = flex.miller_index(sorted(orig["miller_index"])) assert mi1.all_eq(mi2)
def test_find_spots_with_generous_parameters(dials_data, tmpdir): # now with more generous parameters result = procrunner.run( [ "dials.find_spots", "min_spot_size=3", "max_separation=3", "output.reflections=spotfinder.refl", "algorithm=dispersion", ] + [ f.strpath for f in dials_data("centroid_test_data").listdir("centroid*.cbf") ], working_directory=tmpdir.strpath, ) assert not result.returncode and not result.stderr assert tmpdir.join("spotfinder.refl").check(file=1) with tmpdir.join("spotfinder.refl").open("rb") as f: reflections = pickle.load(f) assert len(reflections) in range(678, 680)
def xds(): """ Return information about the XDS installation. Skip the test if XDS is not installed. """ try: result = procrunner.run(["xds"], print_stdout=False) except OSError: pytest.skip("XDS installation required for this test") version = re.search(br"BUILT=([0-9]+)\)", result.stdout) if version: return {"version": int(version.groups()[0])} if result.returncode: pytest.skip( "XDS installation required for this test - Could not run XDS") if b"license expired" in result.stdout: raise RuntimeError( "XDS installation required for this test - XDS license is expired") pytest.skip( "XDS installation required for this test - Could not determine XDS version" )
def test_xds_ccp4a(regression_test, dials_data, tmpdir, ccp4, xds): command_line = [ "xia2", "pipeline=3dii", "nproc=2", "small_molecule=True", "read_all_image_headers=False", "trust_beam_centre=True", "scaler=ccp4a", dials_data("small_molecule_example").strpath, ] result = procrunner.run(command_line, working_directory=tmpdir.strpath) success, issues = xia2.Test.regression.check_result( "small_molecule.ccp4a", result, tmpdir, ccp4, xds, expected_data_files=expected_data_files, ) assert success, issues
def test_refinement_failure_on_max_lattices_a15(dials_regression, tmpdir): """Problem: Sometimes there is enough data to index, but not enough to refine. If this happens in the (N>1)th crystal of max_lattices, then all existing solutions are also dropped.""" data_dir = os.path.join(dials_regression, "indexing_test_data", "lattice_failures") result = procrunner.run( [ "dials.index", os.path.join(data_dir, "lpe4-2-a15_strong.pickle"), os.path.join(data_dir, "lpe4-2-a15_datablock.json"), "max_lattices=3", ], working_directory=tmpdir, ) assert not result.returncode and not result.stderr assert tmpdir.join("indexed.refl").check() and tmpdir.join("indexed.expt").check() experiments_list = load.experiment_list( tmpdir.join("indexed.expt").strpath, check_format=False ) assert len(experiments_list) == 2
def test_export_mosflm(dials_regression, tmpdir): dials_regression_escaped = json.dumps(dials_regression).strip('"') with open( os.path.join(dials_regression, "experiment_test_data/experiment_1.json"), "r") as fi: with (tmpdir / "experiments.json").open("w") as fo: fo.write(fi.read().replace("$DIALS_REGRESSION", dials_regression_escaped)) tmpdir.chdir() result = procrunner.run( ["dials.export", "format=mosflm", "experiments.json"]) assert not result.returncode and not result.stderr assert os.path.exists("mosflm/index.mat") with open("mosflm/index.mat", "rb") as f: lines = f.read() assert (lines == """ -0.01210200 -0.01954526 0.00309519 -0.00416605 -0.00080573 -0.02427340 0.01931593 -0.01241956 -0.00329641 0.000 0.000 0.000 -0.52228050 -0.84350975 0.12535704 -0.17980379 -0.03477015 -0.98308781 0.83360283 -0.53598726 -0.13350648 42.2717 42.2720 39.6704 90.0001 89.9993 89.9998 0.000 0.000 0.000 """.strip("\n")) assert os.path.exists("mosflm/mosflm.in") with open("mosflm/mosflm.in", "rb") as f: lines = f.read() assert (lines == """ DIRECTORY %s%scentroid_test_data TEMPLATE centroid_####.cbf SYMMETRY 89 BEAM 220.002 212.478 DISTANCE 190.1800 MATRIX index.mat """.strip("\n") % (dials_regression, os.path.sep))
def test_refine_bravais_settings_554(dials_regression, tmpdir): data_dir = os.path.join(dials_regression, "dials-554") pickle_path = os.path.join(data_dir, "indexed.pickle") experiments_path = os.path.join(data_dir, "experiments.json") result = procrunner.run( ["dials.refine_bravais_settings", pickle_path, experiments_path], working_directory=tmpdir, ) assert not result.returncode and not result.stderr for i in range(1, 5): assert tmpdir.join("bravais_setting_%i.expt" % i).check() experiments_list = load.experiment_list( tmpdir.join("bravais_setting_5.expt").strpath, check_format=False) assert len(experiments_list) == 7 assert len(experiments_list.crystals()) == 1 crystal = experiments_list.crystals()[0] assert crystal.get_unit_cell().is_similar_to( uctbx.unit_cell((4.75863, 4.75863, 12.9885, 90, 90, 120))) assert crystal.get_space_group().type().hall_symbol() == " R 3" # assert all of the detectors are different for expt in experiments_list[1:]: assert expt.detector != experiments_list[0].detector for i in (0, 1, 6): assert experiments_list[i].detector[0].get_origin() == pytest.approx( (-41, 5.5, -135), abs=1) for i in (2, 3, 4, 5): assert experiments_list[i].detector[0].get_origin() == pytest.approx( (-41, 91, -99), abs=1) assert tmpdir.join("bravais_summary.json").check() with tmpdir.join("bravais_summary.json").open("rb") as fh: bravais_summary = json.load(fh) for i in range(1, 5): assert str(i) in bravais_summary assert bravais_summary["5"]["unit_cell"] == pytest.approx( [4.75863, 4.75863, 12.9885, 90, 90, 120], abs=1e-1) assert bravais_summary["5"]["bravais"] == "hR" assert bravais_summary["5"]["rmsd"] == pytest.approx(0.104, abs=1e-2) assert bravais_summary["5"]["recommended"] is True
def test_split_chunk_sizes(tmpdir, option, with_identifiers): ids = list(range(0, 5)) experiments = ExperimentList() reflections = flex.reflection_table() reflections["id"] = flex.int(ids) reflections["intensity"] = flex.double([(i + 1) * 100.0 for i in ids]) for i in ids: exp = generate_exp() if with_identifiers: exp.identifier = str(i) reflections.experiment_identifiers()[i] = str(i) experiments.append(exp) experiments.as_json(tmpdir.join("tmp.expt").strpath) reflections.as_file(tmpdir.join("tmp.refl").strpath) result = procrunner.run( [ "dials.split_experiments", tmpdir.join("tmp.expt").strpath, tmpdir.join("tmp.refl").strpath, option, ], working_directory=tmpdir, ) assert not result.returncode and not result.stderr for j, n, intensities in zip([0, 1, 2], [2, 2, 1], [[100.0, 200.0], [300.0, 400.0], [500.0]]): assert tmpdir.join(f"split_{j}.refl").check() assert tmpdir.join(f"split_{j}.expt").check() expts = load.experiment_list(tmpdir.join(f"split_{j}.expt"), check_format=False) assert len(expts) == n refls = flex.reflection_table.from_file(tmpdir.join(f"split_{j}.refl")) assert list(set(refls["id"])) == list(range(0, n)) assert list(refls["intensity"]) == intensities refls.assert_experiment_identifiers_are_consistent(expts)
def test_search_multiple(run_in_tmpdir, dials_regression): """Perform a beam-centre search and check that the output is sane. Do the following: 1. Run dials.search_beam_centre on two datablocks and two pickled reflection tables, as output by dials.find_spots; a) Check that the program exits correctly; b) Check that it produces the expected output datablock. 2. Check that the beam centre search has resulted in the expected shift in detector origin. """ data_dir = os.path.join(dials_regression, "indexing_test_data", "trypsin") pickle_path1 = os.path.join(data_dir, "strong_P1_X6_1_0-1.pickle") pickle_path2 = os.path.join(data_dir, "strong_P1_X6_2_0-1.pickle") experiments_path1 = os.path.join(data_dir, "datablock_P1_X6_1.json") experiments_path2 = os.path.join(data_dir, "datablock_P1_X6_2.json") args = [ "dials.search_beam_position", experiments_path1, experiments_path2, pickle_path1, pickle_path2, ] print(args) result = procrunner.run(args) assert not result.returncode and not result.stderr assert os.path.exists("optimised.expt") experiments = load.experiment_list(experiments_path1, check_format=False) original_imageset = experiments.imagesets()[0] optimized_experiments = load.experiment_list("optimised.expt", check_format=False) detector_1 = original_imageset.get_detector() detector_2 = optimized_experiments.detectors()[0] shift = scitbx.matrix.col(detector_1[0].get_origin()) - scitbx.matrix.col( detector_2[0].get_origin()) assert shift.elems == pytest.approx((0.037, 0.061, 0.0), abs=1e-1)
def test_scan_varying_multi_scan_one_crystal(gcb, dials_data, tmpdir): # https://github.com/dials/dials/issues/994 location = dials_data("l_cysteine_dials_output") refls = location.join("indexed.refl") expts = location.join("indexed.expt") # Set options for quick rather than careful refinement result = procrunner.run( ( "dials.refine", expts, refls, "output.history=history.json", "outlier.algorithm=tukey", "max_iterations=3", "unit_cell.smoother.interval_width_degrees=56", "orientation.smoother.interval_width_degrees=56", "gradient_calculation_blocksize=" + gcb, ), working_directory=tmpdir, ) assert not result.returncode and not result.stderr el = ExperimentListFactory.from_json_file( tmpdir.join("refined.expt").strpath, check_format=False) # Crystal has been copied into each experiment for scan-varying refinement assert len(el.crystals()) == 4 # load and check results history = Journal.from_json_file(tmpdir.join("history.json").strpath) expected_rmsds = [ (0.1401658782847504, 0.2225931584837884, 0.002349912655443814), (0.12060230585178289, 0.1585977879739876, 0.002114318828411418), (0.10970832317567975, 0.1348574975434352, 0.001955034565537597), (0.10373159352273859, 0.12827852889951505, 0.0017901404193256304), ] for a, b in zip(history["rmsd"], expected_rmsds): assert a == pytest.approx(b, abs=1e-6)