def exercise_end_of_line(exe_name, verbose): lines = """\ a bc def ghij klmno """.splitlines() open("unix.txt", "wb").write("\n".join(lines)+"\n") open("dos.txt", "wb").write("\r\n".join(lines)+"\r\n") open("dos2.txt", "wb").write("\r\r\n".join(lines)+"\r\n") open("mac.txt", "wb").write("\r".join(lines)+"\r") from libtbx import easy_run from libtbx.utils import remove_files import os op = os.path expected_outputs = [ "a \nbc \ndef \nghij\nklmn\n", "a \nbc \ndef \nghij\nklmn\n", "a\r \nbc\r \ndef\r\nghij\nklmn\n", "a\rbc\n"] for vers,expected in zip(["unix", "dos", "dos2", "mac"], expected_outputs): remove_files(paths=["read_lines_out"]) cmd = "%s < %s.txt > read_lines_out" % (op.join(".", exe_name), vers) if (verbose): print cmd easy_run.fully_buffered(command=cmd).raise_if_errors_or_output() assert op.isfile("read_lines_out") result = open("read_lines_out", "rb").read() assert result == expected.replace("\n", os.linesep)
def exercise(space_group_info, anomalous_flag=False, d_min=2., verbose=0): sg_fcalc = random_structure.xray_structure( space_group_info, elements=("N", "C", "C", "O"), random_f_double_prime=anomalous_flag, random_u_iso=True, random_occupancy=True ).structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() sg_hl = generate_random_hl(sg_fcalc) write_cns_input(sg_fcalc, sg_hl.data()) try: os.unlink("tmp_sg.hkl") except OSError: pass try: os.unlink("tmp_p1.hkl") except OSError: pass easy_run.fully_buffered(command="cns < tmp.cns > tmp.out") \ .raise_if_errors_or_output() sg_cns = read_reflection_arrays("tmp_sg.hkl", anomalous_flag, verbose) p1_cns = read_reflection_arrays("tmp_p1.hkl", anomalous_flag, verbose) verify(sg_fcalc, sg_hl.data(), sg_cns, p1_cns) if (anomalous_flag): hl_merged = sg_hl.average_bijvoet_mates() fc_merged = sg_fcalc.average_bijvoet_mates() write_cns_input(sg_fcalc, sg_hl.data(), test_merge=True) try: os.unlink("tmp_merged.hkl") except OSError: pass easy_run.fully_buffered(command="cns < tmp.cns > tmp.out") \ .raise_if_errors_or_output() reflection_file = reflection_reader.cns_reflection_file( open("tmp_merged.hkl")) if (not sg_fcalc.space_group().is_centric()): fc_merged_cns = reflection_file.reciprocal_space_objects["FCALC"] fc_merged_cns = fc_merged.customized_copy( indices=fc_merged_cns.indices, data=fc_merged_cns.data).map_to_asu().common_set(fc_merged) assert fc_merged_cns.indices().all_eq(fc_merged.indices()) fc_merged_a = fc_merged.select_acentric() fc_merged_cns_a = fc_merged_cns.select_acentric() for part in [flex.real, flex.imag]: cc = flex.linear_correlation( part(fc_merged_a.data()), part(fc_merged_cns_a.data())).coefficient() if (cc < 1-1.e-6): print "FAILURE acentrics", sg_fcalc.space_group_info() if (0): return raise AssertionError names, miller_indices, hl = reflection_file.join_hl_group() assert names == ["PA", "PB", "PC", "PD"] hl_merged_cns = hl_merged.customized_copy(indices=miller_indices, data=hl)\ .map_to_asu().common_set(hl_merged) assert hl_merged_cns.indices().all_eq(hl_merged.indices()) for h,a,b in zip(hl_merged.indices(), hl_merged.data(), hl_merged_cns.data()): if (not approx_equal(a, b, eps=5.e-3)): print h print "cctbx:", a print " cns:", b if (0): return raise AssertionError
def run(self): from libtbx import easy_run # Call dials.integrate easy_run.fully_buffered(["dials.plot_scan_varying_crystal", self.path]).raise_if_errors() print "OK"
def run(self): from os.path import join, exists from libtbx import easy_run import os input_filename = join(self.path, "datablock.json") output_filename = "output_datablock.json" mask_filename = join(self.path, "lookup_mask.pickle") easy_run.fully_buffered( ['dials.apply_mask', 'input.datablock=%s' % input_filename, 'input.mask=%s' % mask_filename, 'output.datablock=%s' % output_filename]).raise_if_errors() from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_json_file(output_filename) assert len(datablocks) == 1 imagesets = datablocks[0].extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] assert imageset.external_lookup.mask.filename == mask_filename print 'OK'
def run(self): from os.path import join, exists template = glob(join(self.directory, "centroid*.cbf")) args = [ "dials.find_spots", ' '.join(template), "output.datablock=datablock.json", "output.reflections=spotfinder.pickle", "output.shoeboxes=True" ] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert exists("datablock.json") assert exists("spotfinder.pickle") args = [ "dials.find_hot_pixels", "input.datablock=datablock.json", "input.reflections=spotfinder.pickle", "output.mask=hot_mask.pickle" ] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert exists("hot_mask.pickle") print 'OK'
def run(self): from libtbx import easy_run from dials.array_family import flex # Call dials.merge_reflection_lists easy_run.fully_buffered([ 'dev.dials.merge_reflection_lists', 'temp1.pickle', 'temp2.pickle', 'method=update' ]).raise_if_errors() table = flex.reflection_table.from_pickle('merged.pickle') assert(len(table) == 360) print 'OK' # Call dials.merge_reflection_lists easy_run.fully_buffered([ 'dev.dials.merge_reflection_lists', 'temp1.pickle', 'temp2.pickle', 'method=extend' ]).raise_if_errors() table = flex.reflection_table.from_pickle('merged.pickle') assert(len(table) == 720) print 'OK'
def tst_scan_varying_prediction(self): from os.path import join from libtbx import easy_run # Call dials.predict easy_run.fully_buffered([ 'dials.predict', join(self.path, 'experiments_scan_varying_crystal.json'), ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('predicted.pickle', 'rb')) assert(len(table) == 1934) print 'OK' # Check the reflection IDs assert('id' in table) assert('miller_index' in table) assert('s1' in table) assert('xyzcal.px' in table) assert('xyzcal.mm' in table) for row in table: assert(row['id'] == 0) print 'OK'
def test4(self): from os.path import join from libtbx import easy_run import os dirname ='test4' os.mkdir(dirname) os.chdir(dirname) # Call dials.integrate easy_run.fully_buffered([ 'dials.integrate', join(self.path, 'experiments.json'), 'profile.fitting=False', 'sampling.integrate_all_reflections=False', ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('integrated.pickle', 'rb')) assert len(table) == 1000 # Call dials.integrate easy_run.fully_buffered([ 'dials.integrate', join(self.path, 'experiments.json'), 'profile.fitting=False', 'sampling.integrate_all_reflections=False', 'sampling.minimum_sample_size=500', ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('integrated.pickle', 'rb')) assert len(table) == 500 print 'OK'
def tst_import_integrate_hkl(self): from dials.array_family import flex # import dependency from os.path import join from libtbx import easy_run # Call dials.import_xds easy_run.fully_buffered([ 'dials.import_xds', 'input.method=reflections', join(self.path, 'INTEGRATE.HKL'), join(self.path, "experiments.json") ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('integrate_hkl.pickle', 'rb')) assert('miller_index' in table) assert('id' in table) assert('panel' in table) assert('xyzcal.px' in table) assert('xyzobs.px.value' in table) assert('intensity.cor.value' in table) assert('intensity.cor.variance' in table) assert(len(table) == 174911) print 'OK'
def exercise_export_bitmaps(): if dials_regression is None: print "Skipping exercise_export_bitmaps(): dials_regression not available" return data_dir = os.path.join(dials_regression, "centroid_test_data") cmd = "dials.export_bitmaps %s/centroid_0001.cbf" % data_dir print cmd result = easy_run.fully_buffered(cmd).raise_if_errors() assert os.path.exists("image0001.png") cmd = " ".join( [ "dials.export_bitmaps", "%s/datablock.json" % data_dir, "prefix=variance_", "binning=2", "display=variance", "colour_scheme=inverse_greyscale", "brightness=25", "kernel_size=5,5", ] ) print cmd result = easy_run.fully_buffered(cmd).raise_if_errors() for i in range(1, 8): assert os.path.exists("variance_000%i.png" % i)
def run(self): from os.path import join from libtbx import easy_run from dials.algorithms.profile_model.factory import phil_scope from libtbx.phil import parse from dxtbx.model.experiment.experiment_list import ExperimentListFactory # Call dials.create_profile_model easy_run.fully_buffered([ 'dials.create_profile_model', join(self.path, 'experiments.json'), join(self.path, 'indexed.pickle'), ]).raise_if_errors() experiments = ExperimentListFactory.from_json_file( "experiments_with_profile_model.json", check_format=False) sigma_b = experiments[0].profile.sigma_b(deg=True) sigma_m = experiments[0].profile.sigma_m(deg=True) eps = 1e-3 try: assert(abs(sigma_b - 0.02195) < eps) assert(abs(sigma_m - 0.06833) < eps) except Exception: print sigma_b print sigma_m raise print 'OK'
def build_run(a_out, n_scatt, n_refl, build_cmd, check_max_a_b): if (op.isfile("a.out")): os.remove("a.out") assert not op.isfile("a.out") if (a_out is None): easy_run.fully_buffered(command=build_cmd).raise_if_errors_or_output() else: open("a.out", "w").write(a_out) os.chmod("a.out", 0755) assert op.isfile("a.out") run_cmd = "/usr/bin/time -p ./a.out" buffers = easy_run.fully_buffered(command=run_cmd) assert len(buffers.stderr_lines) == 3 if (n_scatt <= 10 and n_refl <= 100): assert len(buffers.stdout_lines) == n_scatt + n_refl else: assert len(buffers.stdout_lines) == 1 max_a, max_b = [float(s) for s in buffers.stdout_lines[0].split()] if (check_max_a_b): if (n_scatt == 2000 and n_refl == 20000): assert approx_equal(max_a, 35.047157, eps=1e-4) assert approx_equal(max_b, 25.212738, eps=1e-4) elif (n_scatt == 100 and n_refl == 1000): assert approx_equal(max_a, 4.493645, eps=1e-4) assert approx_equal(max_b, 10.515532, eps=1e-4) elif (n_scatt <= 10 and n_refl <= 100): compare_with_cctbx_structure_factors( n_scatt=n_scatt, n_refl=n_refl, output_lines=buffers.stdout_lines) else: raise RuntimeError, (max_a, max_b) utime = float(buffers.stderr_lines[1].split()[1]) return utime
def exercise () : open("tmp_fmodel_fake_p1.pdb", "w").write("""\ ATOM 47 N TYR A 7 8.292 1.817 6.147 1.00 14.70 N ATOM 48 CA TYR A 7 9.159 2.144 7.299 1.00 15.18 C ATOM 49 C TYR A 7 10.603 2.331 6.885 1.00 15.91 C ATOM 50 O TYR A 7 11.041 1.811 5.855 1.00 15.76 O ATOM 51 CB TYR A 7 9.061 1.065 8.369 1.00 15.35 C ATOM 52 CG TYR A 7 7.665 0.929 8.902 1.00 14.45 C ATOM 53 CD1 TYR A 7 6.771 0.021 8.327 1.00 15.68 C ATOM 54 CD2 TYR A 7 7.210 1.756 9.920 1.00 14.80 C ATOM 55 CE1 TYR A 7 5.480 -0.094 8.796 1.00 13.46 C ATOM 56 CE2 TYR A 7 5.904 1.649 10.416 1.00 14.33 C ATOM 57 CZ TYR A 7 5.047 0.729 9.831 1.00 15.09 C ATOM 58 OH TYR A 7 3.766 0.589 10.291 1.00 14.39 O ATOM 59 OXT TYR A 7 11.358 2.999 7.612 1.00 17.49 O """) args = ["phenix.fmodel", "tmp_fmodel_fake_p1.pdb", "high_resolution=2", "output.file_name=tmp_fmodel_fake_p1.mtz"] result = easy_run.fully_buffered(args) assert (result.return_code != 0) and (len(result.stderr_lines) > 0) args.append("generate_fake_p1_symmetry=True") result = easy_run.fully_buffered(args).raise_if_errors() assert (result.return_code == 0) assert os.path.isfile("tmp_fmodel_fake_p1.mtz") from iotbx import crystal_symmetry_from_any symm = crystal_symmetry_from_any.extract_from("tmp_fmodel_fake_p1.mtz") assert (str(symm.space_group_info()) == "P 1") # FIXME this should fail but doesn't due to a bug in the program #args.append("reference_file=tmp_fmodel_fake_p1.mtz") #args.append("data_column_label=FMODEL,PHIFMODEL") #result = easy_run.fully_buffered(args).raise_if_errors() #print result.return_code print "OK"
def test1(self): from os.path import join, exists from libtbx import easy_run import os from uuid import uuid4 dirname ='tmp_%s' % uuid4().hex os.mkdir(dirname) os.chdir(dirname) assert exists(join(self.path, 'experiments.json')) # Call dials.integrate easy_run.fully_buffered([ 'dials.integrate', join(self.path, 'experiments.json'), 'profile.fitting=False', ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('integrated.pickle', 'rb')) print len(table) assert(len(table) == 751) # assert(len(table) == 764) assert('id' in table) for row in table: assert(row['id'] == 0) self.table = table print 'OK'
def exercise_compilation(): ucif_dist = libtbx.env.dist_path(module_name="ucif") antlr3_dist = libtbx.env.under_dist("ucif", "antlr3") os.environ["LIBTBX_UCIF"] = ucif_dist os.environ["LIBTBX_ANTLR3"] = antlr3_dist assert ucif_dist.find('"') < 0 if sys.platform == "win32": cmd = '"%s/examples/build_cif_parser.bat"' %ucif_dist ext = ".exe" else: cmd = '. "%s/examples/build_cif_parser.sh"' %ucif_dist ext = "" result = easy_run.fully_buffered(cmd) if result.return_code: if len(result.stderr_lines) > 0: raise RuntimeError, result.show_stderr() raise RuntimeError, result.show_stdout() assert os.path.exists("cif_parser"+ext) f = open_tmp_file(suffix=".cif") f.write(cif_string) f.close() cmd = 'cif_parser "%s"' %f.name cmd = os.path.join(".", cmd) r = easy_run.fully_buffered(cmd).raise_if_errors() assert r.stdout_lines[0].startswith("Congratulations!")
def exercise_2(pdb = "enk_gbr_e.pdb", hkl = "enk_gbr.mtz"): pdb = libtbx.env.find_in_repositories( relative_path="phenix_regression/pdb/enk_gbr_e.pdb", test=os.path.isfile) cmd = [ "phenix.refine", pdb, hkl, "main.number_of_macro_cycles=3", "strategy=group_adp", "fake_f_obs.structure_factors_accuracy.cos_sin_table=false", "main.target=ls", "group_b_iso.use_restraints=False", "group_b_iso.run_finite_differences_test=true", "output.write_maps=false", "output.write_map_coefficients=false", "output.write_geo_file=true", "output.write_def_file=false", "group_adp_refinement_mode=group_selection", "output.write_eff_file=false", "structure_factors_and_gradients_accuracy.cos_sin_table=false", "structure_factors_and_gradients_accuracy.algorithm=direct", "fake_f_obs.structure_factors_accuracy.algorithm=direct", "main.scattering_table=wk1995", "fake_f_obs.scattering_table=wk1995", "--overwrite", "refinement.input.xray_data.labels=FOBS", "main.bulk_solvent_and_scale=false", "group_adp_refinement_mode=group_selection", 'adp.group="chain A"', 'adp.group="chain B"', 'adp.group="chain C"', 'adp.group="chain D"', "output.prefix=ref2"] easy_run.fully_buffered(cmd).raise_if_errors()
def run_scons(): import libtbx.load_env from libtbx import easy_run from libtbx.command_line import scons print; print; print '-'*80 m = re.search(r"^(\d+ \. \d+ \. \d+) .*? \[\s*GCC\s* (\d+ \. \d+ \. \d+)", sys.version, re.X|re.M|re.S) print 'Python %s (compiled with gcc %s)' % m.groups() print print "** %s **" % libtbx.env.build_path.basename() libtbx.env.build_options.report() print print '-'*80 for compiler in ('gcc', 'clang',): print easy_run.fully_buffered('type %s' % compiler).stdout_lines[0] print '-'*80 os.chdir(os.environ['XCODE_CCTBX_BUILD']) sys.argv[1:] = os.environ['XCODE_SCONS_OPTIONS'].split() if os.environ.get('XCODE_SCONS_LIB_TARGET'): libs = os.environ['XCODE_SCONS_LIB_TARGET'].split() sys.argv.extend([ "lib/%s.so" % lib for lib in libs ]) print "warning: only library being built: %s" % ', '.join(libs) elif os.environ.get('XCODE_SCONS_PROGRAM_TARGET'): programs = os.environ['XCODE_SCONS_PROGRAM_TARGET'].split() sys.argv.extend(programs) print "warning: only program being built: %s" % ', '.join(programs) scons.run()
def test2(self): from os.path import join from libtbx import easy_run import os # Call dials.integrate easy_run.fully_buffered([ 'dials.integrate', join(self.path, 'experiments.json'), 'profile.fitting=False', 'integration.integrator=3d', ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('integrated.pickle', 'rb')) mask = table.get_flags(table.flags.integrated,all=False) assert(len(table) == 1996) assert(mask.count(True) == 1666) # assert(len(table) == 764) assert('id' in table) for row in table: assert(row['id'] == 0) self.table = table print 'OK'
def test_xds_ascii(self): from libtbx import easy_run from os.path import exists from libtbx.test_utils import approx_equal # Call dials.export easy_run.fully_buffered([ 'dials.export', 'summation=true', 'format=xds_ascii', self.experiments, self.reflections ]).raise_if_errors() assert exists("DIALS.HKL") psi_values = { (-9, 7, -10):153.430361, (-5, 11, -26):175.559441, (-4, 23, 24):129.468070, (2, 10, 20):147.947274 } for record in open('DIALS.HKL', 'r'): if record.startswith('!'): continue tokens = record.split() hkl = tuple(map(int, tokens[:3])) if not hkl in psi_values: continue psi = float(tokens[-1]) assert approx_equal(psi, psi_values[hkl], eps=0.1) print 'OK'
def tst_multiple_sweeps(self): from glob import glob import os from libtbx import easy_run from dxtbx.serialize import load # Find the image files image_files = sorted(glob(os.path.join(self.path, "centroid*.cbf"))) del image_files[4] # Delete filename to force two sweeps image_files = ' '.join(image_files) # provide mosflm beam centre to dials.import cmd = 'dials.import %s output.datablock=datablock_multiple_sweeps.json' % (image_files) try: easy_run.fully_buffered(cmd).raise_if_errors() assert False, "Expected exception" except Exception: pass cmd = 'dials.import %s output.datablock=datablock_multiple_sweeps.json allow_multiple_sweeps=True' % (image_files) easy_run.fully_buffered(cmd).raise_if_errors() assert os.path.exists("datablock_multiple_sweeps.json") datablock = load.datablock("datablock_multiple_sweeps.json")[0] imgset = datablock.extract_imagesets() assert len(imgset) == 2 print 'OK'
def test_sadabs(self): from libtbx import easy_run from os.path import exists from libtbx.test_utils import approx_equal # Call dials.export easy_run.fully_buffered([ 'dials.export', 'summation=true', 'format=sadabs', self.experiments, self.reflections ]).raise_if_errors() assert exists("integrated.sad") direction_cosines = { (-9, 7, -10):(0.51253, -0.72107, 0.84696, -0.68476, -0.14130, -0.10561), (-5, 11, -26):(0.51310, -0.62895, 0.84711, -0.59223, -0.13830, -0.50366), (-4, 23, 24):(0.51308, -0.60578, 0.84711, -0.31416, -0.13840, 0.73099), (2, 10, 20):(0.51239, -0.46605, 0.84693, -0.61521, -0.14204, 0.63586) } for record in open('integrated.sad', 'r'): record = record.replace('-', ' -') tokens = record.split() hkl = tuple(map(int, tokens[:3])) cosines = tuple(map(float, tokens[6:12])) if not hkl in direction_cosines: continue assert approx_equal(cosines, direction_cosines[hkl], eps=0.001) print 'OK'
def test_multi_lattice(self): from os.path import join from libtbx import easy_run import os dirname ='multi_sweep' os.mkdir(dirname) os.chdir(dirname) # Call dials.integrate easy_run.fully_buffered([ 'dials.integrate', join(self.integration_test_data, 'multi_lattice', 'experiments.json'), join(self.integration_test_data, 'multi_lattice', 'indexed.pickle') ]).raise_if_errors() import cPickle as pickle table = pickle.load(open('integrated.pickle', 'rb')) assert len(table) == 5604 # Check output contains from two lattices exp_id = list(set(table['id'])) assert len(exp_id) == 2 # Check both lattices have integrated reflections mask = table.get_flags(table.flags.integrated_prf) table = table.select(mask) exp_id = list(set(table['id'])) assert len(exp_id) == 2 print 'OK'
def tst_import_beam_centre(self): from glob import glob import os from libtbx import easy_run from dxtbx.serialize import load # Find the image files image_files = glob(os.path.join(self.path, "centroid*.cbf")) image_files = ' '.join(image_files) # provide mosflm beam centre to dials.import cmd = 'dials.import %s mosflm_beam_centre=100,200 output.datablock=mosflm_beam_centre.json' %image_files easy_run.fully_buffered(cmd) assert os.path.exists("mosflm_beam_centre.json") datablock = load.datablock("mosflm_beam_centre.json")[0] imgset = datablock.extract_imagesets()[0] beam_centre = imgset.get_detector()[0].get_beam_centre(imgset.get_beam().get_s0()) assert approx_equal(beam_centre, (200,100)) # provide an alternative datablock.json to get geometry from cmd = 'dials.import %s reference_geometry=mosflm_beam_centre.json output.datablock=mosflm_beam_centre2.json' %image_files easy_run.fully_buffered(cmd) assert os.path.exists("mosflm_beam_centre2.json") datablock = load.datablock("mosflm_beam_centre2.json")[0] imgset = datablock.extract_imagesets()[0] beam_centre = imgset.get_detector()[0].get_beam_centre(imgset.get_beam().get_s0()) assert approx_equal(beam_centre, (200,100)) print 'OK'
def run(self): if os.path.isfile(self.out_file): os.remove(self.out_file) if self.command is None: cmd = 'prime.run {}'.format(self.prime_file, self.out_file) else: cmd = self.command easy_run.fully_buffered(cmd, join_stdout_stderr=True)
def run(self): from os.path import join, exists from libtbx import easy_run import os # Run a few commands from stdin stdin_lines = [ "import template=%s" % join(self.path, "centroid_####.cbf"), "find_spots", "discover_better_experimental_model", "index", "refine_bravais_settings", "reindex solution=22", "refine", "goto 6", ] easy_run.fully_buffered( 'idials', stdin_lines=stdin_lines).raise_if_errors() print 'OK' # Check that state works stdin_lines = [ "refine", "integrate", "export", "goto 7", "integrate profile.fitting=False", "export", ] easy_run.fully_buffered('idials', stdin_lines=stdin_lines).raise_if_errors() print 'OK' # Check all the stuff we expect, exists assert exists("dials.state") assert exists("dials-1") assert exists("10_integrated.mtz") assert exists("12_integrated.mtz") assert exists("dials-1/1_import") assert exists("dials-1/2_find_spots") assert exists("dials-1/3_discover_better_experimental_model") assert exists("dials-1/4_index") assert exists("dials-1/5_refine_bravais_settings") assert exists("dials-1/6_reindex") assert exists("dials-1/7_refine") assert exists("dials-1/8_refine") assert exists("dials-1/9_integrate") assert exists("dials-1/10_export") assert exists("dials-1/11_integrate") assert exists("dials-1/12_export") print 'OK'
def run(self): from libtbx import easy_run from dials.array_family import flex # Call dials.merge_reflection_lists easy_run.fully_buffered([ 'dev.dials.show_extensions', ]).raise_if_errors() print 'OK'
def run(self): if os.path.isfile(self.out_file): os.remove(self.out_file) if self.command is None: cmd = 'prime.run {} > {}'.format(self.prime_file, self.out_file) else: cmd = self.command easy_run.fully_buffered(cmd, join_stdout_stderr=True) evt = AllDone(tp_EVT_ALLDONE, -1) wx.PostEvent(self.parent, evt)
def run(args): no_unzipsfx = len(args) > 0 and args[0] == "--no-unzipsfx" if no_unzipsfx: args = args[1:] single_dir = len(args) > 0 and args[0] == "--single_directory" if single_dir: args = args[1:] if len(args) < 2: from libtbx.utils import Usage import libtbx.load_env raise Usage( "%s [--no-unzipsfx] [--single_directory] bundle_prefix platform_string [addl_files...]" % libtbx.env.dispatcher_name ) if os.name == "nt": exe_suffix = ".exe" else: exe_suffix = "" import libtbx.path path_zip = libtbx.path.full_command_path(command="zip" + exe_suffix, search_first=["."]) if path_zip is None: raise RuntimeError("Fatal: zip executable not found.") bundle_prefix = args[0] if (single_dir) and (not os.path.isdir(bundle_prefix)): from libtbx.utils import Sorry raise Sorry("%s does not exist or is not a directory." % bundle_prefix) platform_string = args[1] addl_files = args[2:] zip_file_name = "%(bundle_prefix)s_%(platform_string)s.zip" % vars() open("autorun", "w").write(create_autorun(bundle_prefix, single_dir)) if single_dir: cmd = ('"%(path_zip)s" -q -r -z %(zip_file_name)s' + " %(bundle_prefix)s") % vars() else: cmd = ( '"%(path_zip)s" -q -r -z %(zip_file_name)s' + " %(bundle_prefix)s_sources" + " %(bundle_prefix)s_build" + " %(bundle_prefix)s_install_script.bat" ) % vars() for addl in addl_files: cmd += " " + addl cmd += " < autorun" print cmd from libtbx import easy_run easy_run.fully_buffered(command=cmd).raise_if_errors().show_stdout() if not no_unzipsfx: from libtbx.command_line import create_unzipsfx create_unzipsfx.create(zip_file_name=zip_file_name)
def run(self): from os.path import join, exists from libtbx import easy_run import os # Call dials.integrate easy_run.fully_buffered([ 'dials.plot_scan_varying_crystal', self.path ]).raise_if_errors() print 'OK'
def tst_from_xds_files(self): from os.path import abspath, exists, join from libtbx import easy_run # Import from the image files easy_run.fully_buffered([ 'dials.import_xds', 'input.method=experiment', 'output.filename=import_experiments.json', join(self.path)]).raise_if_errors() assert(exists("import_experiments.json")) print 'OK'
def run(self): '''Execute the script.''' runs = [ "r%04d" % r if isinstance(r, int) else r for r in self.params.striping.run ] if self.params.striping.run: print("processing runs " + ", ".join(runs)) if self.params.striping.rungroup: print("processing rungroups " + ", ".join( ["rg%03d" % rg for rg in self.params.striping.rungroup])) batch_chunks = allocate_chunks( self.params.striping.results_dir, self.params.striping.trial, rgs_selected=[ "rg%03d" % rg for rg in self.params.striping.rungroup ], respect_rungroup_barriers=self.params.striping. respect_rungroup_barriers, runs_selected=runs, stripe=self.params.striping.stripe, max_size=self.params.striping.chunk_size, integrated=self.params.combine_experiments.keep_integrated) self.dirname = os.path.join( self.params.striping.output_folder, "combine_experiments_t%03d" % self.params.striping.trial) self.intermediates = os.path.join(self.dirname, "intermediates") self.extracted = os.path.join(self.dirname, "final_extracted") for d in self.dirname, self.intermediates, self.extracted: if not os.path.isdir(d): os.mkdir(d) if self.params.striping.output_folder is None: self.params.striping.output_folder = os.getcwd() tag = "stripe" if self.params.striping.stripe else "chunk" all_commands = [] for batch, ch_list in six.iteritems(batch_chunks): for idx in range(len(ch_list)): chunk = ch_list[idx] # reset for this chunk/stripe self.filename = "t%03d_%s_%s%03d" % ( self.params.striping.trial, batch, tag, idx) self.command_sequence = [] # set up the file containing input expts and refls (logging) chunk_path = os.path.join(self.params.striping.output_folder, self.intermediates, self.filename) if os.path.isfile(chunk_path): os.remove(chunk_path) with open(chunk_path, "wb") as outfile: for i in (0, 1): # expts then refls outfile.write("\n".join(chunk[i]) + "\n") # set up the params for dials.combine_experiments custom_parts = [" input {"] for expt_path in chunk[0]: custom_parts.append(" experiments = %s" % expt_path) for refl_path in chunk[1]: custom_parts.append(" reflections = %s" % refl_path) custom_parts.append(" }") self.set_up_section("combine_experiments", "dials.combine_experiments", clustering=False, custom_parts=custom_parts) # refinement of the grouped experiments self.set_up_section("refinement", "dials.refine", clustering=self.clustering) # refinement of the grouped experiments self.set_up_section("recompute_mosaicity", "cctbx.xfel.recompute_mosaicity", clustering=self.clustering) # reintegration if self.params.reintegration.enable: custom_parts = [ " integration.mp.nproc = %d" % self.params.mp.nproc ] self.set_up_section("reintegration", "dials.integrate", custom_parts=custom_parts, clustering=self.clustering) # extract results to integration pickles for merging if self.params.postprocessing.enable: lambda_diff_str = lambda diff_str: (diff_str % \ (os.path.join("..", "final_extracted"))).replace("ITER", "%04d") self.set_up_section("postprocessing", "cctbx.xfel.frame_extractor", lambda_diff_str=lambda_diff_str, clustering=self.clustering) # submit queued job from appropriate directory os.chdir(self.intermediates) command = " && ".join(self.command_sequence) if self.params.combine_experiments.clustering.dendrogram: easy_run.fully_buffered( command).raise_if_errors().show_stdout() else: submit_folder = os.path.join( self.params.striping.output_folder, self.intermediates) submit_path = os.path.join(submit_folder, "combine_%s.sh" % self.filename) submit_command = get_submit_command_chooser( command, submit_path, self.intermediates, self.params.mp, log_name=os.path.splitext( os.path.basename(submit_path))[0] + ".out", err_name=os.path.splitext( os.path.basename(submit_path))[0] + ".err", root_dir=submit_folder) all_commands.append(submit_command) if not self.params.striping.dry_run: print("executing command: %s" % submit_command) try: easy_run.fully_buffered( submit_command).raise_if_errors().show_stdout( ) except Exception as e: if not "Warning: job being submitted without an AFS token." in str( e): raise e return all_commands
def run_command( command, verbose=0, buffered=True, log_file_name=None, stdout_file_name=None, result_file_names=[], show_diff_log_stdout=False, sorry_expected=False, join_stdout_stderr=False): """\ This function starts another process to run command, with some pre-call and post-call processing. Before running command, the expected output files are removed: log_file_name stdout_file_name result_file_names After command is finished, log_file_name and stdout_file_name are scanned for Traceback and Sorry. An exception is raised if there are any matches. sorry_expected=True suppresses the scanning for Sorry. With buffered=True easy_run.fully_buffered() is used. If there is any output to stderr of the child process, an exception is raised. The run_command() return value is the result of the easy_run.fully_buffered() call. With buffered=False easy_run.call() is used. I.e. stdout and stderr of the command are connected to stdout and stderr of the parent process. stderr is not checked. The run_command() return value is None. It is generally best to use buffered=True, and even better not to use this function at all if command is another Python script. It is better to organize the command script such that it can be called directly from within the same Python process running the unit tests. """ assert verbose >= 0 if (verbose > 0): print command print show_command_if_error = None else: show_command_if_error = command all_file_names = [log_file_name, stdout_file_name] + result_file_names for file_name in all_file_names: if (file_name is None): continue if (os.path.isfile(file_name)): os.remove(file_name) if (os.path.exists(file_name)): raise RunCommandError( "Unable to remove file: %s" % show_string(file_name)) if (buffered): sys.stdout.flush() sys.stderr.flush() cmd_result = easy_run.fully_buffered( command=command, join_stdout_stderr=join_stdout_stderr) if (len(cmd_result.stderr_lines) != 0): if (verbose == 0): print command print print "\n".join(cmd_result.stdout_lines) cmd_result.raise_if_errors() _check_command_output( lines=cmd_result.stdout_lines, show_command_if_error=show_command_if_error, sorry_expected=sorry_expected) else: easy_run.call(command=command) cmd_result = None for file_name in [log_file_name, stdout_file_name]: if (file_name is None or not os.path.isfile(file_name)): continue _check_command_output( file_name=file_name, show_command_if_error=show_command_if_error, sorry_expected=sorry_expected) for file_name in all_file_names: if (file_name is None): continue if (not os.path.isfile(file_name)): raise RunCommandError( "Missing output file: %s" % show_string(file_name)) if (verbose > 1 and cmd_result is not None): print "\n".join(cmd_result.stdout_lines) print if ( show_diff_log_stdout and log_file_name is not None and stdout_file_name is not None): if (verbose > 0): print "diff %s %s" % (show_string(log_file_name), show_string(stdout_file_name)) print if (show_diff(open(log_file_name).read(), open(stdout_file_name).read())): introspection.show_stack( frames_back=1, reverse=True, prefix="INFO_LOG_STDOUT_DIFFERENCE: ") print "ERROR_LOG_STDOUT_DIFFERENCE" sys.stdout.flush() return cmd_result
def __call__(O, file_info): from libtbx import easy_run from libtbx.str_utils import show_string from libtbx.test_utils import show_diff from six.moves import StringIO import os.path as op import sys opts = O.opts file_name, io_infos = file_info if (opts.verbose): print(file_name) file_path = op.join(O.test_valid, file_name) top_procedures = top_procedures_by_file_name.get(file_name) common_equivalence_simple_list = [ set(common_equivalence_simple_by_file_name.get(file_name, [])) ] if (len(common_equivalence_simple_list[0]) != 0): common_equivalence_simple_list.append([]) for i_ces,common_equivalence_simple in \ enumerate(common_equivalence_simple_list): common_report_stringio = StringIO() try: lines = fable.cout.process( file_names=[file_path], top_procedures=top_procedures, dynamic_parameters=dynamic_parameters_by_file_name.get( file_name), common_equivalence_simple=common_equivalence_simple, common_report_stringio=common_report_stringio) except Exception: if (not opts.keep_going): raise print("\nEXCEPTION: fable.cout.process([%s])\n" % file_name) return 1 have_simple_equivalence = ( "\n".join(lines).find(" // SIMPLE EQUIVALENCE") >= 0) if (len(common_equivalence_simple) != 0): assert have_simple_equivalence else: assert not have_simple_equivalence assert file_name.endswith(".f") base_name = file_name[:-2] if (len(common_equivalence_simple_list) != 1): base_name += "_alt%d" % i_ces fem_cpp = base_name + "_fem.cpp" fem_exe_name = fem_cpp[:-4] + O.comp_env.exe_suffix print("\n".join(lines), file=open(fem_cpp, "w")) if (opts.ifort): ifort_exe_name = base_name + "_ifort" ifort_cmd = "ifort -diag-disable 7000 -o %s %s" % ( ifort_exe_name, show_string(file_path)) else: ifort_exe_name = None ifort_cmd = None if (opts.dry_run): return 0 # n_failures = [0] def handle_exception(e): n_failures[0] += 1 if (not opts.keep_going): raise print() print(str(e)) print() sys.stdout.flush() # class BuildError(RuntimeError): pass try: O.comp_env.build( link=True, file_name_cpp=fem_cpp, exe_name=fem_exe_name, disable_warnings=(file_name in file_names_disable_warnings), show_command=opts.verbose, Error=BuildError) except BuildError as e: handle_exception(e) fem_exe_name = None # if (ifort_cmd is not None): if (opts.verbose): print(ifort_cmd) buffers = easy_run.fully_buffered(command=ifort_cmd) try: buffers.raise_if_errors_or_output(Error=BuildError) except BuildError as e: handle_exception(e) ifort_exe_name = None # for info in io_infos: if (info.skip_run): if (opts.verbose): print("Skipping run:", file_name) continue if (len(info.inp_lines) != 0 and opts.verbose): print(" number of input lines:", len(info.inp_lines)) sys.stdout.flush() for exe_name in [fem_exe_name, ifort_exe_name]: if (exe_name is None): continue cmd = cmd0 = op.join(".", exe_name) if (opts.valgrind): cmd = "valgrind " + cmd if (opts.verbose): print(cmd) sys.stdout.flush() join_stdout_stderr = (opts.valgrind or (file_name in file_names_join_stdout_stderr)) buffers = easy_run.fully_buffered( command=cmd, stdin_lines=info.inp_lines, join_stdout_stderr=join_stdout_stderr) if (not join_stdout_stderr): class ExeError(RuntimeError): pass try: buffers.raise_if_errors(Error=ExeError) except ExeError as e: handle_exception(e) buffers = None if (buffers is not None): text = "\n".join(buffers.stdout_lines) if (opts.valgrind): print(text) else: def check(text): if (file_name == "intrinsics_extra.f"): check_intrinsics_extra(text) return if (file_name == "sf.f"): text = text.replace( " -0.620088", " -0.620087") elif (file_name == "unformatted_experiments.f" ): if (sys.byteorder == "big"): text = text \ .replace( " 1234 5678", " 5678 1234") \ .replace( " 18558553691448", " 23330262356193") have_diffs = show_diff( text, "\n".join(info.out_lines)) def assert_not_have_diffs(): if (opts.keep_going): print( "WARNING: --keep-going after show_diff:", exe_name) else: assert not have_diffs if (have_diffs): if (exe_name is fem_exe_name): assert_not_have_diffs() elif (exe_name is ifort_exe_name): if (not info.ifort_diff_behavior and not info. ifort_diff_floating_point_format ): assert_not_have_diffs() else: raise AssertionError check(text) def run_with_args(args): cmda = cmd0 + " " + args if (opts.verbose): print(cmda) sys.stdout.flush() result = easy_run.fully_buffered( command=cmda, join_stdout_stderr=True) if (opts.valgrind): cmda = "valgrind " + cmda if (opts.verbose): print(cmda) sys.stdout.flush() buffers = easy_run.fully_buffered( command=cmda, join_stdout_stderr=True) print("\n".join(buffers.stdout_lines)) return result if (file_name == "read_lines.f"): exercise_end_of_line(exe_name=exe_name, verbose=opts.verbose) elif (file_name == "dynamic_parameters_1.f"): buffers = run_with_args("--fem-dynamic-parameters=5") assert not show_diff( buffers.stdout_lines, """\ 14 15 16 17 18 19 20 21 22 23 """) buffers = run_with_args("--fem-dynamic-parameters=5,6") assert buffers.stdout_lines[0].endswith( "Too many --fem-dynamic-parameters fields" " (given: 2, max. expected: 1)") buffers = run_with_args("--fem-dynamic-parameters=x") assert buffers.stdout_lines[0].endswith( 'Invalid --fem-dynamic-parameters field (field 1): "x"' ) elif (file_name == "intrinsics_iargc_getarg.f"): buffers = run_with_args("D rP uWq") assert not show_diff( buffers.stdout_lines, "\n".join([ "A", "D ", "rP ", "uWq ", "B", "uWq ", "rP ", "D ", "C", "rP ", "uWq ", "D " ]) + "\n") # return n_failures[0]
def run(self): from os.path import join, exists from libtbx import easy_run assert(exists(join(self.path, "datablock.json"))) input_filename = join(self.path, "datablock.json") # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, ]).raise_if_errors() assert(exists("mask.pickle")) print 'OK' # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, 'output.mask=mask2.pickle', 'output.datablock=masked_datablock.json', 'untrusted.rectangle=100,200,100,200' ]).raise_if_errors().show_stdout() assert(exists("mask2.pickle")) assert(exists("masked_datablock.json")) from dxtbx.serialize import load datablocks = load.datablock("masked_datablock.json") imageset = datablocks[0].extract_imagesets()[0] import os assert imageset.external_lookup.mask.filename == os.path.join( os.path.abspath(os.getcwd()), 'mask2.pickle') print 'OK' # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, 'output.mask=mask3.pickle', 'untrusted.circle=100,100,10' ]).raise_if_errors() assert(exists("mask3.pickle")) print 'OK' # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, 'output.mask=mask4.pickle', 'resolution_range=2,3', ]).raise_if_errors() assert(exists("mask4.pickle")) print 'OK' # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, 'output.mask=mask5.pickle', 'd_min=3', 'd_max=2', ]).raise_if_errors() assert(exists("mask5.pickle")) print 'OK' # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, 'output.mask=mask6.pickle', '\'ice_rings{filter=True;d_min=2}\'', ]).raise_if_errors() assert(exists("mask6.pickle")) print 'OK' # Call dials.integrate easy_run.fully_buffered([ 'dials.generate_mask', input_filename, 'output.mask=mask3.pickle', 'untrusted.polygon=100,100,100,200,200,200,200,100' ]).raise_if_errors() assert(exists("mask3.pickle")) print 'OK'
def run(self): command = '{} {}'.format(self.viewer, self.file_string) easy_run.fully_buffered(command)
def run_probe_clashscore(self, pdb_string): self.n_clashes = 0 self.n_clashes_b_cutoff = 0 self.clashscore_b_cutoff = None self.bad_clashes = [] self.clashscore = None self.n_atoms = 0 self.natoms_b_cutoff = 0 probe_out = easy_run.fully_buffered(self.probe_txt, stdin_lines=pdb_string) if (probe_out.return_code != 0): raise RuntimeError("Probe crashed - dumping stderr:\n%s" % "\n".join(probe_out.stderr_lines)) probe_unformatted = probe_out.stdout_lines # Debugging facility, do not remove! # import random # tempdir = "tmp_for_probe_debug_%d" % random.randint(1000,9999) # while os.path.isdir(tempdir): # tempdir = "tmp_for_probe_debug_%d" % random.randint(1000,9999) # os.mkdir(tempdir) # print "Dumping info to %s" % tempdir # with open(tempdir + os.sep + 'model.pdb', 'w') as f: # f.write(pdb_string) # with open(tempdir + os.sep + 'probe_out.txt', 'w') as f: # f.write('\n'.join(probe_unformatted)) if not self.fast: temp = self.process_raw_probe_output(probe_unformatted) self.n_clashes = len(temp) # XXX Warning: one more probe call here printable_probe_out = easy_run.fully_buffered( self.full_probe_txt, stdin_lines=pdb_string) self.probe_unformatted = "\n".join( printable_probe_out.stdout_lines) else: self.n_clashes = self.get_condensed_clashes(probe_unformatted) # getting number of atoms from probe probe_info = easy_run.fully_buffered( self.probe_atom_txt, stdin_lines=pdb_string) #.raise_if_errors().stdout_lines err = probe_info.format_errors_if_any() if err is not None and err.find("No atom data in input.") > -1: return #if (len(probe_info) == 0): # raise RuntimeError("Empty PROBE output.") n_atoms = 0 for line in probe_info.stdout_lines: try: dump, n_atoms = line.split(":") except KeyboardInterrupt: raise except ValueError: pass # something else (different from expected) got into output self.n_atoms = int(n_atoms) if self.n_atoms == 0: self.clashscore = 0.0 else: self.clashscore = (self.n_clashes * 1000) / self.n_atoms if not self.fast: # The rest is not necessary, we already got clashscore if self.b_factor_cutoff is not None: clashes_b_cutoff = 0 for clash_obj in temp: if clash_obj.max_b_factor < self.b_factor_cutoff: clashes_b_cutoff += 1 self.n_clashes_b_cutoff = clashes_b_cutoff used = [] for clash_obj in sorted(temp): test_key = clash_obj.id_str_no_atom_name() test_key = clash_obj.id_str() if test_key not in used: used.append(test_key) self.bad_clashes.append(clash_obj) if self.probe_atom_b_factor is not None: probe_info_b_factor = easy_run.fully_buffered( self.probe_atom_b_factor, stdin_lines=pdb_string).raise_if_errors().stdout_lines for line in probe_info_b_factor: dump_b, natoms_b_cutoff = line.split(":") self.natoms_b_cutoff = int(natoms_b_cutoff) self.clashscore_b_cutoff = None if self.natoms_b_cutoff == 0: self.clashscore_b_cutoff = 0.0 else: self.clashscore_b_cutoff = \ (self.n_clashes_b_cutoff*1000) / self.natoms_b_cutoff
dials_regression = libtbx.env.dist_path('dials_regression') except KeyError, e: print 'FAIL: dials_regression not configured' exit(0) path = os.path.join(dials_regression, "experiment_test_data/experiment_1.json") newpath = os.path.join(os.getcwd(), 'experiments.json') shutil.copyfile(path, newpath) for line in fileinput.FileInput(newpath, inplace=True): if '$DIALS_REGRESSION' in line: line = line.replace('$DIALS_REGRESSION', dials_regression) print line cmd = "dials.export format=mosflm %s" % newpath result = easy_run.fully_buffered(cmd).raise_if_errors() assert os.path.exists("mosflm/index.mat") with open("mosflm/index.mat", "rb") as f: lines = f.read() assert not show_diff( lines, """\ -0.01210200 -0.01954526 0.00309519 -0.00416605 -0.00080573 -0.02427340 0.01931593 -0.01241956 -0.00329641 0.000 0.000 0.000 -0.52228050 -0.84350975 0.12535704 -0.17980379 -0.03477015 -0.98308781 0.83360283 -0.53598726 -0.13350648 42.2717 42.2720 39.6704 90.0001 89.9993 89.9998 0.000 0.000 0.000 """)
def refine_expanding(params, merged_scope, combine_phil): assert params.start_at_hierarchy_level == 0 if params.rmsd_filter.enable: input_name = "filtered" command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s" command = command % ("%s_combined_experiments.json" % params.tag, "%s_combined_reflections.pickle" % params.tag, "%s_filtered_experiments.json" % params.tag, "%s_filtered_reflections.pickle" % params.tag) command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier print command result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() else: input_name = "combined" # -------------------------- if params.panel_filter is not None: from libtbx import easy_pickle print "Filtering out all reflections except those on panels %s" % ( ", ".join(["%d" % p for p in params.panel_filter])) combined_path = "%s_combined_reflections.pickle" % params.tag data = easy_pickle.load(combined_path) sel = None for panel_id in params.panel_filter: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print "Retaining", len( data.select(sel)), "out of", len(data), "reflections" easy_pickle.dump(combined_path, data.select(sel)) # ---------------------------------- # this is the order to refine the CSPAD in steps = {} steps[0] = [2, 3] steps[1] = steps[0] + [0, 1] steps[2] = steps[1] + [14, 15] steps[3] = steps[2] + [6, 7] steps[4] = steps[3] + [4, 5] steps[5] = steps[4] + [12, 13] steps[6] = steps[5] + [8, 9] steps[7] = steps[6] + [10, 11] for s, panels in steps.iteritems(): rest = [] for p in panels: rest.append(p + 16) rest.append(p + 32) rest.append(p + 48) panels.extend(rest) levels = {0: (0, 1)} # levels 0 and 1 for i in range(7): levels[i + 1] = (2, ) # level 2 previous_step_and_level = None for j in range(8): from libtbx import easy_pickle print "Filtering out all reflections except those on panels %s" % ( ", ".join(["%d" % p for p in steps[j]])) combined_path = "%s_%s_reflections.pickle" % (params.tag, input_name) output_path = "%s_reflections_step%d.pickle" % (params.tag, j) data = easy_pickle.load(combined_path) sel = None for panel_id in steps[j]: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print "Retaining", len( data.select(sel)), "out of", len(data), "reflections" easy_pickle.dump(output_path, data.select(sel)) for i in levels[j]: print "Step", j, "refining at hierarchy level", i refine_phil_file = "%s_refine_step%d_level%d.phil" % (params.tag, j, i) if i == 0: if params.refine_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Tau1" # fix detector rotz else: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Tau1" # fix detector rotz, distance if params.flat_refinement: diff_phil += ",Tau2,Tau3" # Also fix x and y rotations diff_phil += "\n" if params.refine_energy: diff_phil += "refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine else: # Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always if params.flat_refinement and params.flat_refinement_with_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level elif params.flat_refinement: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation else: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything if previous_step_and_level is None: command = "dials.refine %s %s_%s_experiments.json %s_reflections_step%d.pickle"%( \ refine_phil_file, params.tag, input_name, params.tag, j) else: p_step, p_level = previous_step_and_level if p_step == j: command = "dials.refine %s %s_refined_experiments_step%d_level%d.json %s_refined_reflections_step%d_level%d.pickle"%( \ refine_phil_file, params.tag, p_step, p_level, params.tag, p_step, p_level) else: command = "dials.refine %s %s_refined_experiments_step%d_level%d.json %s_reflections_step%d.pickle"%( \ refine_phil_file, params.tag, p_step, p_level, params.tag, j) diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i output_experiments = "%s_refined_experiments_step%d_level%d.json" % ( params.tag, j, i) command += " output.experiments=%s output.reflections=%s_refined_reflections_step%d_level%d.pickle"%( \ output_experiments, params.tag, j, i) scope = merged_scope.fetch(parse(diff_phil)) f = open(refine_phil_file, 'w') f.write(refine_scope.fetch_diff(scope).as_str()) f.close() print command result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() # In expanding mode, if using flat refinement with distance, after having refined this step as a block, unrefined # panels will have been left behind. Read back the new metrology, compute the shift applied to the panels refined # in this step,and apply that shift to the unrefined panels in this step if params.flat_refinement and params.flat_refinement_with_distance and i > 0: from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentListDumper from xfel.command_line.cspad_detector_congruence import iterate_detector_at_level, iterate_panels from scitbx.array_family import flex from scitbx.matrix import col from libtbx.test_utils import approx_equal experiments = ExperimentListFactory.from_json_file( output_experiments, check_format=False) assert len(experiments.detectors()) == 1 detector = experiments.detectors()[0] # Displacements: deltas along the vector normal to the detector displacements = flex.double() # Iterate through the panel groups at this level for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): # Were there panels refined in this step in this panel group? if params.panel_filter: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) if list( detector).index(panel) in params.panel_filter ] else: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) ] if not any(test): continue # Compute the translation along the normal of this panel group. This is defined as distance in dials.refine displacements.append( col(panel_group.get_local_fast_axis()).cross( col(panel_group.get_local_slow_axis())).dot( col(panel_group.get_local_origin()))) # Even though the panels are constrained to move the same amount, there is a bit a variation. stats = flex.mean_and_variance(displacements) displacement = stats.mean() print "Average displacement along normals: %f +/- %f" % ( stats.mean(), stats.unweighted_sample_standard_deviation()) # Verify the variation isn't significant for k in range(1, len(displacements)): assert approx_equal(displacements[0], displacements[k]) # If all of the panel groups in this level moved, no need to do anything. if len(displacements) != len( list( iterate_detector_at_level(detector.hierarchy(), 0, i))): for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): if params.panel_filter: test = [ list(detector).index(panel) in steps[j] and list(detector).index(panel) in params.panel_filter for panel in iterate_panels(panel_group) ] else: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) ] # If any of the panels in this panel group moved, no need to do anything if any(test): continue # None of the panels in this panel group moved in this step, so need to apply displacement from other panel # groups at this level fast = col(panel_group.get_local_fast_axis()) slow = col(panel_group.get_local_slow_axis()) ori = col(panel_group.get_local_origin()) normal = fast.cross(slow) panel_group.set_local_frame( fast, slow, (ori.dot(fast) * fast) + (ori.dot(slow) * slow) + (normal * displacement)) # Check the new displacements. Should be the same across all panels. displacements = [] for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): displacements.append( col(panel_group.get_local_fast_axis()).cross( col(panel_group.get_local_slow_axis())).dot( col(panel_group.get_local_origin()))) for k in range(1, len(displacements)): assert approx_equal(displacements[0], displacements[k]) dump = ExperimentListDumper(experiments) dump.as_json(output_experiments) previous_step_and_level = j, i output_geometry(params)
def query(self, submission_id): if self.queueing_system in ["mpi", "lsf"]: result = easy_run.fully_buffered(command=self.command % \ (submission_id, submission_id)) elif self.queueing_system == 'pbs': result = easy_run.fully_buffered(command=self.command % submission_id) elif self.queueing_system == 'local': import psutil try: process = psutil.Process(int(submission_id)) except psutil.NoSuchProcess: return "DONE" statuses = [ p.status() for p in [process] + process.children(recursive=True) ] if 'running' in statuses: return "RUN" if 'sleeping' in statuses: return "RUN" # zombie jobs can be left because the GUI process that forked them is still running if len(statuses) == 1 and statuses[0] == 'zombie': return "DONE" return ", ".join(statuses) elif self.queueing_system == 'slurm' or self.queueing_system == "shifter": # The current implementation of the shifter mp method assumes that we're # running on NERSC's systems => jobs should be tracked using the _slurm_ # submission tracker. result = easy_run.fully_buffered(command=self.command % submission_id) if len(result.stdout_lines) == 0: return 'UNKWN' status = result.stdout_lines[0].strip().rstrip('+') statuses = { 'COMPLETED': 'DONE', 'COMPLETING': 'RUN', 'FAILED': 'EXIT', 'PENDING': 'PEND', 'PREEMPTED': 'SUSP', 'RUNNING': 'RUN', 'SUSPENDED': 'SUSP', 'STOPPED': 'SUSP', 'CANCELLED': 'EXIT', } return statuses[status] if status in statuses else 'UNKWN' elif self.queueing_system == 'htcondor': # (copied from the man page) # H = on hold, R = running, I = idle (waiting for a machine to execute on), C = completed, # X = removed, S = suspended (execution of a running job temporarily suspended on execute node), # < = transferring input (or queued to do so), and > = transferring output (or queued to do so). statuses = { 'H': 'HOLD', 'R': 'RUN', 'I': 'PEND', 'C': 'DONE', 'X': 'DONE', 'S': 'SUSP', '<': 'RUN', '>': 'RUN' } for c in [self.command1, self.command2]: result = easy_run.fully_buffered(command=c % submission_id) if len(result.stdout_lines) != 1 or len( result.stdout_lines[0]) == 0: continue status = result.stdout_lines[0].split()[5] return statuses[status] if status in statuses else 'UNKWN' return 'ERR' status = "\n".join(result.stdout_lines) error = "\n".join(result.stderr_lines) if error != "" and not "Warning: job being submitted without an AFS token." in error: if "not found" in error: return "ERR" else: return error else: return status
def refine_hierarchical(params, merged_scope, combine_phil): if params.panel_filter is not None: from libtbx import easy_pickle print "Filtering out all reflections except those on panels %s" % ( ", ".join(["%d" % p for p in params.panel_filter])) combined_path = "%s_combined_reflections.pickle" % params.tag data = easy_pickle.load(combined_path) sel = None for panel_id in params.panel_filter: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print "Retaining", len( data.select(sel)), "out of", len(data), "reflections" easy_pickle.dump(combined_path, data.select(sel)) for i in range(params.start_at_hierarchy_level, params.refine_to_hierarchy_level + 1): if params.rmsd_filter.enable: input_name = "filtered" else: if i == params.start_at_hierarchy_level: input_name = "combined" else: input_name = "refined" if params.rmsd_filter.enable: command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s" if i == params.start_at_hierarchy_level: command = command % ( "%s_combined_experiments.json" % params.tag, "%s_combined_reflections.pickle" % params.tag, "%s_filtered_experiments.json" % params.tag, "%s_filtered_reflections.pickle" % params.tag) else: command = command % ("%s_refined_experiments_level%d.json" % (params.tag, i - 1), "%s_refined_reflections_level%d.pickle" % (params.tag, i - 1), "%s_filtered_experiments_level%d.json" % (params.tag, i - 1), "%s_filtered_reflections_level%d.pickle" % (params.tag, i - 1)) command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier print command result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() print "Refining at hierarchy level", i refine_phil_file = "%s_refine_level%d.phil" % (params.tag, i) if i == 0: fix_list = ['Tau1'] # fix detector rotz if not params.refine_distance: fix_list.append('Dist') if params.flat_refinement: fix_list.extend(['Tau2', 'Tau3']) diff_phil = "refinement.parameterisation.detector.fix_list=%s\n" % ",".join( fix_list) if params.refine_energy: diff_phil += " refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine else: # Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always if params.flat_refinement and params.flat_refinement_with_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level elif params.flat_refinement: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation else: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything if i == params.start_at_hierarchy_level: command = "dials.refine %s %s_%s_experiments.json %s_%s_reflections.pickle" % ( refine_phil_file, params.tag, input_name, params.tag, input_name) else: command = "dials.refine %s %s_%s_experiments_level%d.json %s_%s_reflections_level%d.pickle" % ( refine_phil_file, params.tag, input_name, i - 1, params.tag, input_name, i - 1) diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i command += " output.experiments=%s_refined_experiments_level%d.json output.reflections=%s_refined_reflections_level%d.pickle"%( \ params.tag, i, params.tag, i) scope = merged_scope.fetch(parse(diff_phil)) f = open(refine_phil_file, 'w') f.write(refine_scope.fetch_diff(scope).as_str()) f.close() print command result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() output_geometry(params)
def run_probe_clashscore(self, pdb_string): probe_out = easy_run.fully_buffered(self.probe_txt, stdin_lines=pdb_string) if (probe_out.return_code != 0): raise RuntimeError("Probe crashed - dumping stderr:\n%s" % "\n".join(probe_out.stderr_lines)) probe_unformatted = probe_out.stdout_lines printable_probe_out = easy_run.fully_buffered(self.full_probe_txt, stdin_lines=pdb_string) self.probe_unformatted = "\n".join(printable_probe_out.stdout_lines) temp = self.process_raw_probe_output(probe_unformatted) self.n_clashes = len(temp) if self.b_factor_cutoff is not None: clashes_b_cutoff = 0 for clash_obj in temp: if clash_obj.max_b_factor < self.b_factor_cutoff: clashes_b_cutoff += 1 self.n_clashes_b_cutoff = clashes_b_cutoff used = [] self.bad_clashes = [] for clash_obj in sorted(temp): test_key = clash_obj.id_str_no_atom_name() test_key = clash_obj.id_str() if test_key not in used: used.append(test_key) self.bad_clashes.append(clash_obj) probe_info = easy_run.fully_buffered( self.probe_atom_txt, stdin_lines=pdb_string) #.raise_if_errors().stdout_lines err = probe_info.format_errors_if_any() if err is not None and err.find("No atom data in input.") > -1: self.clashscore = None self.clashscore_b_cutoff = None return #if (len(probe_info) == 0) : # raise RuntimeError("Empty PROBE output.") self.n_atoms = 0 n_atoms = 0 for line in probe_info.stdout_lines: processed = False try: dump, n_atoms = line.split(":") except KeyboardInterrupt: raise except ValueError: pass # something else (different from expected) got into output self.n_atoms = int(n_atoms) self.natoms_b_cutoff = None if self.probe_atom_b_factor is not None: probe_info_b_factor = easy_run.fully_buffered( self.probe_atom_b_factor, stdin_lines=pdb_string).raise_if_errors().stdout_lines for line in probe_info_b_factor: dump_b, natoms_b_cutoff = line.split(":") self.natoms_b_cutoff = int(natoms_b_cutoff) if self.n_atoms == 0: clashscore = 0.0 else: clashscore = (self.n_clashes * 1000) / self.n_atoms self.clashscore = clashscore clashscore_b_cutoff = None if self.natoms_b_cutoff is not None and self.natoms_b_cutoff == 0: clashscore_b_cutoff = 0.0 elif self.natoms_b_cutoff is not None: clashscore_b_cutoff = \ (self.n_clashes_b_cutoff*1000) / self.natoms_b_cutoff self.clashscore_b_cutoff = clashscore_b_cutoff
def check_and_add_hydrogen(pdb_hierarchy=None, file_name=None, nuclear=False, keep_hydrogens=True, verbose=False, model_number=0, n_hydrogen_cut_off=0, time_limit=120, allow_multiple_models=True, crystal_symmetry=None, do_flips=False, log=None): """ If no hydrogens present, force addition for clashscore calculation. Use REDUCE to add the hydrogen atoms. Args: pdb_hierarchy : pdb hierarchy file_name (str): pdb file name nuclear (bool): When True use nuclear cloud x-H distances and vdW radii, otherwise use electron cloud x-H distances and vdW radii keep_hydrogens (bool): when True, if there are hydrogen atoms, keep them verbose (bool): verbosity of printout model_number (int): the number of model to use time_limit (int): limit the time it takes to add hydrogen atoms n_hydrogen_cut_off (int): when number of hydrogen atoms < n_hydrogen_cut_off force keep_hydrogens tp True allow_multiple_models (bool): Allow models that contain more than one model crystal_symmetry : must provide crystal symmetry when using pdb_hierarchy Returns: (str): PDB string (bool): True when PDB string was updated """ if not log: log = sys.stdout if file_name: pdb_inp = iotbx.pdb.input(file_name=file_name) pdb_hierarchy = pdb_inp.construct_hierarchy() cryst_sym = pdb_inp.crystal_symmetry() elif not allow_multiple_models: assert crystal_symmetry cryst_sym = crystal_symmetry else: cryst_sym = None assert pdb_hierarchy assert model_number < len(pdb_hierarchy.models()) models = pdb_hierarchy.models() if (len(models) > 1) and (not allow_multiple_models): raise Sorry( "When using CCTBX clashscore, provide only a single model.") model = models[model_number] r = iotbx.pdb.hierarchy.root() mdc = model.detached_copy() r.append_model(mdc) if keep_hydrogens: elements = r.atoms().extract_element() # strangely the elements can have a space when coming from phenix.clashscore # but no space when coming from phenix.molprobity h_count = elements.count('H') if h_count <= n_hydrogen_cut_off: h_count += elements.count(' H') if h_count <= n_hydrogen_cut_off: h_count += elements.count('D') if h_count <= n_hydrogen_cut_off: h_count += elements.count(' D') if h_count > n_hydrogen_cut_off: has_hd = True else: has_hd = False if not has_hd: if verbose: print >> log, "\nNo H/D atoms detected - forcing hydrogen addition!\n" keep_hydrogens = False import libtbx.load_env has_reduce = libtbx.env.has_module(name="reduce") # add hydrogen if needed if has_reduce and (not keep_hydrogens): # set reduce running parameters build = "phenix.reduce -oh -his -flip -keep -allalt -limit{}" if not do_flips: build += " -pen9999" if nuclear: build += " -nuc -" else: build += " -" build = build.format(time_limit) trim = "phenix.reduce -quiet -trim -" stdin_lines = r.as_pdb_string(cryst_sym) clean_out = easy_run.fully_buffered(trim, stdin_lines=stdin_lines) if (clean_out.return_code != 0): msg_str = "Reduce crashed with command '%s' - dumping stderr:\n%s" raise Sorry(msg_str % (trim, "\n".join(clean_out.stderr_lines))) build_out = easy_run.fully_buffered(build, stdin_lines=clean_out.stdout_lines) if (build_out.return_code != 0): msg_str = "Reduce crashed with command '%s' - dumping stderr:\n%s" raise Sorry(msg_str % (build, "\n".join(build_out.stderr_lines))) reduce_str = '\n'.join(build_out.stdout_lines) return reduce_str, True else: if not has_reduce: msg = 'phenix.reduce could not be detected on your system.\n' msg += 'Cannot add hydrogen to PDB file' print >> log, msg return r.as_pdb_string(cryst_sym), False
import inspect import os import re import sys import warnings from libtbx import cpp_function_name symbol_not_found_pat = re.compile( r"[Ss]ymbol[ ]not[ ]found: \s* (\w+) $", re.X | re.M | re.S) python_libstdcxx_so = None if (sys.platform.startswith("linux")): from libtbx import easy_run for line in easy_run.fully_buffered( command='/usr/bin/ldd "%s"' % sys.executable).stdout_lines: if (line.strip().startswith("libstdc++.so")): python_libstdcxx_so = line.split()[0] break def import_ext(name, optional=False): components = name.split(".") if (len(components) > 1): __import__(".".join(components[:-1])) previous_dlopenflags = None if (sys.platform.startswith("linux")) : previous_dlopenflags = sys.getdlopenflags() sys.setdlopenflags(0x100|0x2) try: mod = __import__(name) except ImportError as e: if (optional): return None
def exercise(): from mmtbx.command_line import cif_as_mtz from iotbx import file_reader cif_in = """\ data_r2etdsf # _cell.entry_id 2etd _cell.length_a 80.4540 _cell.length_b 85.2590 _cell.length_c 53.3970 _cell.angle_alpha 90.0000 _cell.angle_beta 90.0000 _cell.angle_gamma 90.0000 # _diffrn.id 1 _diffrn.crystal_id 1 _diffrn.ambient_temp ? _diffrn.crystal_treatment ? # loop_ _diffrn_radiation_wavelength.id _diffrn_radiation_wavelength.wavelength 1 1.0163 2 0.9797 3 0.9796 #_entry.id 2etd # _exptl_crystal.id 1 # _reflns_scale.group_code 1 # _symmetry.entry_id 2etd _symmetry.space_group_name_H-M 'C 2 2 2' # # loop_ _refln.crystal_id _refln.wavelength_id _refln.scale_group_code _refln.index_h _refln.index_k _refln.index_l _refln.status _refln.F_meas_au _refln.F_meas_sigma_au _refln.F_calc _refln.phase_calc 1 1 1 0 2 8 o 149.0 3.1 558.1 0.0 1 1 1 0 2 9 o 217.9 4.3 119.3 180.0 1 1 1 0 2 10 o 33.3 5.3 340.6 180.0 1 1 1 0 2 11 o 389.0 10.0 692.8 0.0 1 1 1 0 2 12 o 143.5 3.7 54.5 180.0 1 1 1 0 2 13 f 801.4 36.0 1185.6 0.0 1 1 1 0 2 14 o 351.5 11.8 403.6 360.0 1 1 1 0 2 15 o 439.4 17.2 644.6 180.0 1 1 1 0 2 16 o 39.5 6.8 161.6 180.0 #END data_r2etdAsf # _cell.entry_id 2etd _cell.length_a 80.454 _cell.length_b 85.259 _cell.length_c 53.397 _cell.angle_alpha 90.000 _cell.angle_beta 90.000 _cell.angle_gamma 90.000 # _diffrn.id 1 _diffrn.crystal_id 1 _diffrn.ambient_temp ? _diffrn.crystal_treatment ? _diffrn.details ; unmerged original index intensities of data set used for refinement and phasing. ; # _entry.id 2etd # _exptl_crystal.id 1 # _reflns_scale.group_code 1 # # loop_ _refln.crystal_id _refln.wavelength_id _refln.scale_group_code _refln.index_h _refln.index_k _refln.index_l _refln.status _refln.intensity_meas _refln.intensity_sigma 1 1 1 0 2 -8 o 1544.6 130.4 1 1 1 0 2 9 o 3450.1 264.9 1 1 1 0 -2 9 o 3243.5 268.5 1 1 1 0 -2 -9 o 3475.4 265.0 1 1 1 0 2 -9 o 3420.1 260.3 1 1 1 0 2 10 o 58.8 31.8 1 1 1 0 -2 -10 o 131.7 50.7 1 1 1 0 2 -10 o 97.9 48.7 1 1 1 0 -2 11 o 9808.3 953.5 1 1 1 0 -2 -11 o 11486.1 970.3 1 1 1 0 2 -11 o 11278.1 967.8 1 1 1 0 2 12 o 1368.9 150.7 1 1 1 0 -2 12 o 1620.9 148.7 1 1 1 0 -2 -12 o 1293.5 147.6 1 1 1 0 2 -12 o 1619.5 155.7 1 1 1 0 -2 13 o 47438.3 4104.1 1 1 1 0 2 14 o 8577.5 1188.0 1 1 1 0 -2 14 o 7996.7 1179.9 1 1 1 0 -2 -14 o 9333.7 1178.1 1 1 1 0 2 -14 o 9642.3 1197.6 1 1 1 0 2 15 o 13577.7 1852.4 1 1 1 0 -2 15 o 14100.9 1852.8 1 1 1 0 -2 -15 o 14184.2 1871.6 1 1 1 0 2 16 o 135.6 76.3 1 1 1 0 -2 16 o 117.0 60.4 #END data_r2etdBsf # _cell.entry_id 2etd _cell.length_a 80.419 _cell.length_b 85.227 _cell.length_c 53.389 _cell.angle_alpha 90.000 _cell.angle_beta 90.000 _cell.angle_gamma 90.000 # _diffrn.id 1 _diffrn.crystal_id 1 _diffrn.ambient_temp ? _diffrn.crystal_treatment ? _diffrn.details ' unmerged original index intensities of data set used for phasing.' # _entry.id 2etd # _exptl_crystal.id 1 # _reflns_scale.group_code 1 # loop_ _refln.crystal_id _refln.wavelength_id _refln.scale_group_code _refln.index_h _refln.index_k _refln.index_l _refln.status _refln.intensity_meas _refln.intensity_sigma 1 2 1 0 2 8 o 1095.4 110.5 1 2 1 0 -2 8 o 1105.1 117.9 1 2 1 0 -2 -8 o 1055.4 108.6 1 2 1 0 2 -8 o 1105.1 118.0 1 2 1 0 2 9 o 3795.1 366.5 1 2 1 0 -2 9 o 3925.3 371.7 1 2 1 0 -2 -9 o 3726.2 366.9 1 2 1 0 2 -9 o 3899.3 371.8 1 2 1 0 2 10 o 27.0 40.7 1 2 1 0 -2 10 o 80.7 34.8 1 2 1 0 -2 -10 o 18.3 40.3 1 2 1 0 2 -10 o 16.5 45.4 1 2 1 0 2 11 o 10830.4 977.6 1 2 1 0 -2 11 o 9647.1 972.1 1 2 1 0 -2 -11 o 10854.6 983.4 1 2 1 0 2 -11 o 10369.4 976.0 1 2 1 0 2 12 o 1143.7 131.7 1 2 1 0 -2 12 o 1378.0 140.1 1 2 1 0 -2 -12 o 1090.9 134.2 1 2 1 0 2 -12 o 1290.6 135.6 1 2 1 0 -2 13 o 44492.5 4083.0 1 2 1 0 2 -13 o 44931.1 4102.4 1 2 1 0 2 14 o 6646.0 952.7 1 2 1 0 -2 14 o 6516.6 949.1 1 2 1 0 -2 -14 o 7589.5 952.0 1 2 1 0 2 -14 o 7559.7 954.2 1 2 1 0 2 16 o 256.6 68.6 1 2 1 0 -2 16 o 137.1 67.8 1 2 1 0 -2 -16 o 307.6 69.9 1 2 1 0 2 -16 o 265.4 71.5 """ open("r2etd-sf.cif", "w").write(cif_in) args = ["phenix.cif_as_mtz", "r2etd-sf.cif", "--merge"] rc = easy_run.fully_buffered(" ".join(args)).raise_if_errors().return_code assert (rc == 0) hkl_in = file_reader.any_file("r2etd-sf.mtz") l_w = [] for array in hkl_in.file_server.miller_arrays: l_w.append( (array.info().label_string(), "%.4f" % array.info().wavelength)) assert (sorted(l_w) == sorted([('FOBS,SIGFOBS', '1.0163'), ('FC,PHIFC', '1.0163'), ('R-free-flags', '1.0163'), ('I(+),SIGI(+),I(-),SIGI(-)', '1.0163'), ('I2(+),SIGI2(+),I2(-),SIGI2(-)', '0.9797') ]))
def run(self, idx, img): if os.path.isfile(self.term_file): raise IOTATermination('IOTA_TRACKER: Termination signal received!') else: # First, parse filepath to create Mosflm template directory = os.path.dirname(img) filepath = os.path.basename(img).split('.') fname = filepath[0] extension = filepath[1] if '_' in fname: suffix = fname.split('_')[-1] elif '-' in fname: suffix = fname.split('-')[-1] elif '.' in fname: suffix = fname.split('.')[-1] img_number = int(''.join(n if n.isdigit() else '' for n in suffix)) prefix = fname.replace(suffix, '') n_suffix = ''.join("#" if c.isdigit() else c for c in suffix) template = '{}{}.{}'.format(prefix, n_suffix, extension) # Create autoindex.com w/ Mosflm script # Write to temporary file and change permissions to run autoindex = [ '#! /bin/tcsh -fe', 'ipmosflm << eof-ipmosflm'.format(fname), 'NEWMATRIX {0}.mat'.format(fname), 'DIRECTORY {}'.format(directory), 'TEMPLATE {}'.format(template), 'AUTOINDEX DPS THRESH 0.1 IMAGE {} PHI 0 0.01'.format( img_number), 'GO', 'eof-ipmosflm' ] autoindex_string = '\n'.join(autoindex) autoindex_filename = 'autoindex_{}.com'.format(idx) with open(autoindex_filename, 'w') as af: af.write(autoindex_string) os.chmod(autoindex_filename, 0755) # Run Mosflm autoindexing command = './{}'.format(autoindex_filename) out = easy_run.fully_buffered(command, join_stdout_stderr=True) # Scrub text output final_spots = [ l for l in out.stdout_lines if 'spots written for image' in l ] final_cell_line = [ l for l in out.stdout_lines if 'Final cell' in l ] final_sg_line = [l for l in out.stdout_lines if 'space group' in l] if final_spots != []: spots = final_spots[0].rsplit()[0] else: spots = 0 if final_cell_line != []: cell = final_cell_line[0].replace( 'Final cell (after refinement) is', '').rsplit() else: cell = None if final_sg_line != []: sg = final_sg_line[0].rsplit()[6] else: sg = None # Temp file cleanup try: os.remove('{}.mat'.format(fname)) except Exception: pass try: os.remove('{}.spt'.format(prefix[:-1])) except Exception: pass try: os.remove('SUMMARY') except Exception: pass try: os.remove(autoindex_filename) except Exception: pass return [idx, spots, img, sg, cell]
def test_sacla_h5(dials_data, run_in_tmpdir, use_mpi, in_memory=False): # Only allow MPI tests if we've got MPI capabilities if use_mpi: pytest.importorskip("mpi4py") # Check the data files for this test exist sacla_path = dials_data("image_examples") image_path = os.path.join(sacla_path, "SACLA-MPCCD-run266702-0-subset.h5") assert os.path.isfile(image_path) geometry_path = os.path.join( sacla_path, "SACLA-MPCCD-run266702-0-subset-refined_experiments_level1.json") assert os.path.isfile(geometry_path) # Write the .phil configuration to a file with open("process_sacla.phil", "w") as f: f.write(sacla_phil % geometry_path) # Call dials.stills_process if use_mpi: command = [ "mpirun", "-n", "4", "dials.stills_process", "mp.method=mpi mp.composite_stride=4 output.logging_dir=.", ] else: command = ["dials.stills_process"] command += [image_path, "process_sacla.phil"] result = easy_run.fully_buffered(command).raise_if_errors() result.show_stdout() def test_refl_table(result_filename, ranges): table = flex.reflection_table.from_file(result_filename) for expt_id, n_refls in enumerate(ranges): subset = table.select(table["id"] == expt_id) assert len(subset) in n_refls, (result_filename, expt_id, len(table)) assert "id" in table assert set(table["id"]) == {0, 1, 2, 3} # large ranges to handle platform-specific differences test_refl_table( "idx-0000_integrated.refl", [ list(range(140, 160)), list(range(575, 600)), list(range(420, 445)), list(range(485, 510)), ], ) test_refl_table( "idx-0000_coset6.refl", [ list(range(145, 160)), list(range(545, 570)), list(range(430, 455)), list(range(490, 515)), ], )
def run_ksdssp_direct(pdb_str): exe_path = get_ksdssp_exe_path() ksdssp_out = easy_run.fully_buffered(command=exe_path, stdin_lines=pdb_str) return ( ksdssp_out.stdout_lines, ksdssp_out.stderr_lines )
def build_run(setup_cmd, ld_preload_flag, n_scatt, n_refl, build_cmd, check_max_a_b): if (op.isfile("a.out")): os.remove("a.out") assert not op.isfile("a.out") print(build_cmd) buffers = easy_run.fully_buffered(command=build_cmd) msg = buffers.format_errors_if_any() if (msg is not None): if (0): print(build_cmd) print() print(msg) print() STOP() return None assert op.isfile("a.out") run_cmd = setup_cmd if (ld_preload_flag): run_cmd += 'env LD_PRELOAD='\ '"/net/marbles/raid1/rwgk/dist/opt_resources/linux64/libimf.so:"'\ '"/net/marbles/raid1/rwgk/dist/opt_resources/linux64/libirc.so" ' utimes = [] run_cmd += '/usr/bin/time -p ./a.out' def run_once(): buffers = easy_run.fully_buffered(command=run_cmd) if (len(buffers.stderr_lines) != 3): print("v" * 79) print("\n".join(buffers.stderr_lines)) print("^" * 79) raise RuntimeError("Unexpected number of output lines" " (3 expected; acutal output see above).") if (n_scatt == 0): pass elif (n_scatt <= 10 and n_refl <= 100): assert len(buffers.stdout_lines) == n_scatt + n_refl else: assert len(buffers.stdout_lines) == 1 max_a, max_b = [float(s) for s in buffers.stdout_lines[0].split()] if (check_max_a_b): if (n_scatt == 2000 and n_refl == 20000): assert approx_equal(max_a, 35.047157, eps=1e-4) assert approx_equal(max_b, 25.212738, eps=1e-4) elif (n_scatt == 100 and n_refl == 1000): assert approx_equal(max_a, 4.493645, eps=1e-4) assert approx_equal(max_b, 10.515532, eps=1e-4) elif (n_scatt <= 10 and n_refl <= 100): if (libtbx.env.has_module(name="cctbx")): compare_with_cctbx_structure_factors( n_scatt=n_scatt, n_refl=n_refl, output_lines=buffers.stdout_lines) else: raise RuntimeError(max_a, max_b) utime = float(buffers.stderr_lines[1].split()[1]) utimes.append(utime) print("sample utime: %.2f" % utime) sys.stdout.flush() for _ in range(8): run_once() return min(utimes)
def exercise_anomalous_isomorphous_difference_map(): pdb_1 = """\ CRYST1 32.247 37.398 33.613 90.00 90.00 90.00 P 1 HETATM 1983 PG AGS A 340 5.029 2.780 3.852 1.00 60.68 P HETATM 1984 S1G AGS A 340 3.762 2.077 2.900 1.00 64.20 S HETATM 1985 O2G AGS A 340 6.396 1.935 3.849 1.00 61.32 O HETATM 1986 O3G AGS A 340 4.528 2.885 5.348 1.00 57.78 O HETATM 1987 PB AGS A 340 4.068 5.371 3.258 1.00 53.67 P HETATM 1988 O1B AGS A 340 3.533 5.492 1.878 1.00 52.45 O HETATM 1989 O2B AGS A 340 2.939 5.014 4.243 1.00 53.83 O HETATM 1990 O3B AGS A 340 5.255 4.278 3.263 1.00 56.71 O HETATM 1991 PA AGS A 340 5.581 6.888 5.132 1.00 42.66 P HETATM 1992 O1A AGS A 340 4.625 6.671 6.259 1.00 43.69 O HETATM 1993 O2A AGS A 340 6.834 5.959 5.235 1.00 45.93 O HETATM 1994 O3A AGS A 340 4.800 6.740 3.720 1.00 48.66 O HETATM 1995 O5' AGS A 340 6.098 8.409 5.107 1.00 36.10 O HETATM 1996 C5' AGS A 340 7.059 8.753 4.138 1.00 31.02 C HETATM 1997 C4' AGS A 340 7.848 9.969 4.591 1.00 29.79 C HETATM 1998 O4' AGS A 340 6.947 11.004 4.917 1.00 30.10 O HETATM 1999 C3' AGS A 340 8.682 9.711 5.831 1.00 30.12 C HETATM 2000 O3' AGS A 340 9.998 9.322 5.503 1.00 29.10 O HETATM 2001 C2' AGS A 340 8.619 11.034 6.578 1.00 30.77 C HETATM 2002 O2' AGS A 340 9.677 11.886 6.190 1.00 29.37 O HETATM 2003 C1' AGS A 340 7.319 11.667 6.111 1.00 30.26 C HETATM 2004 N9 AGS A 340 6.261 11.498 7.129 1.00 27.43 N HETATM 2005 C8 AGS A 340 5.454 10.400 7.302 1.00 27.85 C HETATM 2006 N7 AGS A 340 4.609 10.631 8.330 1.00 29.03 N HETATM 2007 C5 AGS A 340 4.862 11.863 8.817 1.00 26.18 C HETATM 2008 C6 AGS A 340 4.295 12.588 9.859 1.00 26.06 C HETATM 2009 N6 AGS A 340 3.275 12.079 10.559 1.00 28.35 N HETATM 2010 N1 AGS A 340 4.767 13.859 10.138 1.00 29.55 N HETATM 2011 C2 AGS A 340 5.794 14.401 9.385 1.00 29.19 C HETATM 2012 N3 AGS A 340 6.356 13.672 8.351 1.00 28.57 N HETATM 2013 C4 AGS A 340 5.896 12.422 8.071 1.00 25.89 C HETATM 2014 MN MN A 341 2.683 3.335 5.745 1.00 47.43 MN """ pdb_2 = """\ CRYST1 32.247 37.398 33.613 90.00 90.00 90.00 P 1 HETATM 1983 PG AGS A 340 5.029 2.780 3.852 1.00 60.68 P HETATM 1984 S1G AGS A 340 3.762 2.077 2.900 1.00 64.20 S HETATM 1985 O2G AGS A 340 6.396 1.935 3.849 1.00 61.32 O HETATM 1986 O3G AGS A 340 4.528 2.885 5.348 1.00 57.78 O HETATM 1987 PB AGS A 340 4.068 5.371 3.258 1.00 53.67 P HETATM 1988 O1B AGS A 340 3.533 5.492 1.878 1.00 52.45 O HETATM 1989 O2B AGS A 340 2.939 5.014 4.243 1.00 53.83 O HETATM 1990 O3B AGS A 340 5.255 4.278 3.263 1.00 56.71 O HETATM 1991 PA AGS A 340 5.581 6.888 5.132 1.00 42.66 P HETATM 1992 O1A AGS A 340 4.625 6.671 6.259 1.00 43.69 O HETATM 1993 O2A AGS A 340 6.834 5.959 5.235 1.00 45.93 O HETATM 1994 O3A AGS A 340 4.800 6.740 3.720 1.00 48.66 O HETATM 1995 O5' AGS A 340 6.098 8.409 5.107 1.00 36.10 O HETATM 1996 C5' AGS A 340 7.059 8.753 4.138 1.00 31.02 C HETATM 1997 C4' AGS A 340 7.848 9.969 4.591 1.00 29.79 C HETATM 1998 O4' AGS A 340 6.947 11.004 4.917 1.00 30.10 O HETATM 1999 C3' AGS A 340 8.682 9.711 5.831 1.00 30.12 C HETATM 2000 O3' AGS A 340 9.998 9.322 5.503 1.00 29.10 O HETATM 2001 C2' AGS A 340 8.619 11.034 6.578 1.00 30.77 C HETATM 2002 O2' AGS A 340 9.677 11.886 6.190 1.00 29.37 O HETATM 2003 C1' AGS A 340 7.319 11.667 6.111 1.00 30.26 C HETATM 2004 N9 AGS A 340 6.261 11.498 7.129 1.00 27.43 N HETATM 2005 C8 AGS A 340 5.454 10.400 7.302 1.00 27.85 C HETATM 2006 N7 AGS A 340 4.609 10.631 8.330 1.00 29.03 N HETATM 2007 C5 AGS A 340 4.862 11.863 8.817 1.00 26.18 C HETATM 2008 C6 AGS A 340 4.295 12.588 9.859 1.00 26.06 C HETATM 2009 N6 AGS A 340 3.275 12.079 10.559 1.00 28.35 N HETATM 2010 N1 AGS A 340 4.767 13.859 10.138 1.00 29.55 N HETATM 2011 C2 AGS A 340 5.794 14.401 9.385 1.00 29.19 C HETATM 2012 N3 AGS A 340 6.356 13.672 8.351 1.00 28.57 N HETATM 2013 C4 AGS A 340 5.896 12.422 8.071 1.00 25.89 C HETATM 2014 MN MN A 341 1.319 3.281 7.076 0.50 47.43 MN HETATM 60 C ACT 1 2.759 8.395 7.862 1.00 18.56 C HETATM 61 O ACT 1 3.812 8.507 7.193 1.00 19.26 O HETATM 62 OXT ACT 1 2.218 9.388 8.383 1.00 18.36 O HETATM 63 CH3 ACT 1 2.127 7.078 8.030 1.00 18.08 C """ open("tst_anom_iso_diff_1.pdb", "w").write(pdb_1) open("tst_anom_iso_diff_2.pdb", "w").write(pdb_2) base_args = [ "phenix.fmodel", "high_resolution=1.5", "type=real", "label=F", "r_free_flags_fraction=0.1", "wavelength=1.77", "add_random_error_to_amplitudes_percent=5", ] args_1 = base_args + [ "tst_anom_iso_diff_1.pdb", "output.file_name=tst_anom_iso_diff_1.mtz" ] args_2 = base_args + [ "tst_anom_iso_diff_2.pdb", "output.file_name=tst_anom_iso_diff_2.mtz" ] print(" ".join(args_1)) rc = easy_run.fully_buffered( " ".join(args_1)).raise_if_errors().return_code assert (rc == 0) print(" ".join(args_2)) rc = easy_run.fully_buffered( " ".join(args_2)).raise_if_errors().return_code assert (rc == 0) base_args = [ "phenix.fobs_minus_fobs_map", "f_obs_1_file=tst_anom_iso_diff_2.mtz", "f_obs_2_file=tst_anom_iso_diff_1.mtz", "tst_anom_iso_diff_2.pdb", "omit_selection=\"element MN\"", ] args_1 = base_args + [ "output_file=tst_anom_iso_diff_map_coeffs.mtz", "anomalous=True", ] args_2 = base_args + [ "output_file=tst_anom_iso_diff_map_coeffs_control.mtz" ] print(" ".join(args_1)) rc = easy_run.fully_buffered( " ".join(args_1)).raise_if_errors().return_code assert (rc == 0) print(" ".join(args_2)) rc = easy_run.fully_buffered( " ".join(args_2)).raise_if_errors().return_code assert (rc == 0) # The input structures differ in the position of the MN ion and in the # presence of an acetate ion in the second model. I am calculating two # isomorphous difference maps: one using the anomalous differences of each # dataset, and a control using the merged amplitudes. The MN should be # prominent in both, and by far the strongest feature in the anomalous map, # but the ACT should only have density in the control map. from iotbx import file_reader import iotbx.pdb.hierarchy pdb_in = iotbx.pdb.hierarchy.input(pdb_string=pdb_2) xrs = pdb_in.input.xray_structure_simple() mtz_1 = file_reader.any_file("tst_anom_iso_diff_map_coeffs.mtz") map_1 = mtz_1.file_server.miller_arrays[0].fft_map( resolution_factor=0.25).apply_sigma_scaling().real_map_unpadded() mtz_2 = file_reader.any_file("tst_anom_iso_diff_map_coeffs_control.mtz") map_2 = mtz_2.file_server.miller_arrays[0].fft_map( resolution_factor=0.25).apply_sigma_scaling().real_map_unpadded() sites_frac = xrs.sites_frac() anom_max = mn_anom = 0 for i_seq, atom in enumerate(pdb_in.hierarchy.atoms()): site_frac = sites_frac[i_seq] anom_diff = map_1.eight_point_interpolation(site_frac) fobs_diff = map_2.eight_point_interpolation(site_frac) labels = atom.fetch_labels() if (labels.resname.strip() == "MN"): assert (anom_diff > 10) and (fobs_diff > 5) mn_anom = anom_diff elif (labels.resname.strip() == "ACT"): assert (anom_diff < 3) and (fobs_diff > 5) if (anom_diff > anom_max): anom_max = anom_diff assert (anom_max == mn_anom)
def plot_multirun_stats(runs, run_numbers, d_min, n_multiples=2, ratio_cutoff=1, n_strong_cutoff=40, i_sigi_cutoff=1, run_tags=[], run_statuses=[], minimalist=False, interactive=False, easy_run=False, compress_runs=True, xsize=30, ysize=10, high_vis=False, title=None): tset = flex.double() two_theta_low_set = flex.double() two_theta_high_set = flex.double() nset = flex.int() resolutions_set = flex.double() n_lattices = flex.int() boundaries = [] lengths = [] runs_with_data = [] offset = 0 for idx in xrange(len(runs)): r = runs[idx] if len(r[0]) > 0: if compress_runs: tslice = r[0] - r[0][0] + offset offset += (r[0][-1] - r[0][0] + 1 / 120.) else: tslice = r[0] last_end = r[0][-1] tset.extend(tslice) two_theta_low_set.extend(r[1]) two_theta_high_set.extend(r[2]) nset.extend(r[3]) resolutions_set.extend(r[4]) n_lattices.extend(r[5]) boundaries.append(tslice[0]) boundaries.append(tslice[-1]) lengths.append(len(tslice)) runs_with_data.append(run_numbers[idx]) else: boundaries.extend([None] * 2) stats_tuple = get_run_stats(tset, two_theta_low_set, two_theta_high_set, nset, resolutions_set, n_lattices, tuple(boundaries), tuple(lengths), runs_with_data, n_multiples=n_multiples, ratio_cutoff=ratio_cutoff, n_strong_cutoff=n_strong_cutoff, i_sigi_cutoff=i_sigi_cutoff, d_min=d_min) if easy_run: from libtbx import easy_run, easy_pickle easy_pickle.dump( "plot_run_stats_tmp.pickle", (stats_tuple, d_min, n_multiples, run_tags, run_statuses, minimalist, interactive, xsize, ysize, high_vis, title)) result = easy_run.fully_buffered( command= "cctbx.xfel.plot_run_stats_from_stats_pickle plot_run_stats_tmp.pickle" ) try: png = result.stdout_lines[-1] if png == "None": return None except Exception: return None else: png = plot_run_stats(stats_tuple, d_min, n_multiples=n_multiples, run_tags=run_tags, run_statuses=run_statuses, minimalist=minimalist, interactive=interactive, xsize=xsize, ysize=ysize, high_vis=high_vis, title=title) return png
def test2(): """Run scan-varying refinement, comparing RMSD table with expected values. This test automates what was manually done periodically and recorded in dials_regression/refinement_test_data/centroid/README.txt""" dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use the i04_weak_data for this test data_dir = os.path.join(dials_regression, "refinement_test_data", "centroid") experiments_path = os.path.join(data_dir, "experiments_XPARM_REGULARIZED.json") pickle_path = os.path.join(data_dir, "spot_all_xds.pickle") for pth in (experiments_path, pickle_path): assert os.path.exists(pth) # scan-static refinement first to get refined_experiments.json as start point cmd1 = "dials.refine " + experiments_path + " " + pickle_path + \ " reflections_per_degree=50 " + \ " outlier.algorithm=null close_to_spindle_cutoff=0.05" cmd2 = "dials.refine refined_experiments.json " + pickle_path + \ " scan_varying=true output.history=history.pickle " + \ " reflections_per_degree=50 " + \ " outlier.algorithm=null close_to_spindle_cutoff=0.05" # work in a temporary directory cwd = os.path.abspath(os.curdir) tmp_dir = open_tmp_directory(suffix="test_dials_refine") os.chdir(tmp_dir) try: print cmd1 result1 = easy_run.fully_buffered(command=cmd1).raise_if_errors() print cmd2 result2 = easy_run.fully_buffered(command=cmd2).raise_if_errors() # load and check results import cPickle as pickle history = pickle.load(open("history.pickle", "r")) expected_rmsds = [(0.088488398, 0.114583571, 0.001460382), (0.080489334, 0.086406517, 0.001284069), (0.078835086, 0.086052630, 0.001195882), (0.077476911, 0.086194611, 0.001161143), (0.076755840, 0.086090630, 0.001157239), (0.076586376, 0.085939462, 0.001155641), (0.076603722, 0.085878953, 0.001155065), (0.076611382, 0.085862959, 0.001154863), (0.076608732, 0.085856935, 0.001154384), (0.076605731, 0.085852271, 0.001153858), (0.076604576, 0.085852318, 0.001153643), (0.076603981, 0.085854175, 0.001153594)] assert approx_equal(history['rmsd'], expected_rmsds) # check that the used_in_refinement flag got set correctly rt = flex.reflection_table.from_pickle('refined.pickle') uir = rt.get_flags(rt.flags.used_in_refinement) assert uir.count(True) == history['num_reflections'][-1] finally: os.chdir(cwd) print "OK" return
def run_angle(prefix): edits = """ refinement { geometry_restraints.edits { angle { action = %s atom_selection_1 = %s atom_selection_2 = %s atom_selection_3 = %s angle_ideal = 10 sigma = 1 } } } """ sel_old_1 = "chain A and resseq 7 and name N" sel_old_2 = "chain A and resseq 7 and name CA" sel_old_3 = "chain A and resseq 7 and name CB" sel_new_1 = "chain A and resseq 7 and name N" sel_new_2 = "chain A and resseq 7 and name CA" sel_new_3 = "chain A and resseq 7 and name CG2" cntr = 0 for action in ["add", "delete", "change"]: for kind in ["new", "old"]: prefix_ = "%s_angle_%s_%s" % (prefix, kind, action) with open("%s.eff" % prefix_, "w") as f: f.write(edits % (action, eval("sel_%s_1" % kind), eval("sel_%s_2" % kind), eval("sel_%s_3" % kind))) cmd = cmd_base % (prefix, prefix_) print(cmd) r = easy_run.fully_buffered(cmd) if action == "add": if kind == "new": cntr += 1 assert_lines_in_file(file_name="%s.pdb.geo" % prefix, lines=""" angle pdb=" N ILE A 7 " pdb=" CA ILE A 7 " pdb=" CG2 ILE A 7 " ideal model delta sigma weight residual 10.00 143.31 -133.31 1.00e+00 1.00e+00 1.78e+04""") elif kind == "old": cntr += 1 assert r.stderr_lines[0] == \ "Sorry: Some restraints were not added because they are already present." if action == "delete": cntr += 1 assert r.stderr_lines[0] == \ 'Sorry: geometry_restraints.edits.angle.action = delete not implemented.' if action == "change": if kind == "new": cntr += 1 assert r.stderr_lines[0] == \ 'Sorry: Angle below is not restrained, nothing to change.' elif kind == "old": cntr += 1 assert_lines_in_file(file_name="%s.pdb.geo" % prefix, lines=""" angle pdb=" N ILE A 7 " pdb=" CA ILE A 7 " pdb=" CB ILE A 7 " ideal model delta sigma weight residual 10.00 109.54 -99.54 1.00e+00 1.00e+00 9.91e+03""") assert cntr == 6
def submit(self, previous_job=None): from xfel.command_line.striping import Script from xfel.command_line.cxi_mpi_submit import get_submission_id from libtbx import easy_run configs_dir = os.path.join(settings_dir, "cfgs") identifier_string = self.get_identifier_string() target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil") with open(target_phil_path, 'w') as f: if self.task.parameters: f.write(self.task.parameters) path = get_run_path(self.app.params.output_folder, self.trial, self.rungroup, self.run, self.task) os.mkdir(path) arguments = """ mp.queue={} mp.nproc={} mp.nproc_per_node={} mp.method={} {} mp.use_mpi=False striping.results_dir={} striping.trial={} striping.rungroup={} striping.run={} {} striping.chunk_size=3000 striping.stripe=False striping.dry_run=True striping.output_folder={} reintegration.integration.lookup.mask={} mp.local.include_mp_in_command=False """.format( self.app.params.mp.queue if len(self.app.params.mp.queue) > 0 else None, self.app.params.mp.nproc, self.app.params.mp.nproc_per_node, self.app.params.mp.method, '\n'.join([ 'mp.env_script={}'.format(p) for p in self.app.params.mp.env_script if p ]), self.app.params.output_folder, self.trial.trial, self.rungroup.id, self.run.run, target_phil_path, path, self.rungroup.untrusted_pixel_mask_path, ).split() commands = Script(arguments).run() submission_ids = [] if self.app.params.mp.method == 'local': self.status = "RUNNING" for command in commands: try: result = easy_run.fully_buffered(command=command) result.raise_if_errors() except Exception as e: if not "Warning: job being submitted without an AFS token." in str( e): raise e submission_ids.append( get_submission_id(result, self.app.params.mp.method)) if self.app.params.mp.method == 'local': self.status = "DONE" else: return ",".join(submission_ids)
def exercise_misc(): utils.host_and_user().show(prefix="### ") time_in_seconds = 1.1 for i_trial in range(55): time_in_seconds = time_in_seconds**1.1 time_units, time_unit = utils.human_readable_time( time_in_seconds=time_in_seconds) assert approx_equal( utils.human_readable_time_as_seconds(time_units, time_unit), time_in_seconds) # fts = utils.format_timestamp f12 = utils.format_timestamp_12_hour f24 = utils.format_timestamp_24_hour def check(string, expected): assert len(string) == len(expected) check(f12(1280007000), 'Jul 24 2010 02:30 PM') check(f24(1280007000), 'Jul 24 2010 14:30') check(f12(1280007000, True), '24-07-10 02:30 PM') check(f24(1280007000, True), '24-07-10 14:30') check(fts(1280007000), 'Jul 24 2010 02:30 PM') # nfs = utils.number_from_string for string in ["True", "False"]: try: nfs(string=string) except ValueError as e: assert str(e) == 'Error interpreting "%s" as a numeric expression.' % ( string) else: raise Exception_expected assert nfs(string="-42") == -42 assert approx_equal(nfs(string="3.14"), 3.14) assert approx_equal(nfs(string="cos(0)"), 1) try: nfs(string="xxx(0)") except ValueError as e: assert str(e).startswith( 'Error interpreting "xxx(0)" as a numeric expression: ') else: raise Exception_expected # s = "[0.143139, -0.125121, None, -0.308607]" assert numstr(values=eval(s)) == s # for s,i in {"2000000" : 2000000, "2k" : 2048, "2Kb" : 2048, "2 Kb" : 2048, "5Mb" : 5*1024*1024, "2.5Gb" : 2.5*1024*1024*1024, "1T": 1024*1024*1024*1024, 10000 : 10000, 5.5 : 5.5, }.items(): assert utils.get_memory_from_string(s) == i # assert utils.tupleize(1) == (1,) assert utils.tupleize("abcde") == ('a', 'b', 'c', 'd', 'e') assert utils.tupleize([1,2,3]) == (1,2,3) # sf = utils.search_for assert sf(pattern="fox", mode="==", lines=["fox", "foxes"]) \ == ["fox"] assert sf(pattern="o", mode="find", lines=["fox", "bird", "mouse"]) \ == ["fox", "mouse"] assert sf(pattern="fox", mode="startswith", lines=["fox", "foxes"]) \ == ["fox", "foxes"] assert sf(pattern="xes", mode="endswith", lines=["fox", "foxes"]) \ == ["foxes"] assert sf(pattern="es$", mode="re.search", lines=["geese", "foxes"]) \ == ["foxes"] assert sf(pattern="ge", mode="re.match", lines=["geese", "angel"]) \ == ["geese"] # nd1d = utils.n_dim_index_from_one_dim for size in range(1,5): for i1d in range(size): assert nd1d(i1d=i1d, sizes=(size,)) == [i1d] for sizes in [(1,1), (1,3), (3,1), (2,3)]: ni, nj = sizes for i in range(ni): for j in range(nj): i1d = i*nj+j assert nd1d(i1d=i1d, sizes=sizes) == [i,j] for sizes in [(1,1,1), (1,3,1), (3,2,1), (4,3,2)]: ni, nj, nk = sizes for i in range(ni): for j in range(nj): for k in range(nk): i1d = (i*nj+j)*nk+k assert nd1d(i1d=i1d, sizes=sizes) == [i,j,k] # from libtbx import easy_run b = easy_run.fully_buffered( command="libtbx.raise_exception_for_testing") for lines in [b.stdout_lines, b.stderr_lines]: assert lines[0].startswith("EXCEPTION_INFO: show_stack(0): ") assert lines[-1] == "EXCEPTION_INFO: RuntimeError: Just for testing." b = easy_run.fully_buffered( command="libtbx.raise_exception_for_testing silent") b.raise_if_errors_or_output() # frange = utils.frange samples = utils.samples assert approx_equal([i/10. for i in range(-2,2)], frange(-0.2,0.2,0.1)) assert approx_equal([i/10. for i in range(-2,2+1)], samples(-0.2,0.2,0.1)) assert approx_equal([i/10. for i in range(2,-2,-1)], frange(0.2,-0.2,-0.1)) assert approx_equal([i/10. for i in range(2,-2-1,-1)], samples(0.2,-0.2,-0.1)) assert approx_equal([i/4. for i in range(4,8)], frange(1, 2, 0.25)) assert approx_equal([i/4. for i in range(4,8+1)], samples(1, 2, 0.25)) assert approx_equal([0.2+i/3. for i in range(4)], frange(0.2, 1.3, 1./3)) assert approx_equal([0.2+i/3. for i in range(4)], samples(0.2, 1.3, 1./3)) assert approx_equal(list(range(5)) , frange(5)) assert approx_equal(list(range(5+1)) , samples(5)) assert approx_equal(list(range(-5)), frange(-5)) assert approx_equal(list(range(-5-1)), samples(-5)) assert approx_equal(list(range(1,3)), frange(1, 3)) assert approx_equal(list(range(1,3+1)), samples(1, 3)) assert approx_equal([i/10. for i in range(20,9,-2)], frange(2.0,0.9,-0.2)) assert approx_equal([i/10. for i in range(20,9,-2)], samples(2.0,0.9,-0.2)) # ff = utils.format_float_with_standard_uncertainty assert ff(21.234567, 0.0013) == "21.2346(13)" assert ff(21.234567, 0.0023) == "21.235(2)" assert ff(12345, 45) == "12350(50)" assert ff(12.3,1.2) == "12.3(12)" assert ff(-0.2451, 0.8135) == "-0.2(8)" assert ff(1.234, 0.196) == "1.2(2)" assert ff(1.234, 0.193) == "1.23(19)" # for n in range(4): assert len(utils.random_hex_code(number_of_digits=n)) == n # print("multiprocessing problem:", utils.detect_multiprocessing_problem()) # print("base36_timestamp():", utils.base36_timestamp(), "now") print("base36_timestamp():", utils.base36_timestamp( seconds_since_epoch=115855*365.2425*24*60*60), "year 115855 CE") # print("get_svn_revision():", utils.get_svn_revision()) print("get_build_tag():", utils.get_build_tag()) # concatenate_python_script # XXX the string concatenation here is required to trick libtbx.find_clutter, # which will warn about repetition of the future division import. script = """ from __future__ """ + """import division import os.path def foo(): print "bar" """ d = tempfile.mkdtemp() name = os.path.join(d, "tst_libtbx_utils_python_script.py") name2 = os.path.join(d, "tst_libtbx_utils_python_script2.py") open(name, "w").write(script) f = open(name2, "w") utils.concatenate_python_script(out=f, file_name=name) f.close() lines = open(name2).readlines() have_def = False for line in lines : assert (not "__future__" in line) if line.startswith("def foo"): have_def = True assert have_def
def run_combinations(compiler_versions, all_utimes, n_scatt, n_refl, compiler_build_opts_list, real_list): for lang, setup_sh_list, compiler, build_opts in compiler_build_opts_list: for setup_sh in setup_sh_list: if (setup_sh is None): setup_cmd = "" else: setup_cmd = ". %s/%s; " % (setup_dir, setup_sh) compiler_version = easy_run.fully_buffered( command=setup_cmd + compiler + " --version", join_stdout_stderr=True).stdout_lines[0] if (lang in ["f", "c"]): ld_preload_flags = [False, True] else: ld_preload_flags = [False] for ld_preload_flag in ld_preload_flags: iml = ["", " Intel Math Lib"][int(ld_preload_flag)] compiler_versions.append(compiler_version + iml) build_cmd = " ".join([setup_cmd + compiler, build_opts]) print(build_cmd) utimes = [] if (n_scatt != 0): for real in real_list: print(" %s" % real) for replace_cos in [False, True]: print(" replace_cos", replace_cos) for replace_exp in [False, True]: print(" replace_exp", replace_exp) sys.stdout.flush() if (compiler_version != "n/a"): utime = write_build_run( setup_cmd=setup_cmd, ld_preload_flag=ld_preload_flag, n_scatt=n_scatt, n_refl=n_refl, real=real, lang=lang, build_cmd=build_cmd, replace_cos=replace_cos, replace_exp=replace_exp) if (utime is not None): print(" %4.2f" % utime) else: utime = -1.0 print(" err") else: utime = -1.0 print(" n/a") utimes.append(utime) sys.stdout.flush() else: if (lang.lower() == "f"): f_source = libtbx.env.find_in_repositories( relative_path="lapack_fem/dsyev_test.f", test=op.isfile, optional=False) build_cmd_compl = build_cmd + " " + f_source else: cpp_source = libtbx.env.find_in_repositories( relative_path="lapack_fem/dsyev_test.cpp", test=op.isfile, optional=False) build_cmd_compl = build_cmd + finalize_cpp_build_cmd( cpp_source) utime = build_run(setup_cmd=setup_cmd, ld_preload_flag=ld_preload_flag, n_scatt=n_scatt, n_refl=n_refl, build_cmd=build_cmd_compl, check_max_a_b=False) if (utime is None): print("err") utime = -1.0 else: print("min utime: %.2f" % utime) sys.stdout.flush() utimes.append(utime) all_utimes.append((utimes, build_cmd + iml))
def number_of_processors(return_value_if_unknown=None): global _number_of_processors if (_number_of_processors is Auto): _number_of_processors = None try: import multiprocessing except ImportError: pass else: try: n = multiprocessing.cpu_count() except NotImplementedError: pass else: _number_of_processors = n if (_number_of_processors is None): try: import boost_adaptbx.boost.python as bp except ImportError: pass else: n = bp.ext.number_of_processors() if (n != 0): _number_of_processors = n if (_number_of_processors is None): cpuinfo = "/proc/cpuinfo" # Linux if os.path.isfile(cpuinfo): n = 0 for line in open(cpuinfo).read().splitlines(): if (not line.startswith("processor")): continue line = line[9:].replace(" ", "").replace("\t", "") if (not line.startswith(":")): continue n += 1 if (n != 0): _number_of_processors = n if (_number_of_processors is None): cmd = "/usr/sbin/system_profiler" # Mac OS X if os.path.isfile(cmd): keys = [ "Total Number Of Cores: ", "Number Of CPUs: ", "Number Of Processors: " ] ns = [None] * len(keys) from libtbx import easy_run for line in easy_run.fully_buffered( command=cmd + " SPHardwareDataType").stdout_lines: line = line.strip() for i, key in enumerate(keys): if (line.startswith(key)): try: n = int(line[len(key):]) except ValueError: continue if (n > 0 and ns[i] is None): ns[i] = n for n in ns: if (n is not None): _number_of_processors = n break if (_number_of_processors is None): n = os.environ.get("NUMBER_OF_PROCESSORS") # Windows if (n is not None): try: n = int(n) except ValueError: pass else: _number_of_processors = n if (_number_of_processors is None): cmd = "/sbin/hinv" # IRIX if os.path.isfile(cmd): from libtbx import easy_run for line in easy_run.fully_buffered(command=cmd).stdout_lines: if (line.endswith(" Processors")): try: n = int(line.split(" ", 1)[0]) except ValueError: continue if (n > 0): _number_of_processors = n break if (_number_of_processors is not None): return _number_of_processors return return_value_if_unknown
for sizes in [(1, 1), (1, 3), (3, 1), (2, 3)]: ni, nj = sizes for i in xrange(ni): for j in xrange(nj): i1d = i * nj + j assert nd1d(i1d=i1d, sizes=sizes) == [i, j] for sizes in [(1, 1, 1), (1, 3, 1), (3, 2, 1), (4, 3, 2)]: ni, nj, nk = sizes for i in xrange(ni): for j in xrange(nj): for k in xrange(nk): i1d = (i * nj + j) * nk + k assert nd1d(i1d=i1d, sizes=sizes) == [i, j, k] # from libtbx import easy_run b = easy_run.fully_buffered(command="libtbx.raise_exception_for_testing") for lines in [b.stdout_lines, b.stderr_lines]: assert lines[0].startswith("EXCEPTION_INFO: show_stack(0): ") assert lines[-1] == "EXCEPTION_INFO: RuntimeError: Just for testing." b = easy_run.fully_buffered( command="libtbx.raise_exception_for_testing silent") b.raise_if_errors_or_output() # frange = utils.frange samples = utils.samples assert approx_equal([i / 10. for i in range(-2, 2)], frange(-0.2, 0.2, 0.1)) assert approx_equal([i / 10. for i in range(-2, 2 + 1)], samples(-0.2, 0.2, 0.1)) assert approx_equal([i / 10. for i in range(2, -2, -1)], frange(0.2, -0.2, -0.1))
def run_bond(prefix): edits = """ refinement { geometry_restraints.edits { bond { action = %s atom_selection_1 = %s atom_selection_2 = %s distance_ideal = 2.0 sigma = 1.0 } } } """ sel_old_1 = "chain A and resseq 7 and name C" sel_old_2 = "chain A and resseq 7 and name O" sel_new_1 = "chain A and resseq 7 and name CA" sel_new_2 = "chain A and resseq 7 and name O" cntr = 0 for action in ["add", "delete", "change"]: for kind in ["new", "old"]: prefix_ = "%s_bond_%s_%s" % (prefix, kind, action) with open("%s.eff" % prefix_, "w") as f: f.write( edits % (action, eval("sel_%s_1" % kind), eval("sel_%s_2" % kind))) cmd = cmd_base % (prefix, prefix_) print(cmd) r = easy_run.fully_buffered(cmd) if action == "delete": cntr += 1 assert r.stderr_lines[0] == \ 'Sorry: geometry_restraints.edits.bond.action = delete not implemented.' else: if kind == "new": if action == 'add': cntr += 1 assert_lines_in_file( file_name="%s.pdb.geo" % prefix, lines="""bond pdb=" CA ILE A 7 " pdb=" O ILE A 7 " ideal model delta sigma weight residual 2.000 2.394 -0.394 1.00e+00 1.00e+00 1.56e-01""") elif action == 'change': cntr += 1 assert r.stderr_lines[ 0] == "Sorry: Bond below does not exists, use action=add instead." elif (kind == "old"): if action == 'add': cntr += 1 assert r.stderr_lines[ 0] == "Sorry: Bond below exists, use action=change instead." elif action == 'change': cntr += 1 assert_lines_in_file( file_name="%s.pdb.geo" % prefix, lines="""bond pdb=" C ILE A 7 " pdb=" O ILE A 7 " ideal model delta sigma weight residual 2.000 1.229 0.771 1.00e+00 1.00e+00 5.94e-01""") assert cntr == 6
def test2(): """Test joint refinement where two detectors are constrained to enforce a differential distance (along the shared initial normal vector) of 1 mm. This test can be constructed on the fly from data already in dials_regression""" if not libtbx.env.has_module("dials_regression"): print "Skipping test2 in " + __file__ + " as dials_regression not present" return dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use the 'centroid' data for this test. The 'regularized' experiments are # useful because the detector has fast and slow exactly aligned with X, -Y # so the distance is exactly along the normal vector and can be altered # directly by changing the Z component of the orgin vector data_dir = os.path.join(dials_regression, "refinement_test_data", "centroid") experiments_path = os.path.join(data_dir, "experiments_XPARM_REGULARIZED.json") pickle_path = os.path.join(data_dir, "spot_1000_xds.pickle") # work in a temporary directory cwd = os.path.abspath(os.curdir) tmp_dir = open_tmp_directory(suffix="test_dials_constraints") os.chdir(tmp_dir) # load the experiments and spots el = ExperimentListFactory.from_json_file(experiments_path, check_format=False) rt = flex.reflection_table.from_pickle(pickle_path) # adjust the detector distance by -0.5 mm detector = el[0].detector panel = detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() origin = panel.get_origin() panel.set_frame(fast, slow, origin[0:2] + (origin[2] + 0.5,)) # duplicate the experiment and adjust distance by +1 mm e2 = deepcopy(el[0]) detector = e2.detector panel = detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() origin = panel.get_origin() panel.set_frame(fast, slow, origin[0:2] + (origin[2] - 1.0,)) # append to the experiment list and write out el.append(e2) dump = ExperimentListDumper(el) dump.as_json('foo_experiments.json') # duplicate the reflections and increment the experiment id rt2 = deepcopy(rt) rt2['id'] = rt2['id'] + 1 # concatenate reflections and write out rt.extend(rt2) rt.as_pickle('foo_reflections.pickle') # set up refinement, constraining the distance parameter cmd = ("dials.refine foo_experiments.json foo_reflections.pickle " "history=history.pickle refinement.parameterisation.detector." "constraints.parameter=Dist") try: result = easy_run.fully_buffered(command=cmd).raise_if_errors() # load refinement history import cPickle as pickle with open('history.pickle') as f: history = pickle.load(f) ref_exp = ExperimentListFactory.from_json_file('refined_experiments.json', check_format=False) finally: os.chdir(cwd) # we expect 8 steps of constrained refinement assert history.get_nrows() == 8 # get parameter vector from the final step pvec = history['parameter_vector'][-1] # the constrained parameters have indices 0 and 6 in this case. Check they # are still exactly 1 mm apart assert pvec[0] == pvec[6] - 1.0 # NB because the other detector parameters were not also constrained, the # refined lab frame distances may not in fact differ by 1 mm. The constraint # acts along the initial detector normal vector during composition of a new # detector position. After refinement of tilt/twist type rotations, # the final distances along the new normal vectors will change det1, det2 = ref_exp.detectors() p1 = det1[0] p2 = det2[0] assert approx_equal(p2.get_distance() - p1.get_distance(), 0.9987655) print "OK"
def run(): cmd = os.path.join(libtbx.env.under_build('bin'), 'libtbx.import_all_ext') easy_run.fully_buffered(cmd).raise_if_errors() print "OK"