def exercise_basic(verbose=False): symm = crystal.symmetry(space_group_info=sgtbx.space_group_info("P212121"), unit_cell=uctbx.unit_cell((6, 7, 8, 90, 90, 90))) set1 = miller.build_set(crystal_symmetry=symm, anomalous_flag=True, d_min=1.0) assert (set1.indices().size() == 341) data0 = flex.double(set1.indices().size(), 100.) sigmas1 = flex.double(set1.indices().size(), 4.) for i in range(10): data0[2 + i * 30] = -1 for i in range(10): data0[5 + i * 30] = 7.5 array0 = set1.array(data=data0, sigmas=sigmas1) array0.set_observation_type_xray_intensity() flags = array0.generate_r_free_flags( use_lattice_symmetry=True).average_bijvoet_mates() mtz0 = array0.as_mtz_dataset(column_root_label="I-obs") mtz0.add_miller_array(flags, column_root_label="R-free-flags") mtz0.mtz_object().write("tst_data.mtz") # convert intensities to amplitudes new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst1.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data.mtz labels = I-obs(+),SIGI-obs(+),I-obs(-),SIGI-obs(-) output_labels = I-obs(+) SIGI-obs(+) I-obs(-) SIGI-obs(-) } miller_array { file_name = tst_data.mtz labels = R-free-flags output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() log = sys.stdout if (not verbose): log = null_out() def run_and_reload(params, file_name): p = reflection_file_editor.process_arrays(params, log=log) p.finish() mtz_in = file_reader.any_file(file_name) miller_arrays = mtz_in.file_object.as_miller_arrays() return miller_arrays params.mtz_file.miller_array[0].output_as = "amplitudes" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry, e: assert ("inconsistent with" in str(e)), str(e)
def exercise_basic (verbose=False) : symm = crystal.symmetry( space_group_info=sgtbx.space_group_info("P212121"), unit_cell=uctbx.unit_cell((6,7,8,90,90,90))) set1 = miller.build_set( crystal_symmetry=symm, anomalous_flag=True, d_min=1.0) assert (set1.indices().size() == 341) data0 = flex.double(set1.indices().size(), 100.) sigmas1 = flex.double(set1.indices().size(), 4.) for i in range(10) : data0[2+i*30] = -1 for i in range(10) : data0[5+i*30] = 7.5 array0 = set1.array(data=data0, sigmas=sigmas1) array0.set_observation_type_xray_intensity() flags = array0.generate_r_free_flags( use_lattice_symmetry=True).average_bijvoet_mates() mtz0 = array0.as_mtz_dataset(column_root_label="I-obs") mtz0.add_miller_array(flags, column_root_label="R-free-flags") mtz0.mtz_object().write("tst_data.mtz") # convert intensities to amplitudes new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst1.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data.mtz labels = I-obs(+),SIGI-obs(+),I-obs(-),SIGI-obs(-) output_labels = I-obs(+) SIGI-obs(+) I-obs(-) SIGI-obs(-) } miller_array { file_name = tst_data.mtz labels = R-free-flags output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() log = sys.stdout if (not verbose) : log = null_out() def run_and_reload (params, file_name) : p = reflection_file_editor.process_arrays(params, log=log) p.finish() mtz_in = file_reader.any_file(file_name) miller_arrays = mtz_in.file_object.as_miller_arrays() return miller_arrays params.mtz_file.miller_array[0].output_as = "amplitudes" try : miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry, e : assert ("inconsistent with" in str(e)), str(e)
miller_arrays = run_and_reload(params, "tst1.mtz") assert (not miller_arrays[0].anomalous_flag()) params.mtz_file.miller_array[1].anomalous_data = "anomalous" try: run_and_reload(params, "tst1.mtz") except Sorry, e: assert ("not specified enough" in str(e)), str(e) else: raise Exception_expected params.mtz_file.miller_array[1].output_labels = [ 'R-free-flags(+)', 'R-free-flags(-)' ] miller_arrays = run_and_reload(params, "tst1.mtz") assert miller_arrays[1].anomalous_flag() # filter by signal-to-noise ratio params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].filter_by_signal_to_noise = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") data = miller_arrays[0].data() sigmas = miller_arrays[0].sigmas() assert (data.size() == 321) and ((data / sigmas).all_ge(2.0)) # data scaling params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].scale_factor = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") data2 = miller_arrays[0].data() assert approx_equal(flex.max(data2), 2. * flex.max(data)) # remove negatives params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].remove_negatives = True miller_arrays = run_and_reload(params, "tst1.mtz")
def exercise_basic(verbose=False): symm = crystal.symmetry(space_group_info=sgtbx.space_group_info("P212121"), unit_cell=uctbx.unit_cell((6, 7, 8, 90, 90, 90))) set1 = miller.build_set(crystal_symmetry=symm, anomalous_flag=True, d_min=1.0) assert (set1.indices().size() == 341) data0 = flex.double(set1.indices().size(), 100.) sigmas1 = flex.double(set1.indices().size(), 4.) for i in range(10): data0[2 + i * 30] = -1 for i in range(10): data0[5 + i * 30] = 7.5 array0 = set1.array(data=data0, sigmas=sigmas1) array0.set_observation_type_xray_intensity() flags = array0.generate_r_free_flags( use_lattice_symmetry=True).average_bijvoet_mates() mtz0 = array0.as_mtz_dataset(column_root_label="I-obs") mtz0.add_miller_array(flags, column_root_label="R-free-flags") mtz0.mtz_object().write("tst_data.mtz") # convert intensities to amplitudes new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst1.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data.mtz labels = I-obs(+),SIGI-obs(+),I-obs(-),SIGI-obs(-) output_labels = I-obs(+) SIGI-obs(+) I-obs(-) SIGI-obs(-) } miller_array { file_name = tst_data.mtz labels = R-free-flags output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() log = sys.stdout if (not verbose): log = null_out() def run_and_reload(params, file_name): p = reflection_file_editor.process_arrays(params, log=log) p.finish() mtz_in = file_reader.any_file(file_name) miller_arrays = mtz_in.file_object.as_miller_arrays() return miller_arrays params.mtz_file.miller_array[0].output_as = "amplitudes" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("inconsistent with" in str(e)), str(e) else: raise Exception_expected params.mtz_file.miller_array[0].output_labels = [ 'F-obs(+)', 'SIGF-obs(+)', 'F-obs(-)', 'SIGF-obs(-)' ] miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].info().labels == [ 'F-obs(+)', 'SIGF-obs(+)', 'F-obs(-)', 'SIGF-obs(-)' ]) assert miller_arrays[0].is_xray_amplitude_array() data1 = miller_arrays[0].data() assert (flex.min(data1) == 0) # now with French-Wilson treatment params.mtz_file.miller_array[0].output_as = "amplitudes_fw" miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].info().labels == [ 'F-obs(+)', 'SIGF-obs(+)', 'F-obs(-)', 'SIGF-obs(-)' ]) assert miller_arrays[0].is_xray_amplitude_array() assert (len(miller_arrays[0].data()) == 341) assert (flex.min(miller_arrays[0].data()) > 0) # force data type change params.mtz_file.miller_array[0].output_as = "auto" params.mtz_file.miller_array[0].force_type = "amplitudes" params.mtz_file.miller_array[0].output_labels = [ 'F-obs(+)', 'SIGF-obs(+)', 'F-obs(-)', 'SIGF-obs(-)' ] params.mtz_file.output_file = "tst2.mtz" miller_arrays = run_and_reload(params, "tst2.mtz") mtz_orig = file_reader.any_file("tst_data.mtz") orig_arrays = mtz_orig.file_server.miller_arrays data0 = orig_arrays[0].data() data2 = miller_arrays[0].data() assert miller_arrays[0].is_xray_amplitude_array() assert (data0.all_eq(data2)) and (data0.all_ne(data1)) # convert to and from anomalous data params.mtz_file.output_file = "tst1.mtz" params.mtz_file.miller_array[0].force_type = "auto" params.mtz_file.miller_array[0].anomalous_data = "merged" try: run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("too many column labels" in str(e)), str(e) else: raise Exception_expected params.mtz_file.miller_array[0].output_labels = ["I-obs", "SIGI-obs"] miller_arrays = run_and_reload(params, "tst1.mtz") assert (not miller_arrays[0].anomalous_flag()) params.mtz_file.miller_array[1].anomalous_data = "anomalous" try: run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("not specified enough" in str(e)), str(e) else: raise Exception_expected params.mtz_file.miller_array[1].output_labels = [ 'R-free-flags(+)', 'R-free-flags(-)' ] miller_arrays = run_and_reload(params, "tst1.mtz") assert miller_arrays[1].anomalous_flag() # filter by signal-to-noise ratio params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].filter_by_signal_to_noise = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") data = miller_arrays[0].data() sigmas = miller_arrays[0].sigmas() assert (data.size() == 321) and ((data / sigmas).all_ge(2.0)) # data scaling params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].scale_factor = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") data2 = miller_arrays[0].data() assert approx_equal(flex.max(data2), 2. * flex.max(data)) # remove negatives params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].remove_negatives = True miller_arrays = run_and_reload(params, "tst1.mtz") n_refl = miller_arrays[0].data().size() assert (n_refl == 331), n_refl # scale to maximum value params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].scale_max = 2000. miller_arrays = run_and_reload(params, "tst1.mtz") data3 = miller_arrays[0].data() assert (flex.max(data3) == 2000.), flex.max(data3) # apply isotropic B-factor params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].add_b_iso = 20.0 miller_arrays = run_and_reload(params, "tst1.mtz") data_b = miller_arrays[0].data() sigmas_b = miller_arrays[0].sigmas() assert approx_equal(data_b[0], 72.68358, eps=0.00001) # apply anisotropic B-factor params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].add_b_aniso = (20., 20., 20., 0., 0., 0.) miller_arrays = run_and_reload(params, "tst1.mtz") data_b = miller_arrays[0].data() sigmas_b = miller_arrays[0].sigmas() assert approx_equal(data_b[0], 72.68358, eps=0.00001) # shuffle data randomly params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].shuffle_values = True miller_arrays = run_and_reload(params, "tst1.mtz") data_shuffled = miller_arrays[0].data() assert (not data0.all_eq(data_shuffled)) params.mtz_file.miller_array[0].reset_values_to = 12345.6 miller_arrays = run_and_reload(params, "tst1.mtz") data_reset = miller_arrays[0].data() assert (data_reset.all_eq(data_reset[0])) # improper operations on R-free flags params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].scale_factor = None params.mtz_file.miller_array[1].output_as = "amplitudes" # this one won't actually crash (it will be ignored) miller_arrays = run_and_reload(params, "tst1.mtz") params.mtz_file.miller_array[1].output_as = "auto" params.mtz_file.miller_array[1].force_type = "amplitudes" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: pass else: raise Exception_expected params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[1].filter_by_signal_to_noise = 2.0 try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: pass else: raise Exception_expected # improper output labels params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[1].output_labels.append("NULL") try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("number of output labels" in str(e)) else: raise Exception_expected params.mtz_file.miller_array[1].output_labels.pop() params.mtz_file.miller_array[0].output_labels.pop() try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("number of output labels" in str(e)) else: raise Exception_expected # use column_root_label instead params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].column_root_label = "I_tst" params.mtz_file.miller_array[1].column_root_label = "FREE" miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].info().label_string() == "I_tst(+),SIGI_tst(+),I_tst(-),SIGI_tst(-)") assert (miller_arrays[1].info().label_string() == "FREE") # incorrect root label params.mtz_file.miller_array[0].column_root_label = "asdf" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as s: assert ("inconsistent" in str(s)) else: raise Exception_expected params.mtz_file.miller_array[0].column_root_label = "F_tst" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as s: assert ("inconsistent" in str(s)) else: raise Exception_expected # same root label, but now with compatible array type params.mtz_file.miller_array[0].output_as = "amplitudes" miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].info().label_string() == "F_tst(+),SIGF_tst(+),F_tst(-),SIGF_tst(-)") # R-free label conflicts params = master_phil.fetch(source=new_phil).extract() params.mtz_file.r_free_flags.force_generate = True params.mtz_file.r_free_flags.new_label = "R-free-flags" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry: pass else: raise Exception_expected params.mtz_file.resolve_label_conflicts = True miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[-1].info().label_string() == "R-free-flags_2") # resolution filter params = master_phil.fetch(source=new_phil).extract() params.mtz_file.d_min = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") assert approx_equal(miller_arrays[0].d_min(), 2.0, eps=0.01) assert approx_equal(miller_arrays[1].d_min(), 2.0, eps=0.01) # resolution filter for a specific array params.mtz_file.d_min = None params.mtz_file.miller_array[0].d_min = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") assert approx_equal(miller_arrays[0].d_min(), 2.0, eps=0.01) assert approx_equal(miller_arrays[1].d_min(), 1.0, eps=0.01) # filter by index params = master_phil.fetch(source=new_phil).extract() params.mtz_file.exclude_reflection.append((1, 1, 1)) params.mtz_file.exclude_reflection.append((1, 2, 3)) miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].indices().size() == (set1.indices().size() - 2)) # change-of-basis (reindexing) params = master_phil.fetch(source=new_phil).extract() params.mtz_file.crystal_symmetry.change_of_basis = "b,a,c" miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].unit_cell().parameters() == (7.0, 6.0, 8.0, 90.0, 90.0, 90.0)) # again, with bad operator params.mtz_file.crystal_symmetry.change_of_basis = "P222" try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry: pass else: raise Exception_expected # r-free flags in same ASU params.mtz_file.crystal_symmetry.change_of_basis = "l,k,-h" params.mtz_file.output_file = "tst_c_o_b.mtz" miller_arrays = run_and_reload(params, "tst_c_o_b.mtz") assert (miller_arrays[0].completeness(use_binning=False) == miller_arrays[1].completeness(use_binning=False) == 1.0) # expand symmetry (expand_to_p1=True) params.mtz_file.output_file = "tst1.mtz" params = master_phil.fetch(source=new_phil).extract() params.mtz_file.crystal_symmetry.expand_to_p1 = True miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].space_group_info().type().number() == 1) # change space group without changing anything else params = master_phil.fetch(source=new_phil).extract() params.mtz_file.crystal_symmetry.output_space_group = \ sgtbx.space_group_info("P21212") miller_arrays = run_and_reload(params, "tst1.mtz") ma_new = miller_arrays[0].customized_copy(crystal_symmetry=array0) ma_new, array_orig = ma_new.common_sets(other=array0) assert ma_new.sigmas().all_eq(array_orig.sigmas()) # expand symmetry (different output space group) params = master_phil.fetch(source=new_phil).extract() params.mtz_file.crystal_symmetry.output_space_group = \ sgtbx.space_group_info("P21") miller_arrays = run_and_reload(params, "tst1.mtz") assert (miller_arrays[0].space_group_info().type().number() == 4) # incompatible input space_group/unit_cell params.mtz_file.crystal_symmetry.space_group = sgtbx.space_group_info("P4") try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("Input unit cell" in str(e)) else: raise Exception_expected # incompatible output space_group/unit_cell params = master_phil.fetch(source=new_phil).extract() params.mtz_file.crystal_symmetry.output_space_group = \ sgtbx.space_group_info("P6322") try: miller_arrays = run_and_reload(params, "tst1.mtz") except Sorry as e: assert ("incompatible with the specified" in str(e)) else: raise Exception_expected # R-free manipulation (starting from incomplete flags) params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[1].d_min = 1.2 params.mtz_file.output_file = "tst_data2.mtz" miller_arrays = run_and_reload(params, "tst_data2.mtz") assert (miller_arrays[1].data().size() == 138) flags1 = miller_arrays[1].data() params = master_phil.fetch(source=new_phil).extract() # now extend the incomplete flags params.mtz_file.r_free_flags.extend = True for ma in params.mtz_file.miller_array: ma.file_name = "tst_data2.mtz" params.mtz_file.output_file = "tst4a.mtz" miller_arrays = run_and_reload(params, "tst4a.mtz") assert (miller_arrays[1].data().size() == 221) params.mtz_file.r_free_flags.relative_to_complete_set = True miller_arrays = run_and_reload(params, "tst4a.mtz") assert (miller_arrays[1].data().size() == 222) # and now to arbitrarily high resolution params.mtz_file.r_free_flags.relative_to_complete_set = False params.mtz_file.d_min = 0.8 miller_arrays = run_and_reload(params, "tst4a.mtz") assert (miller_arrays[1].data().size() == 221) params.mtz_file.r_free_flags.relative_to_complete_set = True miller_arrays = run_and_reload(params, "tst4a.mtz") assert (miller_arrays[1].data().size() == 428) # export for ccp4 programs (flag=0, everything else > 0) params = master_phil.fetch(source=new_phil).extract() params.mtz_file.r_free_flags.export_for_ccp4 = True params.mtz_file.r_free_flags.preserve_input_values = False params.mtz_file.output_file = "tst_data3.mtz" miller_arrays = run_and_reload(params, "tst_data3.mtz") f_obs_orig = miller_arrays[1] free_selection = (miller_arrays[1].data() == 0) old_selection = (orig_arrays[1].data() == 1) assert (free_selection.all_eq(old_selection)) # preserve input values (now in tst_data3.mtz) params = master_phil.fetch(source=new_phil).extract() params.mtz_file.output_file = "tst4.mtz" for ma in params.mtz_file.miller_array: ma.file_name = "tst_data3.mtz" # flags will be preserved here... miller_arrays = run_and_reload(params, "tst4.mtz") new_selection = (miller_arrays[1].data() == 0) assert (free_selection.all_eq(new_selection)) # ...and here... [extending a CCP4 test set] params.mtz_file.miller_array[1].d_min = 1.2 params.mtz_file.output_file = "tst_data4.mtz" truncated_arrays = run_and_reload(params, "tst_data4.mtz") params = master_phil.fetch(source=new_phil).extract() params.mtz_file.r_free_flags.extend = True for ma in params.mtz_file.miller_array: ma.file_name = "tst_data4.mtz" params.mtz_file.output_file = "tst4.mtz" miller_arrays = run_and_reload(params, "tst4.mtz") common_flags = truncated_arrays[1].common_set(other=miller_arrays[1]) common_flags2 = miller_arrays[1].common_set(other=truncated_arrays[1]) assert (common_flags2.data().all_eq(common_flags.data())) # ...but not here for ma in params.mtz_file.miller_array: ma.file_name = "tst_data3.mtz" params.mtz_file.output_file = "tst4.mtz" params.mtz_file.miller_array[1].d_min = None params.mtz_file.r_free_flags.preserve_input_values = False miller_arrays = run_and_reload(params, "tst4.mtz") new_selection = (miller_arrays[1].common_set(other=f_obs_orig).data() == 1) assert (free_selection.all_eq(new_selection)) # # XXX note that the tests for adjust_fraction in cctbx.r_free_utils are # much more rigorous, because they use a larger test set - I keep it small # here to save on speed (especially I/O time). # # expand the test set params = master_phil.fetch(source=new_phil).extract() params.mtz_file.output_file = "tst4.mtz" params.mtz_file.r_free_flags.fraction = 0.15 params.mtz_file.r_free_flags.adjust_fraction = True try: miller_arrays = run_and_reload(params, "tst4.mtz") except Sorry as s: assert ("resizing" in str(s)) else: raise Exception_expected params.mtz_file.r_free_flags.preserve_input_values = False miller_arrays = run_and_reload(params, "tst4.mtz") flags_expanded = miller_arrays[-1] fraction_free = flags_expanded.data().count( 1) / flags_expanded.data().size() assert (fraction_free > 0.145) and (fraction_free < 0.155) # now shrink it params.mtz_file.r_free_flags.fraction = 0.05 miller_arrays = run_and_reload(params, "tst4.mtz") flags_shrunken = miller_arrays[-1] fraction_free = flags_shrunken.data().count( 1) / flags_shrunken.data().size() assert (fraction_free > 0.049) and (fraction_free < 0.051) # more R-free manipulations mtz2 = array0.as_mtz_dataset(column_root_label="I-obs") flags2 = flags.generate_bijvoet_mates() mtz2.add_miller_array(flags2, column_root_label="R-free-flags") mtz2.mtz_object().write("tst_data4.mtz") new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst5.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data4.mtz labels = I-obs(+),SIGI-obs(+),I-obs(-),SIGI-obs(-) output_labels = I-obs(+) SIGI-obs(+) I-obs(-) SIGI-obs(-) } miller_array { file_name = tst_data4.mtz labels = R-free-flags(+),R-free-flags(-) output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() miller_arrays = run_and_reload(params, "tst5.mtz") assert ((miller_arrays[0].anomalous_flag()) and (not miller_arrays[1].anomalous_flag())) # flags all the same value mtz2 = array0.as_mtz_dataset(column_root_label="I-obs") flags2 = flags.generate_bijvoet_mates() flags2 = flags2.customized_copy(data=flex.int(flags2.data().size(), 1)) mtz2.add_miller_array(flags2, column_root_label="R-free-flags") mtz2.mtz_object().write("tst_data4.mtz") try: miller_arrays = run_and_reload(params, "tst5.mtz") except Sorry as s: pass else: raise Exception_expected # now force them through (no conversion to flex.bool) params.mtz_file.r_free_flags.extend = False params.mtz_file.r_free_flags.warn_if_all_same_value = False miller_arrays = run_and_reload(params, "tst5.mtz") assert miller_arrays[1].data().all_eq(1) # reconstructed amplitudes, yuck mtz3 = array0.as_mtz_dataset(column_root_label="I-obs") indices = array0.average_bijvoet_mates().indices() # XXX why does this come out as an unmerged array? mtz3.add_column(label="F", type="F").set_reals(miller_indices=indices, data=flex.double(indices.size(), 100.)) mtz3.add_column(label="SIGF", type="Q").set_reals(miller_indices=indices, data=flex.double(indices.size(), 5.)) mtz3.add_column(label="DANO", type="D").set_reals(miller_indices=indices, data=flex.double(indices.size(), 10.)) mtz3.add_column(label="SIGDANO", type="Q").set_reals(miller_indices=indices, data=flex.double(indices.size(), 4.)) mtz3.mtz_object().write("tst_data5.mtz") new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst6.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data5.mtz labels = I-obs(+),SIGI-obs(+),I-obs(-),SIGI-obs(-) output_labels = I-obs(+) SIGI-obs(+) I-obs(-) SIGI-obs(-) } miller_array { file_name = tst_data5.mtz labels = F,SIGF,DANO,SIGDANO,merged output_labels = F SIGF DANO SIGDANO } }""") params = master_phil.fetch(source=new_phil).extract() try: miller_arrays = run_and_reload(params, "tst6.mtz") except Sorry as e: assert ("Five columns" in str(e)) else: raise Exception_expected #params.mtz_file.miller_array[1].output_labels = ["F","SIGF"] #params.mtz_file.miller_array #print miller_arrays[1].info().label_string() params.mtz_file.miller_array[1].output_labels.append("ISYM") miller_arrays = run_and_reload(params, "tst6.mtz") arrays = mtz3.mtz_object().as_miller_arrays() assert (arrays[1].is_xray_reconstructed_amplitude_array()) labels = reflection_file_editor.guess_array_output_labels(arrays[1]) assert (labels == ["F", "SIGF", "DANO", "SIGDANO", "ISYM"]) # now merged params.mtz_file.miller_array[1].output_labels = ["F", "SIGF"] params.mtz_file.miller_array[1].anomalous_data = "merged" miller_arrays = run_and_reload(params, "tst6.mtz") assert (miller_arrays[1].is_xray_amplitude_array()) assert (miller_arrays[1].info().label_string() == "F,SIGF") # handle duplicate array labels params.mtz_file.output_file = "tst7.mtz" params.mtz_file.miller_array[1].anomalous_data = "Auto" params.mtz_file.miller_array[1].file_name = "tst_data.mtz" params.mtz_file.miller_array[1].labels = \ "I-obs(+),SIGI-obs(+),I-obs(-),SIGI-obs(-)" params.mtz_file.miller_array[1].output_labels = \ "I-obs(+) SIGI-obs(+) I-obs(-) SIGI-obs(-)".split() try: miller_arrays = run_and_reload(params, "tst7.mtz") except Sorry as e: assert ("Duplicate column label 'I-obs(+)'" in str(e)) else: raise Exception_expected params.mtz_file.resolve_label_conflicts = True miller_arrays = run_and_reload(params, "tst7.mtz") assert (miller_arrays[1].info().label_string() == "I-obs_2(+),SIGI-obs_2(+),I-obs_2(-),SIGI-obs_2(-)") # bad output path params.mtz_file.output_file = os.path.join("/foo", "bar", str(os.getpid()), "data.mtz") try: miller_arrays = run_and_reload(params, "tst6.mtz") except Sorry as e: pass else: raise Exception_expected # no input arrays params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array = [] try: miller_arrays = run_and_reload(params, "tst6.mtz") except Sorry as e: pass else: raise Exception_expected # eliminate_sys_absent symm2 = symm.customized_copy( space_group_info=sgtbx.space_group_info("P222")) set2 = miller.build_set(crystal_symmetry=symm2, anomalous_flag=True, d_min=1.0) data2 = flex.double(set2.indices().size(), 100.) sigmas2 = flex.double(set2.indices().size(), 4.) array2 = set2.array(data=data2, sigmas=sigmas2) array2.set_observation_type_xray_amplitude() flags2 = array2.generate_r_free_flags( use_lattice_symmetry=True).average_bijvoet_mates() mtz2 = array2.as_mtz_dataset(column_root_label="F-obs", wavelength=1.54) mtz2.add_miller_array(flags2, column_root_label="R-free-flags") mtz2.mtz_object().write("tst_data8.mtz") assert (array2.indices().size() == 352) new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst8.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data8.mtz labels = F-obs(+),SIGF-obs(+),F-obs(-),SIGF-obs(-) output_labels = F-obs(+) SIGF-obs(+) F-obs(-) SIGF-obs(-) } miller_array { file_name = tst_data8.mtz labels = R-free-flags output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() miller_arrays = run_and_reload(params, "tst8.mtz") assert (miller_arrays[0].indices().size() == 341) params.mtz_file.crystal_symmetry.eliminate_sys_absent = False miller_arrays = run_and_reload(params, "tst8.mtz") assert (miller_arrays[0].indices().size() == 352) # wavelength assert approx_equal(miller_arrays[0].info().wavelength, 1.54) params.mtz_file.wavelength = 1.116 params.mtz_file.output_file = "tst9.mtz" miller_arrays = run_and_reload(params, "tst9.mtz") assert approx_equal(miller_arrays[0].info().wavelength, 1.116) # second MTZ file with wavelength set by default to 1.0 (and ignored) mtz3 = array2.average_bijvoet_mates().as_mtz_dataset( column_root_label="F-obs_2", wavelength=None) mtz3.add_miller_array(flags2, column_root_label="R-free-flags") mtz3.mtz_object().write("tst_data9.mtz") new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst9.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data8.mtz labels = F-obs(+),SIGF-obs(+),F-obs(-),SIGF-obs(-) output_labels = F-obs(+) SIGF-obs(+) F-obs(-) SIGF-obs(-) } miller_array { file_name = tst_data9.mtz labels = F-obs_2,SIGF-obs_2 output_labels = F-obs_2 SIGF-obs_2 } miller_array { file_name = tst_data8.mtz labels = R-free-flags output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() miller_arrays = run_and_reload(params, "tst9.mtz") assert approx_equal(miller_arrays[0].info().wavelength, 1.54) # now try two MTZ files with wavelengths not equal to 1.0 mtz3 = array2.average_bijvoet_mates().as_mtz_dataset( column_root_label="F-obs_2", wavelength=1.116) mtz3.add_miller_array(flags2, column_root_label="R-free-flags") mtz3.mtz_object().write("tst_data9.mtz") try: miller_arrays = run_and_reload(params, "tst9.mtz") except Sorry as s: assert ( str(s) == """Multiple wavelengths present in input experimental data arrays: 1.116, 1.54. Please specify the wavelength parameter explicitly.""" ) else: raise Exception_expected params.mtz_file.wavelength = 1.54 miller_arrays = run_and_reload(params, "tst9.mtz") assert approx_equal(miller_arrays[0].info().wavelength, 1.54) # Make sure wavelengths from non-experimental data arrays are ignored new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst9.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data8.mtz labels = F-obs(+),SIGF-obs(+),F-obs(-),SIGF-obs(-) output_labels = F-obs(+) SIGF-obs(+) F-obs(-) SIGF-obs(-) } miller_array { file_name = tst_data9.mtz labels = R-free-flags output_labels = R-free-flags } }""") params = master_phil.fetch(source=new_phil).extract() miller_arrays = run_and_reload(params, "tst9.mtz") assert approx_equal(miller_arrays[0].info().wavelength, 1.54) # map coefficients n_refl = len(array2.indices()) d1 = flex.random_double(n_refl) d2 = flex.random_double(n_refl) coeffs_data = flex.complex_double(d1, d2) map_coeffs = array2.customized_copy(data=coeffs_data, sigmas=None).average_bijvoet_mates() map_coeffs.as_mtz_dataset( column_root_label="2FOFCWT").mtz_object().write("tst_data10.mtz") new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst10.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 6,7,8,90,90,90 miller_array { file_name = tst_data10.mtz labels = 2FOFCWT,PHI2FOFCWT output_labels = 2FOFCWT PHI2FOFCWT } }""") params = master_phil.fetch(source=new_phil).extract() miller_arrays = run_and_reload(params, "tst10.mtz") # R-free mismatches symm = crystal.symmetry(space_group_info=sgtbx.space_group_info("P212121"), unit_cell=uctbx.unit_cell( (60, 70, 80, 90, 90, 90))) set1 = miller.build_set(crystal_symmetry=symm, anomalous_flag=True, d_min=1.0) flags = set1.generate_r_free_flags(fraction=0.1, max_free=None) n_flipped = 0 for i_hkl, (h, k, l) in enumerate(flags.indices()): if (i_hkl % 100 == 0) and (h > 0) and (k > 0) and (l > 0): flag = flags.data()[i_hkl] if (not flag): n_flipped += 1 flags.data()[i_hkl] = True assert (n_flipped > 0) flags.as_mtz_dataset( column_root_label="FreeR_flag").mtz_object().write("tst_data11.mtz") new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst11.mtz crystal_symmetry.space_group = P212121 crystal_symmetry.unit_cell = 60,70,80,90,90,90 miller_array { file_name = tst_data11.mtz labels = FreeR_flag(+),FreeR_flag(-) output_labels = FreeR_flag } }""") params = master_phil.fetch(source=new_phil).extract() try: miller_arrays = run_and_reload(params, "tst11.mtz") except Sorry: pass else: raise Exception_expected params.mtz_file.r_free_flags.remediate_mismatches = True params.mtz_file.r_free_flags.preserve_input_values = False miller_arrays = run_and_reload(params, "tst11.mtz") # R-free flags extension - corner case where (1,1,0) was being missed symm = crystal.symmetry(unit_cell=(45.23, 51.24, 80.49, 90, 90, 90), space_group_symbol="P212121") ms = miller.build_set(crystal_symmetry=symm, d_min=1.67290, anomalous_flag=True).resolution_filter(d_max=33.9095) ma = ms.array(data=flex.double(ms.size(), 100.0), sigmas=flex.double(ms.size(), 1.0)) #FIXME check on other systems #assert (ma.average_bijvoet_mates().size() == 22238) ma.set_observation_type_xray_intensity() ma.export_as_scalepack_unmerged(file_name="tst_data12.sca") flags = ma.generate_r_free_flags().average_bijvoet_mates() cs2 = symm.customized_copy(unit_cell=(45.215, 51.203, 80.425, 90, 90, 90)) flags = flags.customized_copy(crystal_symmetry=cs2).resolution_filter( d_max=24.396, d_min=1.94701) flags.as_mtz_dataset( column_root_label="FreeR_flag").mtz_object().write("tst_flags.mtz") new_phil = libtbx.phil.parse(""" mtz_file { output_file = tst12.mtz crystal_symmetry { unit_cell = 45.215 51.203 80.425 90 90 90 space_group = P 21 21 21 output_unit_cell = 45.23 51.24 80.49 90 90 90 } miller_array { file_name = tst_data12.sca labels = I(+),SIGI(+),I(-),SIGI(-),merged output_labels = I(+) SIGI(+) I(-) SIGI(-) } miller_array { file_name = tst_flags.mtz labels = FreeR_flag output_labels = FreeR_flag } r_free_flags { extend = True relative_to_complete_set = True } }""") params = master_phil.fetch(source=new_phil).extract() miller_arrays = run_and_reload(params, "tst12.mtz") f_obs, r_free_flags = miller_arrays ls = f_obs.average_bijvoet_mates().lone_set(other=r_free_flags) assert (ls.size() == 0)
params.mtz_file.miller_array[0].output_labels = ["I-obs", "SIGI-obs"] miller_arrays = run_and_reload(params, "tst1.mtz") assert (not miller_arrays[0].anomalous_flag()) params.mtz_file.miller_array[1].anomalous_data = "anomalous" try : run_and_reload(params, "tst1.mtz") except Sorry, e : assert ("not specified enough" in str(e)), str(e) else : raise Exception_expected params.mtz_file.miller_array[1].output_labels = ['R-free-flags(+)', 'R-free-flags(-)'] miller_arrays = run_and_reload(params, "tst1.mtz") assert miller_arrays[1].anomalous_flag() # filter by signal-to-noise ratio params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].filter_by_signal_to_noise = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") data = miller_arrays[0].data() sigmas = miller_arrays[0].sigmas() assert (data.size() == 321) and ((data / sigmas).all_ge(2.0)) # data scaling params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].scale_factor = 2.0 miller_arrays = run_and_reload(params, "tst1.mtz") data2 = miller_arrays[0].data() assert approx_equal(flex.max(data2), 2.*flex.max(data)) # remove negatives params = master_phil.fetch(source=new_phil).extract() params.mtz_file.miller_array[0].remove_negatives = True miller_arrays = run_and_reload(params, "tst1.mtz")