def test_real_map_datatype(): data_dir = os.path.dirname(os.path.abspath(__file__)) data_ccp4 = os.path.join(data_dir, 'data', 'non_zero_origin_map.ccp4') dm = DataManager(['real_map', 'phil']) dm.process_real_map_file(data_ccp4) assert dm.has_real_maps() # test custom PHIL dm.write_phil_file(dm.export_phil_scope().as_str(), filename='test.phil', overwrite=True) loaded_phil = iotbx.phil.parse(file_name='test.phil') new_dm = DataManager(['real_map', 'phil']) new_dm.load_phil_scope(loaded_phil) assert data_ccp4 == new_dm.get_default_real_map_name() os.remove('test.phil') # test writing and reading file mm = dm.get_real_map() mm.shift_origin() dm.write_real_map_file(mm, filename='test.ccp4', overwrite=True) dm.process_real_map_file('test.ccp4') new_mm = dm.get_real_map('test.ccp4') assert not new_mm.is_similar(mm) new_mm.shift_origin() assert new_mm.is_similar(mm) os.remove('test.ccp4')
def test_miller_array_datatype(): data_dir = os.path.dirname(os.path.abspath(__file__)) data_mtz = os.path.join(data_dir, 'data', 'insulin_unmerged_cutted_from_ccp4.mtz') dm = DataManager(['miller_array', 'phil']) dm.process_miller_array_file(data_mtz) # test labels labels = [ 'M_ISYM', 'BATCH', 'I, SIGI', 'IPR, SIGIPR', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS' ] for label in dm.get_miller_array_labels(): assert (label in labels) assert (len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels())) # test access by label label = dm.get_miller_array_labels()[3] new_label = ', '.join( dm.get_miller_arrays(labels=[label])[0].info().labels) assert (label == new_label) # test custom PHIL dm.write_phil_file('test.phil', dm.export_phil_scope().as_str(), overwrite=True) loaded_phil = libtbx.phil.parse(file_name='test.phil') new_dm = DataManager(['miller_array', 'phil']) new_dm.load_phil_scope(loaded_phil) assert (data_mtz == new_dm.get_default_miller_array_name()) for label in new_dm.get_miller_array_labels(): assert (label in labels) os.remove('test.phil')
def test_miller_array_datatype(): data_dir = os.path.dirname(os.path.abspath(__file__)) data_mtz = os.path.join(data_dir, 'data', 'insulin_unmerged_cutted_from_ccp4.mtz') dm = DataManager(['miller_array', 'phil']) dm.process_miller_array_file(data_mtz) # test labels labels = [ 'M_ISYM', 'BATCH', 'I,SIGI,merged', 'IPR,SIGIPR,merged', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS' ] for label in dm.get_miller_array_labels(): assert label in labels assert len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels()) # test access by label label = dm.get_miller_array_labels()[3] new_label = dm.get_miller_arrays(labels=[label])[0].info().label_string() assert label == new_label # test custom PHIL dm.write_phil_file(dm.export_phil_scope().as_str(), filename='test.phil', overwrite=True) loaded_phil = iotbx.phil.parse(file_name='test.phil') new_dm = DataManager(['miller_array', 'phil']) new_dm.load_phil_scope(loaded_phil) assert data_mtz == new_dm.get_default_miller_array_name() for label in new_dm.get_miller_array_labels(): assert label in labels os.remove('test.phil') # test type assert dm.get_miller_array_type() == 'x_ray' label = labels[3] dm.set_miller_array_type(data_mtz, label, 'electron') assert dm.get_miller_array_type(label=label) == 'electron' dm.write_phil_file(dm.export_phil_scope().as_str(), filename='test_phil', overwrite=True) loaded_phil = iotbx.phil.parse(file_name='test_phil') new_dm.load_phil_scope(loaded_phil) assert new_dm.get_miller_array_type(label=label) == 'electron' new_dm = DataManager(['miller_array']) try: new_dm.set_default_miller_array_type('q') except Sorry: pass new_dm.set_default_miller_array_type('neutron') new_dm.process_miller_array_file(data_mtz) assert new_dm.get_miller_array_type(label=label) == 'neutron' os.remove('test_phil') # test writing file arrays = dm.get_miller_arrays() dataset = arrays[2].as_mtz_dataset(column_root_label='label1') dataset.add_miller_array(miller_array=arrays[3], column_root_label='label2') mtz_object = dataset.mtz_object() dm.write_miller_array_file(mtz_object, filename='test.mtz', overwrite=True) dm.process_miller_array_file('test.mtz') new_labels = dm.get_miller_array_labels('test.mtz') assert 'label1,SIGlabel1' in new_labels assert 'label2,SIGlabel2' in new_labels os.remove('test.mtz') # test file server fs1 = dm.get_reflection_file_server() fs2 = dm.get_reflection_file_server([data_mtz, data_mtz]) assert 2 * len(fs1.miller_arrays) == len(fs2.miller_arrays) cs = crystal.symmetry( unit_cell=dm.get_miller_arrays()[0].crystal_symmetry().unit_cell(), space_group_symbol='P1') fs = dm.get_reflection_file_server(crystal_symmetry=cs) assert fs.crystal_symmetry.is_similar_symmetry(cs) assert not fs.crystal_symmetry.is_similar_symmetry( dm.get_miller_arrays()[0].crystal_symmetry()) fs = dm.get_reflection_file_server(labels=['I,SIGI,merged']) assert len(fs.get_miller_arrays(None)) == 1 miller_array = fs.get_amplitudes(None, None, True, None, None) assert miller_array.info().label_string() == 'I,as_amplitude_array,merged' for label in dm.get_miller_array_labels(): dm.set_miller_array_type(label=label, array_type='electron') fs = dm.get_reflection_file_server(array_type='x_ray') assert len(fs.get_miller_arrays(None)) == 0 fs = dm.get_reflection_file_server(array_type='electron') assert len(fs.get_miller_arrays(None)) == 13 fs = dm.get_reflection_file_server( filenames=[data_mtz], labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']], array_type='neutron') assert len(fs.get_miller_arrays(None)) == 0 for label in ['I,SIGI,merged', 'IPR,SIGIPR,merged']: dm.set_miller_array_type(label=label, array_type='x_ray') fs = dm.get_reflection_file_server( filenames=[data_mtz], labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']], array_type='x_ray') assert len(fs.get_miller_arrays(data_mtz)) == 2 fs = dm.get_reflection_file_server(filenames=[data_mtz], array_type='x_ray') assert len(fs.get_miller_arrays(data_mtz)) == 2 fs = dm.get_reflection_file_server(filenames=[data_mtz], array_type='electron') assert len(fs.get_miller_arrays(data_mtz)) == 11 # test subset of labels label_subset = labels[3:8] dm = DataManager(['miller_array', 'phil']) dm.process_miller_array_file(data_mtz) dm._miller_array_labels[data_mtz] = label_subset dm.set_miller_array_type(label=label_subset[2], array_type='electron') assert dm.get_miller_array_type(label=label_subset[2]) == 'electron' dm.write_phil_file(dm.export_phil_scope().as_str(), filename='test.phil', overwrite=True) loaded_phil = iotbx.phil.parse(file_name='test.phil') new_dm = DataManager(['miller_array', 'phil']) new_dm.load_phil_scope(loaded_phil) assert new_dm.get_miller_array_type(label=label_subset[2]) == 'electron' fs = new_dm.get_reflection_file_server(array_type='x_ray') assert len(fs.get_miller_arrays(None)) == 4 fs = new_dm.get_reflection_file_server(array_type='electron') assert len(fs.get_miller_arrays(None)) == 1 os.remove('test.phil') label_subset = list() dm = DataManager(['miller_array', 'phil']) dm.process_miller_array_file(data_mtz) dm._miller_array_labels[data_mtz] = label_subset dm.write_phil_file(dm.export_phil_scope().as_str(), filename='test.phil', overwrite=True) loaded_phil = iotbx.phil.parse(file_name='test.phil') new_dm = DataManager(['miller_array', 'phil']) new_dm.load_phil_scope(loaded_phil) fs = new_dm.get_reflection_file_server(array_type='x_ray') assert len(fs.get_miller_arrays(None)) == 13 fs = new_dm.get_reflection_file_server(array_type='electron') assert len(fs.get_miller_arrays(None)) == 0 os.remove('test.phil')
def test_data_manager(): a = DataManager(['model']) a.add_model('a', 'b') a.add_model('c', 'd') assert a.get_model() == 'b' assert a.get_model('a') == 'b' assert a.get_model('c') == 'd' assert a.get_model_names() == ['a', 'c'] assert a.has_models() assert a.has_models(exact_count=True, expected_n=2) assert not a.has_models(expected_n=3, raise_sorry=False) # exporting phil working_phil = a.export_phil_scope() assert len(working_phil.extract().data_manager.model) == 2 # data tracking try: a.has_models(expected_n=3, raise_sorry=True) except Sorry: pass try: a.has_models(exact_count=True, raise_sorry=True) except Sorry: pass a.set_default_model('c') assert a.get_model() == 'd' assert a.get_model_names() == ['a', 'c' ] or a.get_model_names() == ['c', 'a'] a.remove_model('c') try: a.get_model() except Sorry: pass try: a.get_model('missing') except Sorry: pass try: a.set_default_model('missing') except Sorry: pass a = DataManager(datatypes=['sequence', 'phil']) assert a.get_sequence_names() == [] assert not hasattr(a, 'get_model') # phil functions test_phil_str = ''' data_manager { phil_files = data_manager_test.eff } ''' with open('data_manager_test.eff', 'w') as f: f.write(test_phil_str) # loading file with get function assert len(a.get_phil_names()) == 0 p = a.get_phil('data_manager_test.eff') assert type(p) == libtbx.phil.scope assert 'data_manager_test.eff' in a.get_phil_names() # loading file with phil a = DataManager(datatypes=['phil']) test_phil = iotbx.phil.parse(test_phil_str) a.load_phil_scope(test_phil) assert 'data_manager_test.eff' in a.get_phil_names() assert a.get_default_phil_name() == 'data_manager_test.eff' os.remove('data_manager_test.eff') # writing a = DataManager(datatypes=['model', 'phil', 'sequence']) a.add_model('a', 'b') a.add_phil('c', 'd') a.add_sequence('e', 'f') a.write_model_file(a.get_model(), filename='a.dat', overwrite=True) a.write_phil_file(a.get_phil(), filename='c.dat', overwrite=True) a.write_sequence_file(a.get_sequence(), filename='e.dat', overwrite=True) with open('a.dat', 'r') as f: lines = f.readlines() assert lines[0] == 'b' os.remove('a.dat') os.remove('c.dat') os.remove('e.dat')
def test_model_datatype(): import mmtbx.monomer_library.server try: mon_lib_srv = mmtbx.monomer_library.server.server() except mmtbx.monomer_library.server.MonomerLibraryServerError: print( "Can not initialize monomer_library, skipping test_model_datatype." ) return # 1yjp model_str = ''' CRYST1 21.937 4.866 23.477 90.00 107.08 90.00 P 1 21 1 2 ORIGX1 1.000000 0.000000 0.000000 0.00000 ORIGX2 0.000000 1.000000 0.000000 0.00000 ORIGX3 0.000000 0.000000 1.000000 0.00000 SCALE1 0.045585 0.000000 0.014006 0.00000 SCALE2 0.000000 0.205508 0.000000 0.00000 SCALE3 0.000000 0.000000 0.044560 0.00000 ATOM 1 N GLY A 1 -9.009 4.612 6.102 1.00 16.77 N ATOM 2 CA GLY A 1 -9.052 4.207 4.651 1.00 16.57 C ATOM 3 C GLY A 1 -8.015 3.140 4.419 1.00 16.16 C ATOM 4 O GLY A 1 -7.523 2.521 5.381 1.00 16.78 O ATOM 5 N ASN A 2 -7.656 2.923 3.155 1.00 15.02 N ATOM 6 CA ASN A 2 -6.522 2.038 2.831 1.00 14.10 C ATOM 7 C ASN A 2 -5.241 2.537 3.427 1.00 13.13 C ATOM 8 O ASN A 2 -4.978 3.742 3.426 1.00 11.91 O ATOM 9 CB ASN A 2 -6.346 1.881 1.341 1.00 15.38 C ATOM 10 CG ASN A 2 -7.584 1.342 0.692 1.00 14.08 C ATOM 11 OD1 ASN A 2 -8.025 0.227 1.016 1.00 17.46 O ATOM 12 ND2 ASN A 2 -8.204 2.155 -0.169 1.00 11.72 N ATOM 13 N ASN A 3 -4.438 1.590 3.905 1.00 12.26 N ATOM 14 CA ASN A 3 -3.193 1.904 4.589 1.00 11.74 C ATOM 15 C ASN A 3 -1.955 1.332 3.895 1.00 11.10 C ATOM 16 O ASN A 3 -1.872 0.119 3.648 1.00 10.42 O ATOM 17 CB ASN A 3 -3.259 1.378 6.042 1.00 12.15 C ATOM 18 CG ASN A 3 -2.006 1.739 6.861 1.00 12.82 C ATOM 19 OD1 ASN A 3 -1.702 2.925 7.072 1.00 15.05 O ATOM 20 ND2 ASN A 3 -1.271 0.715 7.306 1.00 13.48 N ATOM 21 N GLN A 4 -1.005 2.228 3.598 1.00 10.29 N ATOM 22 CA GLN A 4 0.384 1.888 3.199 1.00 10.53 C ATOM 23 C GLN A 4 1.435 2.606 4.088 1.00 10.24 C ATOM 24 O GLN A 4 1.547 3.843 4.115 1.00 8.86 O ATOM 25 CB GLN A 4 0.656 2.148 1.711 1.00 9.80 C ATOM 26 CG GLN A 4 1.944 1.458 1.213 1.00 10.25 C ATOM 27 CD GLN A 4 2.504 2.044 -0.089 1.00 12.43 C ATOM 28 OE1 GLN A 4 2.744 3.268 -0.190 1.00 14.62 O ATOM 29 NE2 GLN A 4 2.750 1.161 -1.091 1.00 9.05 N ATOM 30 N GLN A 5 2.154 1.821 4.871 1.00 10.38 N ATOM 31 CA GLN A 5 3.270 2.361 5.640 1.00 11.39 C ATOM 32 C GLN A 5 4.594 1.768 5.172 1.00 11.52 C ATOM 33 O GLN A 5 4.768 0.546 5.054 1.00 12.05 O ATOM 34 CB GLN A 5 3.056 2.183 7.147 1.00 11.96 C ATOM 35 CG GLN A 5 1.829 2.950 7.647 1.00 10.81 C ATOM 36 CD GLN A 5 1.344 2.414 8.954 1.00 13.10 C ATOM 37 OE1 GLN A 5 0.774 1.325 9.002 1.00 10.65 O ATOM 38 NE2 GLN A 5 1.549 3.187 10.039 1.00 12.30 N ATOM 39 N ASN A 6 5.514 2.664 4.856 1.00 11.99 N ATOM 40 CA ASN A 6 6.831 2.310 4.318 1.00 12.30 C ATOM 41 C ASN A 6 7.854 2.761 5.324 1.00 13.40 C ATOM 42 O ASN A 6 8.219 3.943 5.374 1.00 13.92 O ATOM 43 CB ASN A 6 7.065 3.016 2.993 1.00 12.13 C ATOM 44 CG ASN A 6 5.961 2.735 2.003 1.00 12.77 C ATOM 45 OD1 ASN A 6 5.798 1.604 1.551 1.00 14.27 O ATOM 46 ND2 ASN A 6 5.195 3.747 1.679 1.00 10.07 N ATOM 47 N TYR A 7 8.292 1.817 6.147 1.00 14.70 N ATOM 48 CA TYR A 7 9.159 2.144 7.299 1.00 15.18 C ATOM 49 C TYR A 7 10.603 2.331 6.885 1.00 15.91 C ATOM 50 O TYR A 7 11.041 1.811 5.855 1.00 15.76 O ATOM 51 CB TYR A 7 9.061 1.065 8.369 1.00 15.35 C ATOM 52 CG TYR A 7 7.665 0.929 8.902 1.00 14.45 C ATOM 53 CD1 TYR A 7 6.771 0.021 8.327 1.00 15.68 C ATOM 54 CD2 TYR A 7 7.210 1.756 9.920 1.00 14.80 C ATOM 55 CE1 TYR A 7 5.480 -0.094 8.796 1.00 13.46 C ATOM 56 CE2 TYR A 7 5.904 1.649 10.416 1.00 14.33 C ATOM 57 CZ TYR A 7 5.047 0.729 9.831 1.00 15.09 C ATOM 58 OH TYR A 7 3.766 0.589 10.291 1.00 14.39 O ATOM 59 OXT TYR A 7 11.358 2.999 7.612 1.00 17.49 O TER 60 TYR A 7 HETATM 61 O HOH A 8 -6.471 5.227 7.124 1.00 22.62 O HETATM 62 O HOH A 9 10.431 1.858 3.216 1.00 19.71 O HETATM 63 O HOH A 10 -11.286 1.756 -1.468 1.00 17.08 O HETATM 64 O HOH A 11 11.808 4.179 9.970 1.00 23.99 O HETATM 65 O HOH A 12 13.605 1.327 9.198 1.00 26.17 O HETATM 66 O HOH A 13 -2.749 3.429 10.024 1.00 39.15 O HETATM 67 O HOH A 14 -1.500 0.682 10.967 1.00 43.49 O MASTER 238 0 0 0 0 0 0 6 66 1 0 1 END ''' # test reading/writing PDB test_filename = 'test_model.pdb' test_output_filename = 'test_model_output.pdb' test_eff = 'model.eff' dm = DataManager(['model']) dm.process_model_str(test_filename, model_str) dm.write_model_file(model_str, filename=test_output_filename, overwrite=True) m = dm.get_model(test_output_filename) assert test_output_filename in dm.get_model_names() dm.write_model_file(m, overwrite=True) pdb_filename = 'cctbx_program.pdb' assert os.path.exists(pdb_filename) dm.process_model_file(pdb_filename) assert not dm.get_model(pdb_filename).input_model_format_cif() dm.write_model_file(m, test_filename, overwrite=True) # test reading PDB writing CIF test_filename = 'test_model.pdb' test_output_filename = 'test_model.cif' dm = DataManager(['model']) dm.process_model_str(test_filename, model_str) m = dm.get_model(test_filename) dm.write_model_file(m, filename=test_output_filename, format='cif', overwrite=True) m = dm.get_model(test_output_filename) assert test_output_filename in dm.get_model_names() dm.write_model_file(m, overwrite=True) cif_filename = 'cctbx_program.cif' assert os.path.exists(cif_filename) dm.process_model_file(cif_filename) assert dm.get_model(cif_filename).input_model_format_cif() # test type assert dm.get_model_type() == 'x_ray' dm.set_model_type(test_filename, 'neutron') assert dm.get_model_type() == 'neutron' phil_scope = dm.export_phil_scope() extract = phil_scope.extract() assert extract.data_manager.model[0].type == 'neutron' with open(test_eff, 'w') as f: f.write(phil_scope.as_str()) new_phil_scope = iotbx.phil.parse(file_name=test_eff) new_dm = DataManager(['model']) new_dm.load_phil_scope(new_phil_scope) assert new_dm.get_model_type(test_filename) == 'neutron' new_dm = DataManager(['model']) try: new_dm.set_default_model_type('nonsense') except Sorry: pass new_dm.set_default_model_type('electron') new_dm.process_model_file(test_filename) assert new_dm.get_model_type() == 'electron' assert len(new_dm.get_model_names()) == 1 assert len(new_dm.get_model_names(model_type='electron')) == 1 assert len(new_dm.get_model_names(model_type='neutron')) == 0 os.remove(test_eff) os.remove(test_filename) # test reading/writing CIF test_filename = 'test_model_datatype.cif' dm.write_model_file(dm.get_model().model_as_mmcif(), filename=test_filename, overwrite=True) dm.process_model_file(test_filename) os.remove(test_filename) assert test_filename in dm.get_model_names() m = dm.get_model(test_filename) dm.write_model_file(m, overwrite=True) cif_filename = 'cctbx_program.cif' assert os.path.exists(cif_filename) dm.process_model_file(cif_filename) assert dm.get_model(cif_filename).input_model_format_cif() os.remove(pdb_filename) os.remove(cif_filename) # test pdb_interpretation extract = mmtbx.model.manager.get_default_pdb_interpretation_params() extract.pdb_interpretation.use_neutron_distances = True dm.update_pdb_interpretation_for_model(test_filename, extract) assert dm.get_model(test_filename).restraints_manager is None
class CCTBXParser(ParserBase): def __init__(self, program_class, custom_process_arguments=None, logger=None, *args, **kwargs): ''' ''' # program name self.prog = os.getenv('LIBTBX_DISPATCHER_NAME') if (self.prog is None): self.prog = sys.argv[0] self.prefix = self.prog.split('.')[-1] # PHIL filenames self.data_filename = self.prefix + '_data.eff' self.modified_filename = self.prefix + '_modified.eff' self.all_filename = self.prefix + '_all.eff' # terminal width self.text_width = 79 # print header border = '-' * self.text_width description = border + program_class.description + border epilog = border + program_class.epilog super(CCTBXParser, self).__init__( prog=self.prog, description=description, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter, *args, **kwargs) # default values self.program_class = program_class self.custom_process_arguments = custom_process_arguments self.logger = logger if (self.logger is None): self.logger = logging.getLogger('main') self.data_manager = DataManager(datatypes=program_class.datatypes, logger=self.logger) # add PHIL converters if available if (len(program_class.phil_converters) > 0): iotbx.phil.default_converter_registry = \ libtbx.phil.extended_converter_registry( additional_converters=program_class.phil_converters, base_registry=iotbx.phil.default_converter_registry) # set up master and working PHIL scopes self.master_phil = iotbx.phil.parse(program_class.master_phil_str, process_includes=True) required_output_phil = iotbx.phil.parse( ProgramTemplate.output_phil_str) self.master_phil.adopt_scope(required_output_phil) self.working_phil = None self.add_default_options() # --------------------------------------------------------------------------- def add_default_options(self): ''' ''' # --show-defaults by itself is set to 0 # --show-defaults=n sets it to n and it can only be {0, 1, 2, 3} self.add_argument( '--show-defaults', '--show_defaults', nargs='?', const=0, type=int, choices=list(range(0, 4)), help='show default parameters with expert level (default=0)') # --attributes-level by itself is set to 1 # --attributes-level=n sets it to n and it can only be {1, 2, 3} self.add_argument('--attributes-level', '--attributes_level', nargs='?', const=1, type=int, choices=list(range(0, 4)), help='show parameters with attributes (default=0)') # --write-data # switch for writing only DataManager PHIL parameters self.add_argument( '--write-data', '--write_data', action='store_true', help='write DataManager PHIL parameters to file (%s)' % \ self.data_filename ) # --write-modified # switch for writing only modified PHIL parameters self.add_argument( '--write-modified', '--write_modified', action='store_true', help='write modifed PHIL parameters to file (%s)' % \ self.modified_filename ) # --write-all # switch for writing all PHIL parameters self.add_argument( '--write-all', '--write_all', action='store_true', help= 'write all (modified + default + data) PHIL parameters to file (%s)' % self.all_filename) # --overwrite # switch for overwriting files, takes precedence over PHIL definition self.add_argument( '--overwrite', action='store_true', default=False, help= 'overwrite files, this overrides the output.overwrite PHIL parameter' ) # --citations will use the default format # --citations=<format> will use the specified format self.add_argument( '--citations', nargs='?', const='default', type=str, choices=['default', 'cell', 'iucr'], help='show citation(s) for program in different formats') # --------------------------------------------------------------------------- def parse_args(self, args): ''' ''' # default behavior with no arguments if (len(args) == 0): self.print_help() self.exit() # parse arguments self.namespace = super(CCTBXParser, self).parse_args(args) # process command-line options if (self.namespace.attributes_level is not None): if (self.namespace.show_defaults is None): self.error( '--attributes-level requires --show-defaults to be set') if (self.namespace.show_defaults is not None): self.master_phil.show( expert_level=self.namespace.show_defaults, attributes_level=self.namespace.attributes_level, out=self.logger) self.exit() if (self.namespace.citations is not None): self.show_citations() self.exit() # start program header print('Starting %s' % self.prog, file=self.logger) print('on %s by %s' % (time.asctime(), getpass.getuser()), file=self.logger) print('=' * self.text_width, file=self.logger) print('', file=self.logger) # process files if (self.parse_files): self.process_files(self.namespace.files) # process phil and phil files if (self.parse_phil): self.process_phil(self.namespace.phil) # process directories if (self.parse_dir): self.process_dir(self.namespace.dir) # custom processing of arguments (if available) # the function for custom processing of arguments should take a CCTBXParser # object as its argument. The function should modify the # CCTBXParser.namespace.unknown, CCTBXParser.data_manager, # CCTBXParser.working_phil, or other CCTBXParser members. # At the end of the function, CCTBXParser.working_phil should have the final # libtbx.phil.scope object (not libtbx.phil.scope_extract) for the program # A libtbx.phil.scope_extract object can be converted into a # libtbx.phil.scope object by # CCTBXParser.master_phil.format(python_object=<scope_extract>) if (self.custom_process_arguments is not None): self.custom_process_arguments(self) assert (isinstance(self.working_phil, libtbx.phil.scope)) # post processing after all arguments are parsed self.post_process() # show final PHIL parameters self.show_phil_summary() return self.namespace # --------------------------------------------------------------------------- def process_files(self, file_list, message='Processing files:'): ''' Second pass to process files. The first pass already checked that these files exist. There may be conditions where the file is deleted in the time between the first pass and calling this function. Use iotbx.file_reader.any_file to process files. Will need updating to work with mmtbx.model.manager class more efficiently ''' print(message, file=self.logger) print('-' * self.text_width, file=self.logger) print('', file=self.logger) printed_something = False unused_files = list() for filename in file_list: a = any_file(filename) process_function = 'process_%s_file' % data_manager_type.get( a.file_type) if (hasattr(self.data_manager, process_function)): getattr(self.data_manager, process_function)(filename) print(' Found %s, %s' % (data_manager_type[a.file_type], filename), file=self.logger) printed_something = True else: unused_files.append(filename) # show unrecognized files if (len(unused_files) > 0): if (printed_something): print('', file=self.logger) print(' Files not used by program:', file=self.logger) print(' --------------------------', file=self.logger) for filename in unused_files: print(' %s' % filename, file=self.logger) printed_something = True # process PHIL files for DataManager scope in order from command-line # files are appended and the default is not overridden # files from the command-line take precedence phil_names = self.data_manager.get_phil_names() for name in phil_names: phil = self.data_manager.get_phil(name) if (hasattr(phil.extract(), 'data_manager')): self.data_manager.load_phil_scope(phil) if (not printed_something): print(' No files found', file=self.logger) print('', file=self.logger) # --------------------------------------------------------------------------- def process_phil(self, phil_list): '''' Process PHIL arguments Also checks PHIL arguments (command line and files) for parameters that specify files (.type = path) ''' print('Processing PHIL parameters:', file=self.logger) print('-' * self.text_width, file=self.logger) print('', file=self.logger) printed_something = False data_sources = list() sources = list() unused_phil = list() # PHIL files are processed in order from command-line if (self.data_manager.has_phils()): phil_names = self.data_manager.get_phil_names() phil = list() print(' Adding PHIL files:', file=self.logger) print(' ------------------', file=self.logger) for name in phil_names: # remove DataManager scope since input files are already loaded phil_scope = self.data_manager.get_phil(name) for phil_object in phil_scope.objects: if (phil_object.name == 'data_manager'): phil_scope.objects.remove(phil_object) phil.append(phil_scope) print(' %s' % name, file=self.logger) data_sources.extend(phil) print('', file=self.logger) printed_something = True # command-line PHIL arguments override any previous settings and are # processed in given order def custom_processor(arg): unused_phil.append(arg) return True if (len(phil_list) > 0): interpreter = self.master_phil.command_line_argument_interpreter( home_scope='') print(' Adding command-line PHIL:', file=self.logger) print(' -------------------------', file=self.logger) for phil in phil_list: print(' %s' % phil, file=self.logger) print('', file=self.logger) printed_something = True working = interpreter.process_args( phil_list, custom_processor=custom_processor) if (len(working) > 0): sources.extend(working) if (self.namespace.overwrite): # override overwrite if True sources.append(iotbx.phil.parse('output.overwrite=True')) if ((len(data_sources) + len(sources)) > 0): self.working_phil, more_unused_phil = self.master_phil.fetch( sources=data_sources + sources, track_unused_definitions=True) unused_phil.extend(more_unused_phil) else: self.working_phil = self.master_phil.fetch() # show unrecognized parameters and abort if (len(unused_phil) > 0): print(' Unrecognized PHIL parameters:', file=self.logger) print(' -----------------------------', file=self.logger) for phil in unused_phil: print(' %s' % phil, file=self.logger) print('', file=self.logger) error_message = 'Some PHIL parameters are not recognized by %s.\n' % \ self.prog error_message += wordwrap( 'Please run this program with the --show-defaults option to see what parameters are available.', max_chars=self.text_width) + '\n' error_message += wordwrap( 'PHIL parameters in files should be fully specified (e.g. "output.overwrite" instead of just "overwrite")', max_chars=self.text_width) + '\n' raise Sorry(error_message) # process input phil for file/directory defintions and add to DataManager # Note: if a PHIL file is input as a PHIL parameter, the contents of the # file will NOT be parsed and validated. The PHIL file should be provided # as a command-line argument. This is mostly for finding data files # defined by PHIL parameters that should be added to the DataManager diff_phil = self.master_phil.fetch_diff(self.working_phil) paths = self.check_phil_for_paths(diff_phil) if (len(paths) > 0): files = list() dirs = list() for path in paths: if (path is not None): if (os.path.isfile(path)): files.append(path) elif (os.path.isdir(path)): dirs.append(path) if (self.parse_files): self.process_files(files, message='Processing files from PHIL:') if (self.parse_dir): self.process_dir(dirs, message='Processing directories from PHIL:') if (not printed_something): print(' No PHIL parameters found', file=self.logger) print('', file=self.logger) # --------------------------------------------------------------------------- def process_dir(self, dir_list, message='Processing directories:'): ''' ''' print(message, file=self.logger) print('-' * self.text_width, file=self.logger) print('', file=self.logger) # --------------------------------------------------------------------------- def check_phil_for_paths(self, phil_scope): ''' Recursively check PHIL scope if there is a 'path' type. Returns the paths (empty list means no paths were found) ''' paths = list() if (phil_scope.is_definition): if (phil_scope.type.phil_type == 'path'): if phil_scope.style is not None and 'new_file' in phil_scope.style: pass else: paths.append(phil_scope.extract()) elif (phil_scope.is_scope): for phil_object in phil_scope.objects: paths.extend(self.check_phil_for_paths(phil_object)) return paths # --------------------------------------------------------------------------- def post_process(self): ''' Post processing of inputs after all arguments are parsed ''' working_phil_extract = self.working_phil.extract() # update default model with program pdb interpretation scope if (self.data_manager.supports('model') and (self.data_manager.get_default_model_name() is not None)): self.data_manager.update_pdb_interpretation_for_model( self.data_manager.get_default_model_name(), working_phil_extract) # --------------------------------------------------------------------------- def show_phil_summary(self): ''' Show final, modified PHIL parameters after all processing is complete Also, write phil scopes based on command-line flags ''' overwrite = (self.namespace.overwrite or \ self.working_phil.extract().output.overwrite) # check for any remaining unknown arguments if (len(self.namespace.unknown) > 0): error_message = 'The following arguments are not recognized:\n' for value in self.namespace.unknown: error_message += ' %s\n' % value raise Sorry(error_message) # get differences try: data_diff = self.data_manager.master_phil.fetch_diff( self.data_manager.export_phil_scope()) except RuntimeError as err: raise Sorry(err) try: phil_diff = self.master_phil.fetch_diff(self.working_phil) except RuntimeError as err: raise Sorry(err) data_is_different = (len(data_diff.as_str()) > 0) phil_is_different = (len(phil_diff.as_str()) > 0) is_different = data_is_different or phil_is_different # show final processed phil scope print('Final processed PHIL parameters:', file=self.logger) print('-' * self.text_width, file=self.logger) if (is_different): data_diff.show(prefix=' ', out=self.logger) phil_diff.show(prefix=' ', out=self.logger) else: print(' All parameters are set to their defaults', file=self.logger) print('', file=self.logger) # write scopes if requested if (self.namespace.write_data or self.namespace.write_modified or self.namespace.write_all): print('Writing program PHIL file(s):', file=self.logger) # write DataManager scope if (self.namespace.write_data): if (data_is_different): self.data_manager.write_phil_file( self.data_manager.export_phil_scope().as_str(), filename=self.data_filename, overwrite=overwrite) print(' Input file PHIL written to %s.' % self.data_filename, file=self.logger) else: print(' No input file PHIL to write', file=self.logger) # write differences if (self.namespace.write_modified): if (phil_is_different): self.data_manager.write_phil_file( phil_diff.as_str(), filename=self.modified_filename, overwrite=overwrite) print(' Modified PHIL parameters written to %s.' % self.modified_filename, file=self.logger) else: print(' No PHIL modifications to write', file=self.logger) # write all parameters (DataManager + Program) if (self.namespace.write_all): all_phil = self.data_manager.export_phil_scope().as_str() all_phil += self.working_phil.as_str(expert_level=3) self.data_manager.write_phil_file(all_phil, filename=self.all_filename, overwrite=overwrite) print(' All PHIL parameters written to %s.' % self.all_filename, file=self.logger) print('', file=self.logger) # --------------------------------------------------------------------------- def show_citations(self): # build list of program-specific citations program_citations = list() if (self.program_class.citations is not None): class_citations = citations.master_citation_phil.fetch( source=self.program_class.citations).extract() for citation in class_citations.citation: program_citations.append(citation) for article_id in self.program_class.known_article_ids: citation = citations.citations_db.get(article_id) if (citation is not None): program_citations.append(citation) else: raise Sorry('"%s" not found citations database' % article_id) # show program-specific citations and general citation for CCTBX if (len(program_citations) > 0): print('Citation(s) for %s:' % self.prog, file=self.logger) print('-' * self.text_width, file=self.logger) print('', file=self.logger) for citation in program_citations: citations.show_citation(citation, out=self.logger, format=self.namespace.citations) self.program_class.show_template_citation( text_width=self.text_width, logger=self.logger, citation_format=self.namespace.citations)
def test_miller_array_datatype(): data_dir = os.path.dirname(os.path.abspath(__file__)) data_mtz = os.path.join(data_dir, 'data', 'insulin_unmerged_cutted_from_ccp4.mtz') dm = DataManager(['miller_array', 'phil']) dm.process_miller_array_file(data_mtz) # test labels labels = ['M_ISYM', 'BATCH', 'I,SIGI,merged', 'IPR,SIGIPR,merged', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS'] for label in dm.get_miller_array_labels(): assert(label in labels) assert(len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels())) # test access by label label = dm.get_miller_array_labels()[3] new_label = dm.get_miller_arrays(labels=[label])[0].info().label_string() assert(label == new_label) # test custom PHIL dm.write_phil_file('test.phil', dm.export_phil_scope().as_str(), True) loaded_phil = iotbx.phil.parse(file_name='test.phil') new_dm = DataManager(['miller_array', 'phil']) new_dm.load_phil_scope(loaded_phil) assert(data_mtz == new_dm.get_default_miller_array_name()) for label in new_dm.get_miller_array_labels(): assert(label in labels) os.remove('test.phil') # test type assert(dm.get_miller_array_type() == 'x_ray') dm.set_miller_array_type(data_mtz, 'electron') assert(dm.get_miller_array_type() == 'electron') dm.write_phil_file('test_phil', dm.export_phil_scope().as_str(), True) loaded_phil = iotbx.phil.parse(file_name='test_phil') new_dm.load_phil_scope(loaded_phil) assert(new_dm.get_miller_array_type() == 'electron') new_dm = DataManager(['miller_array']) try: new_dm.set_default_miller_array_type('q') except Sorry: pass new_dm.set_default_miller_array_type('neutron') new_dm.process_miller_array_file(data_mtz) assert(new_dm.get_miller_array_type() == 'neutron') # test file server fs1 = dm.get_reflection_file_server() fs2 = dm.get_reflection_file_server([data_mtz, data_mtz]) assert(2*len(fs1.miller_arrays) == len(fs2.miller_arrays)) cs = crystal.symmetry( unit_cell=dm.get_miller_arrays()[0].crystal_symmetry().unit_cell(), space_group_symbol='P1') fs = dm.get_reflection_file_server(crystal_symmetry=cs) assert(fs.crystal_symmetry.is_similar_symmetry(cs)) assert(not fs.crystal_symmetry.is_similar_symmetry( dm.get_miller_arrays()[0].crystal_symmetry())) fs = dm.get_reflection_file_server(labels=['I,SIGI,merged']) assert(len(fs.get_miller_arrays(None)) == 1) miller_array = fs.get_amplitudes(None,None,True,None,None) assert(miller_array.info().label_string() == 'I,as_amplitude_array,merged') fs = dm.get_reflection_file_server(array_type='x_ray') assert(len(fs.get_miller_arrays(None)) == 0) fs = dm.get_reflection_file_server(array_type='electron') assert(len(fs.get_miller_arrays(None)) == 13) fs = dm.get_reflection_file_server(filenames=[data_mtz], labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']], array_type='neutron') assert(len(fs.get_miller_arrays(None)) == 0) dm.set_miller_array_type(data_mtz, 'x_ray') fs = dm.get_reflection_file_server(filenames=[data_mtz], labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']], array_type='x_ray') assert(len(fs.get_miller_arrays(data_mtz)) == 2)
def test_miller_array_datatype(): data_dir = os.path.dirname(os.path.abspath(__file__)) data_mtz = os.path.join(data_dir, 'data', 'insulin_unmerged_cutted_from_ccp4.mtz') dm = DataManager(['miller_array', 'phil']) dm.process_miller_array_file(data_mtz) # test labels labels = [ 'M_ISYM', 'BATCH', 'I,SIGI,merged', 'IPR,SIGIPR,merged', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS' ] for label in dm.get_miller_array_labels(): assert (label in labels) assert (len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels())) # test access by label label = dm.get_miller_array_labels()[3] new_label = dm.get_miller_arrays(labels=[label])[0].info().label_string() assert (label == new_label) # test custom PHIL dm.write_phil_file('test.phil', dm.export_phil_scope().as_str(), overwrite=True) loaded_phil = libtbx.phil.parse(file_name='test.phil') new_dm = DataManager(['miller_array', 'phil']) new_dm.load_phil_scope(loaded_phil) assert (data_mtz == new_dm.get_default_miller_array_name()) for label in new_dm.get_miller_array_labels(): assert (label in labels) os.remove('test.phil') # test file server fs1 = dm.get_reflection_file_server() fs2 = dm.get_reflection_file_server([data_mtz, data_mtz]) assert (2 * len(fs1.miller_arrays) == len(fs2.miller_arrays)) cs = crystal.symmetry( unit_cell=dm.get_miller_arrays()[0].crystal_symmetry().unit_cell(), space_group_symbol='P1') fs = dm.get_reflection_file_server(crystal_symmetry=cs) assert (fs.crystal_symmetry.is_similar_symmetry(cs)) assert (not fs.crystal_symmetry.is_similar_symmetry( dm.get_miller_arrays()[0].crystal_symmetry())) fs = dm.get_reflection_file_server(labels=['I,SIGI,merged']) assert (len(fs.get_miller_arrays(None)) == 1) miller_array = fs.get_amplitudes(None, None, True, None, None) assert ( miller_array.info().label_string() == 'I,as_amplitude_array,merged') master_phil_str = ''' include scope iotbx.data_manager.miller_array.miller_array_phil_str ''' master_phil = libtbx.phil.parse(master_phil_str, process_includes=True) master_extract = master_phil.extract() master_extract.data[0].file_name = data_mtz master_extract.data[0].labels = 'IPR,SIGIPR,merged' fs = get_reflection_file_server(dm, master_extract) assert (len(fs.get_miller_arrays(None)) == 1) master_extract.data[0].type = 'neutron' fs = get_reflection_file_server(dm, master_extract) assert (fs is None) fs = get_reflection_file_server(dm, master_extract, datatype='neutron') assert (len(fs.get_miller_arrays(None)) == 1)