def test_real_map_datatype():
data_dir = os.path.dirname(os.path.abspath(__file__))
data_ccp4 = os.path.join(data_dir, 'data', 'non_zero_origin_map.ccp4')
dm = DataManager(['real_map', 'phil'])
dm.process_real_map_file(data_ccp4)
assert dm.has_real_maps()
# test custom PHIL
dm.write_phil_file(dm.export_phil_scope().as_str(),
filename='test.phil',
overwrite=True)
loaded_phil = iotbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['real_map', 'phil'])
new_dm.load_phil_scope(loaded_phil)
assert data_ccp4 == new_dm.get_default_real_map_name()
os.remove('test.phil')
# test writing and reading file
mm = dm.get_real_map()
mm.shift_origin()
dm.write_real_map_file(mm, filename='test.ccp4', overwrite=True)
dm.process_real_map_file('test.ccp4')
new_mm = dm.get_real_map('test.ccp4')
assert not new_mm.is_similar(mm)
new_mm.shift_origin()
assert new_mm.is_similar(mm)
os.remove('test.ccp4')
def test_miller_array_datatype():
data_dir = os.path.dirname(os.path.abspath(__file__))
data_mtz = os.path.join(data_dir, 'data',
'insulin_unmerged_cutted_from_ccp4.mtz')
dm = DataManager(['miller_array', 'phil'])
dm.process_miller_array_file(data_mtz)
# test labels
labels = [
'M_ISYM', 'BATCH', 'I, SIGI', 'IPR, SIGIPR', 'FRACTIONCALC', 'XDET',
'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG', 'BGPKRATIOS'
]
for label in dm.get_miller_array_labels():
assert (label in labels)
assert (len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels()))
# test access by label
label = dm.get_miller_array_labels()[3]
new_label = ', '.join(
dm.get_miller_arrays(labels=[label])[0].info().labels)
assert (label == new_label)
# test custom PHIL
dm.write_phil_file('test.phil',
dm.export_phil_scope().as_str(),
overwrite=True)
loaded_phil = libtbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['miller_array', 'phil'])
new_dm.load_phil_scope(loaded_phil)
assert (data_mtz == new_dm.get_default_miller_array_name())
for label in new_dm.get_miller_array_labels():
assert (label in labels)
os.remove('test.phil')
def test_miller_array_datatype():
data_dir = os.path.dirname(os.path.abspath(__file__))
data_mtz = os.path.join(data_dir, 'data',
'insulin_unmerged_cutted_from_ccp4.mtz')
dm = DataManager(['miller_array', 'phil'])
dm.process_miller_array_file(data_mtz)
# test labels
labels = [
'M_ISYM', 'BATCH', 'I,SIGI,merged', 'IPR,SIGIPR,merged',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG',
'BGPKRATIOS'
]
for label in dm.get_miller_array_labels():
assert label in labels
assert len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels())
# test access by label
label = dm.get_miller_array_labels()[3]
new_label = dm.get_miller_arrays(labels=[label])[0].info().label_string()
assert label == new_label
# test custom PHIL
dm.write_phil_file(dm.export_phil_scope().as_str(),
filename='test.phil',
overwrite=True)
loaded_phil = iotbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['miller_array', 'phil'])
new_dm.load_phil_scope(loaded_phil)
assert data_mtz == new_dm.get_default_miller_array_name()
for label in new_dm.get_miller_array_labels():
assert label in labels
os.remove('test.phil')
# test type
assert dm.get_miller_array_type() == 'x_ray'
label = labels[3]
dm.set_miller_array_type(data_mtz, label, 'electron')
assert dm.get_miller_array_type(label=label) == 'electron'
dm.write_phil_file(dm.export_phil_scope().as_str(),
filename='test_phil',
overwrite=True)
loaded_phil = iotbx.phil.parse(file_name='test_phil')
new_dm.load_phil_scope(loaded_phil)
assert new_dm.get_miller_array_type(label=label) == 'electron'
new_dm = DataManager(['miller_array'])
try:
new_dm.set_default_miller_array_type('q')
except Sorry:
pass
new_dm.set_default_miller_array_type('neutron')
new_dm.process_miller_array_file(data_mtz)
assert new_dm.get_miller_array_type(label=label) == 'neutron'
os.remove('test_phil')
# test writing file
arrays = dm.get_miller_arrays()
dataset = arrays[2].as_mtz_dataset(column_root_label='label1')
dataset.add_miller_array(miller_array=arrays[3],
column_root_label='label2')
mtz_object = dataset.mtz_object()
dm.write_miller_array_file(mtz_object, filename='test.mtz', overwrite=True)
dm.process_miller_array_file('test.mtz')
new_labels = dm.get_miller_array_labels('test.mtz')
assert 'label1,SIGlabel1' in new_labels
assert 'label2,SIGlabel2' in new_labels
os.remove('test.mtz')
# test file server
fs1 = dm.get_reflection_file_server()
fs2 = dm.get_reflection_file_server([data_mtz, data_mtz])
assert 2 * len(fs1.miller_arrays) == len(fs2.miller_arrays)
cs = crystal.symmetry(
unit_cell=dm.get_miller_arrays()[0].crystal_symmetry().unit_cell(),
space_group_symbol='P1')
fs = dm.get_reflection_file_server(crystal_symmetry=cs)
assert fs.crystal_symmetry.is_similar_symmetry(cs)
assert not fs.crystal_symmetry.is_similar_symmetry(
dm.get_miller_arrays()[0].crystal_symmetry())
fs = dm.get_reflection_file_server(labels=['I,SIGI,merged'])
assert len(fs.get_miller_arrays(None)) == 1
miller_array = fs.get_amplitudes(None, None, True, None, None)
assert miller_array.info().label_string() == 'I,as_amplitude_array,merged'
for label in dm.get_miller_array_labels():
dm.set_miller_array_type(label=label, array_type='electron')
fs = dm.get_reflection_file_server(array_type='x_ray')
assert len(fs.get_miller_arrays(None)) == 0
fs = dm.get_reflection_file_server(array_type='electron')
assert len(fs.get_miller_arrays(None)) == 13
fs = dm.get_reflection_file_server(
filenames=[data_mtz],
labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']],
array_type='neutron')
assert len(fs.get_miller_arrays(None)) == 0
for label in ['I,SIGI,merged', 'IPR,SIGIPR,merged']:
dm.set_miller_array_type(label=label, array_type='x_ray')
fs = dm.get_reflection_file_server(
filenames=[data_mtz],
labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']],
array_type='x_ray')
assert len(fs.get_miller_arrays(data_mtz)) == 2
fs = dm.get_reflection_file_server(filenames=[data_mtz],
array_type='x_ray')
assert len(fs.get_miller_arrays(data_mtz)) == 2
fs = dm.get_reflection_file_server(filenames=[data_mtz],
array_type='electron')
assert len(fs.get_miller_arrays(data_mtz)) == 11
# test subset of labels
label_subset = labels[3:8]
dm = DataManager(['miller_array', 'phil'])
dm.process_miller_array_file(data_mtz)
dm._miller_array_labels[data_mtz] = label_subset
dm.set_miller_array_type(label=label_subset[2], array_type='electron')
assert dm.get_miller_array_type(label=label_subset[2]) == 'electron'
dm.write_phil_file(dm.export_phil_scope().as_str(),
filename='test.phil',
overwrite=True)
loaded_phil = iotbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['miller_array', 'phil'])
new_dm.load_phil_scope(loaded_phil)
assert new_dm.get_miller_array_type(label=label_subset[2]) == 'electron'
fs = new_dm.get_reflection_file_server(array_type='x_ray')
assert len(fs.get_miller_arrays(None)) == 4
fs = new_dm.get_reflection_file_server(array_type='electron')
assert len(fs.get_miller_arrays(None)) == 1
os.remove('test.phil')
label_subset = list()
dm = DataManager(['miller_array', 'phil'])
dm.process_miller_array_file(data_mtz)
dm._miller_array_labels[data_mtz] = label_subset
dm.write_phil_file(dm.export_phil_scope().as_str(),
filename='test.phil',
overwrite=True)
loaded_phil = iotbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['miller_array', 'phil'])
new_dm.load_phil_scope(loaded_phil)
fs = new_dm.get_reflection_file_server(array_type='x_ray')
assert len(fs.get_miller_arrays(None)) == 13
fs = new_dm.get_reflection_file_server(array_type='electron')
assert len(fs.get_miller_arrays(None)) == 0
os.remove('test.phil')
def test_data_manager():
a = DataManager(['model'])
a.add_model('a', 'b')
a.add_model('c', 'd')
assert a.get_model() == 'b'
assert a.get_model('a') == 'b'
assert a.get_model('c') == 'd'
assert a.get_model_names() == ['a', 'c']
assert a.has_models()
assert a.has_models(exact_count=True, expected_n=2)
assert not a.has_models(expected_n=3, raise_sorry=False)
# exporting phil
working_phil = a.export_phil_scope()
assert len(working_phil.extract().data_manager.model) == 2
# data tracking
try:
a.has_models(expected_n=3, raise_sorry=True)
except Sorry:
pass
try:
a.has_models(exact_count=True, raise_sorry=True)
except Sorry:
pass
a.set_default_model('c')
assert a.get_model() == 'd'
assert a.get_model_names() == ['a', 'c'
] or a.get_model_names() == ['c', 'a']
a.remove_model('c')
try:
a.get_model()
except Sorry:
pass
try:
a.get_model('missing')
except Sorry:
pass
try:
a.set_default_model('missing')
except Sorry:
pass
a = DataManager(datatypes=['sequence', 'phil'])
assert a.get_sequence_names() == []
assert not hasattr(a, 'get_model')
# phil functions
test_phil_str = '''
data_manager {
phil_files = data_manager_test.eff
}
'''
with open('data_manager_test.eff', 'w') as f:
f.write(test_phil_str)
# loading file with get function
assert len(a.get_phil_names()) == 0
p = a.get_phil('data_manager_test.eff')
assert type(p) == libtbx.phil.scope
assert 'data_manager_test.eff' in a.get_phil_names()
# loading file with phil
a = DataManager(datatypes=['phil'])
test_phil = iotbx.phil.parse(test_phil_str)
a.load_phil_scope(test_phil)
assert 'data_manager_test.eff' in a.get_phil_names()
assert a.get_default_phil_name() == 'data_manager_test.eff'
os.remove('data_manager_test.eff')
# writing
a = DataManager(datatypes=['model', 'phil', 'sequence'])
a.add_model('a', 'b')
a.add_phil('c', 'd')
a.add_sequence('e', 'f')
a.write_model_file(a.get_model(), filename='a.dat', overwrite=True)
a.write_phil_file(a.get_phil(), filename='c.dat', overwrite=True)
a.write_sequence_file(a.get_sequence(), filename='e.dat', overwrite=True)
with open('a.dat', 'r') as f:
lines = f.readlines()
assert lines[0] == 'b'
os.remove('a.dat')
os.remove('c.dat')
os.remove('e.dat')
def test_model_datatype():
import mmtbx.monomer_library.server
try:
mon_lib_srv = mmtbx.monomer_library.server.server()
except mmtbx.monomer_library.server.MonomerLibraryServerError:
print(
"Can not initialize monomer_library, skipping test_model_datatype."
)
return
# 1yjp
model_str = '''
CRYST1 21.937 4.866 23.477 90.00 107.08 90.00 P 1 21 1 2
ORIGX1 1.000000 0.000000 0.000000 0.00000
ORIGX2 0.000000 1.000000 0.000000 0.00000
ORIGX3 0.000000 0.000000 1.000000 0.00000
SCALE1 0.045585 0.000000 0.014006 0.00000
SCALE2 0.000000 0.205508 0.000000 0.00000
SCALE3 0.000000 0.000000 0.044560 0.00000
ATOM 1 N GLY A 1 -9.009 4.612 6.102 1.00 16.77 N
ATOM 2 CA GLY A 1 -9.052 4.207 4.651 1.00 16.57 C
ATOM 3 C GLY A 1 -8.015 3.140 4.419 1.00 16.16 C
ATOM 4 O GLY A 1 -7.523 2.521 5.381 1.00 16.78 O
ATOM 5 N ASN A 2 -7.656 2.923 3.155 1.00 15.02 N
ATOM 6 CA ASN A 2 -6.522 2.038 2.831 1.00 14.10 C
ATOM 7 C ASN A 2 -5.241 2.537 3.427 1.00 13.13 C
ATOM 8 O ASN A 2 -4.978 3.742 3.426 1.00 11.91 O
ATOM 9 CB ASN A 2 -6.346 1.881 1.341 1.00 15.38 C
ATOM 10 CG ASN A 2 -7.584 1.342 0.692 1.00 14.08 C
ATOM 11 OD1 ASN A 2 -8.025 0.227 1.016 1.00 17.46 O
ATOM 12 ND2 ASN A 2 -8.204 2.155 -0.169 1.00 11.72 N
ATOM 13 N ASN A 3 -4.438 1.590 3.905 1.00 12.26 N
ATOM 14 CA ASN A 3 -3.193 1.904 4.589 1.00 11.74 C
ATOM 15 C ASN A 3 -1.955 1.332 3.895 1.00 11.10 C
ATOM 16 O ASN A 3 -1.872 0.119 3.648 1.00 10.42 O
ATOM 17 CB ASN A 3 -3.259 1.378 6.042 1.00 12.15 C
ATOM 18 CG ASN A 3 -2.006 1.739 6.861 1.00 12.82 C
ATOM 19 OD1 ASN A 3 -1.702 2.925 7.072 1.00 15.05 O
ATOM 20 ND2 ASN A 3 -1.271 0.715 7.306 1.00 13.48 N
ATOM 21 N GLN A 4 -1.005 2.228 3.598 1.00 10.29 N
ATOM 22 CA GLN A 4 0.384 1.888 3.199 1.00 10.53 C
ATOM 23 C GLN A 4 1.435 2.606 4.088 1.00 10.24 C
ATOM 24 O GLN A 4 1.547 3.843 4.115 1.00 8.86 O
ATOM 25 CB GLN A 4 0.656 2.148 1.711 1.00 9.80 C
ATOM 26 CG GLN A 4 1.944 1.458 1.213 1.00 10.25 C
ATOM 27 CD GLN A 4 2.504 2.044 -0.089 1.00 12.43 C
ATOM 28 OE1 GLN A 4 2.744 3.268 -0.190 1.00 14.62 O
ATOM 29 NE2 GLN A 4 2.750 1.161 -1.091 1.00 9.05 N
ATOM 30 N GLN A 5 2.154 1.821 4.871 1.00 10.38 N
ATOM 31 CA GLN A 5 3.270 2.361 5.640 1.00 11.39 C
ATOM 32 C GLN A 5 4.594 1.768 5.172 1.00 11.52 C
ATOM 33 O GLN A 5 4.768 0.546 5.054 1.00 12.05 O
ATOM 34 CB GLN A 5 3.056 2.183 7.147 1.00 11.96 C
ATOM 35 CG GLN A 5 1.829 2.950 7.647 1.00 10.81 C
ATOM 36 CD GLN A 5 1.344 2.414 8.954 1.00 13.10 C
ATOM 37 OE1 GLN A 5 0.774 1.325 9.002 1.00 10.65 O
ATOM 38 NE2 GLN A 5 1.549 3.187 10.039 1.00 12.30 N
ATOM 39 N ASN A 6 5.514 2.664 4.856 1.00 11.99 N
ATOM 40 CA ASN A 6 6.831 2.310 4.318 1.00 12.30 C
ATOM 41 C ASN A 6 7.854 2.761 5.324 1.00 13.40 C
ATOM 42 O ASN A 6 8.219 3.943 5.374 1.00 13.92 O
ATOM 43 CB ASN A 6 7.065 3.016 2.993 1.00 12.13 C
ATOM 44 CG ASN A 6 5.961 2.735 2.003 1.00 12.77 C
ATOM 45 OD1 ASN A 6 5.798 1.604 1.551 1.00 14.27 O
ATOM 46 ND2 ASN A 6 5.195 3.747 1.679 1.00 10.07 N
ATOM 47 N TYR A 7 8.292 1.817 6.147 1.00 14.70 N
ATOM 48 CA TYR A 7 9.159 2.144 7.299 1.00 15.18 C
ATOM 49 C TYR A 7 10.603 2.331 6.885 1.00 15.91 C
ATOM 50 O TYR A 7 11.041 1.811 5.855 1.00 15.76 O
ATOM 51 CB TYR A 7 9.061 1.065 8.369 1.00 15.35 C
ATOM 52 CG TYR A 7 7.665 0.929 8.902 1.00 14.45 C
ATOM 53 CD1 TYR A 7 6.771 0.021 8.327 1.00 15.68 C
ATOM 54 CD2 TYR A 7 7.210 1.756 9.920 1.00 14.80 C
ATOM 55 CE1 TYR A 7 5.480 -0.094 8.796 1.00 13.46 C
ATOM 56 CE2 TYR A 7 5.904 1.649 10.416 1.00 14.33 C
ATOM 57 CZ TYR A 7 5.047 0.729 9.831 1.00 15.09 C
ATOM 58 OH TYR A 7 3.766 0.589 10.291 1.00 14.39 O
ATOM 59 OXT TYR A 7 11.358 2.999 7.612 1.00 17.49 O
TER 60 TYR A 7
HETATM 61 O HOH A 8 -6.471 5.227 7.124 1.00 22.62 O
HETATM 62 O HOH A 9 10.431 1.858 3.216 1.00 19.71 O
HETATM 63 O HOH A 10 -11.286 1.756 -1.468 1.00 17.08 O
HETATM 64 O HOH A 11 11.808 4.179 9.970 1.00 23.99 O
HETATM 65 O HOH A 12 13.605 1.327 9.198 1.00 26.17 O
HETATM 66 O HOH A 13 -2.749 3.429 10.024 1.00 39.15 O
HETATM 67 O HOH A 14 -1.500 0.682 10.967 1.00 43.49 O
MASTER 238 0 0 0 0 0 0 6 66 1 0 1
END
'''
# test reading/writing PDB
test_filename = 'test_model.pdb'
test_output_filename = 'test_model_output.pdb'
test_eff = 'model.eff'
dm = DataManager(['model'])
dm.process_model_str(test_filename, model_str)
dm.write_model_file(model_str,
filename=test_output_filename,
overwrite=True)
m = dm.get_model(test_output_filename)
assert test_output_filename in dm.get_model_names()
dm.write_model_file(m, overwrite=True)
pdb_filename = 'cctbx_program.pdb'
assert os.path.exists(pdb_filename)
dm.process_model_file(pdb_filename)
assert not dm.get_model(pdb_filename).input_model_format_cif()
dm.write_model_file(m, test_filename, overwrite=True)
# test reading PDB writing CIF
test_filename = 'test_model.pdb'
test_output_filename = 'test_model.cif'
dm = DataManager(['model'])
dm.process_model_str(test_filename, model_str)
m = dm.get_model(test_filename)
dm.write_model_file(m,
filename=test_output_filename,
format='cif',
overwrite=True)
m = dm.get_model(test_output_filename)
assert test_output_filename in dm.get_model_names()
dm.write_model_file(m, overwrite=True)
cif_filename = 'cctbx_program.cif'
assert os.path.exists(cif_filename)
dm.process_model_file(cif_filename)
assert dm.get_model(cif_filename).input_model_format_cif()
# test type
assert dm.get_model_type() == 'x_ray'
dm.set_model_type(test_filename, 'neutron')
assert dm.get_model_type() == 'neutron'
phil_scope = dm.export_phil_scope()
extract = phil_scope.extract()
assert extract.data_manager.model[0].type == 'neutron'
with open(test_eff, 'w') as f:
f.write(phil_scope.as_str())
new_phil_scope = iotbx.phil.parse(file_name=test_eff)
new_dm = DataManager(['model'])
new_dm.load_phil_scope(new_phil_scope)
assert new_dm.get_model_type(test_filename) == 'neutron'
new_dm = DataManager(['model'])
try:
new_dm.set_default_model_type('nonsense')
except Sorry:
pass
new_dm.set_default_model_type('electron')
new_dm.process_model_file(test_filename)
assert new_dm.get_model_type() == 'electron'
assert len(new_dm.get_model_names()) == 1
assert len(new_dm.get_model_names(model_type='electron')) == 1
assert len(new_dm.get_model_names(model_type='neutron')) == 0
os.remove(test_eff)
os.remove(test_filename)
# test reading/writing CIF
test_filename = 'test_model_datatype.cif'
dm.write_model_file(dm.get_model().model_as_mmcif(),
filename=test_filename,
overwrite=True)
dm.process_model_file(test_filename)
os.remove(test_filename)
assert test_filename in dm.get_model_names()
m = dm.get_model(test_filename)
dm.write_model_file(m, overwrite=True)
cif_filename = 'cctbx_program.cif'
assert os.path.exists(cif_filename)
dm.process_model_file(cif_filename)
assert dm.get_model(cif_filename).input_model_format_cif()
os.remove(pdb_filename)
os.remove(cif_filename)
# test pdb_interpretation
extract = mmtbx.model.manager.get_default_pdb_interpretation_params()
extract.pdb_interpretation.use_neutron_distances = True
dm.update_pdb_interpretation_for_model(test_filename, extract)
assert dm.get_model(test_filename).restraints_manager is None
class CCTBXParser(ParserBase):
def __init__(self,
program_class,
custom_process_arguments=None,
logger=None,
*args,
**kwargs):
'''
'''
# program name
self.prog = os.getenv('LIBTBX_DISPATCHER_NAME')
if (self.prog is None):
self.prog = sys.argv[0]
self.prefix = self.prog.split('.')[-1]
# PHIL filenames
self.data_filename = self.prefix + '_data.eff'
self.modified_filename = self.prefix + '_modified.eff'
self.all_filename = self.prefix + '_all.eff'
# terminal width
self.text_width = 79
# print header
border = '-' * self.text_width
description = border + program_class.description + border
epilog = border + program_class.epilog
super(CCTBXParser, self).__init__(
prog=self.prog,
description=description,
epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter,
*args,
**kwargs)
# default values
self.program_class = program_class
self.custom_process_arguments = custom_process_arguments
self.logger = logger
if (self.logger is None):
self.logger = logging.getLogger('main')
self.data_manager = DataManager(datatypes=program_class.datatypes,
logger=self.logger)
# add PHIL converters if available
if (len(program_class.phil_converters) > 0):
iotbx.phil.default_converter_registry = \
libtbx.phil.extended_converter_registry(
additional_converters=program_class.phil_converters,
base_registry=iotbx.phil.default_converter_registry)
# set up master and working PHIL scopes
self.master_phil = iotbx.phil.parse(program_class.master_phil_str,
process_includes=True)
required_output_phil = iotbx.phil.parse(
ProgramTemplate.output_phil_str)
self.master_phil.adopt_scope(required_output_phil)
self.working_phil = None
self.add_default_options()
# ---------------------------------------------------------------------------
def add_default_options(self):
'''
'''
# --show-defaults by itself is set to 0
# --show-defaults=n sets it to n and it can only be {0, 1, 2, 3}
self.add_argument(
'--show-defaults',
'--show_defaults',
nargs='?',
const=0,
type=int,
choices=list(range(0, 4)),
help='show default parameters with expert level (default=0)')
# --attributes-level by itself is set to 1
# --attributes-level=n sets it to n and it can only be {1, 2, 3}
self.add_argument('--attributes-level',
'--attributes_level',
nargs='?',
const=1,
type=int,
choices=list(range(0, 4)),
help='show parameters with attributes (default=0)')
# --write-data
# switch for writing only DataManager PHIL parameters
self.add_argument(
'--write-data', '--write_data', action='store_true',
help='write DataManager PHIL parameters to file (%s)' % \
self.data_filename
)
# --write-modified
# switch for writing only modified PHIL parameters
self.add_argument(
'--write-modified', '--write_modified', action='store_true',
help='write modifed PHIL parameters to file (%s)' % \
self.modified_filename
)
# --write-all
# switch for writing all PHIL parameters
self.add_argument(
'--write-all',
'--write_all',
action='store_true',
help=
'write all (modified + default + data) PHIL parameters to file (%s)'
% self.all_filename)
# --overwrite
# switch for overwriting files, takes precedence over PHIL definition
self.add_argument(
'--overwrite',
action='store_true',
default=False,
help=
'overwrite files, this overrides the output.overwrite PHIL parameter'
)
# --citations will use the default format
# --citations=<format> will use the specified format
self.add_argument(
'--citations',
nargs='?',
const='default',
type=str,
choices=['default', 'cell', 'iucr'],
help='show citation(s) for program in different formats')
# ---------------------------------------------------------------------------
def parse_args(self, args):
'''
'''
# default behavior with no arguments
if (len(args) == 0):
self.print_help()
self.exit()
# parse arguments
self.namespace = super(CCTBXParser, self).parse_args(args)
# process command-line options
if (self.namespace.attributes_level is not None):
if (self.namespace.show_defaults is None):
self.error(
'--attributes-level requires --show-defaults to be set')
if (self.namespace.show_defaults is not None):
self.master_phil.show(
expert_level=self.namespace.show_defaults,
attributes_level=self.namespace.attributes_level,
out=self.logger)
self.exit()
if (self.namespace.citations is not None):
self.show_citations()
self.exit()
# start program header
print('Starting %s' % self.prog, file=self.logger)
print('on %s by %s' % (time.asctime(), getpass.getuser()),
file=self.logger)
print('=' * self.text_width, file=self.logger)
print('', file=self.logger)
# process files
if (self.parse_files):
self.process_files(self.namespace.files)
# process phil and phil files
if (self.parse_phil):
self.process_phil(self.namespace.phil)
# process directories
if (self.parse_dir):
self.process_dir(self.namespace.dir)
# custom processing of arguments (if available)
# the function for custom processing of arguments should take a CCTBXParser
# object as its argument. The function should modify the
# CCTBXParser.namespace.unknown, CCTBXParser.data_manager,
# CCTBXParser.working_phil, or other CCTBXParser members.
# At the end of the function, CCTBXParser.working_phil should have the final
# libtbx.phil.scope object (not libtbx.phil.scope_extract) for the program
# A libtbx.phil.scope_extract object can be converted into a
# libtbx.phil.scope object by
# CCTBXParser.master_phil.format(python_object=<scope_extract>)
if (self.custom_process_arguments is not None):
self.custom_process_arguments(self)
assert (isinstance(self.working_phil, libtbx.phil.scope))
# post processing after all arguments are parsed
self.post_process()
# show final PHIL parameters
self.show_phil_summary()
return self.namespace
# ---------------------------------------------------------------------------
def process_files(self, file_list, message='Processing files:'):
'''
Second pass to process files. The first pass already checked that these
files exist. There may be conditions where the file is deleted in the time
between the first pass and calling this function.
Use iotbx.file_reader.any_file to process files.
Will need updating to work with mmtbx.model.manager class more efficiently
'''
print(message, file=self.logger)
print('-' * self.text_width, file=self.logger)
print('', file=self.logger)
printed_something = False
unused_files = list()
for filename in file_list:
a = any_file(filename)
process_function = 'process_%s_file' % data_manager_type.get(
a.file_type)
if (hasattr(self.data_manager, process_function)):
getattr(self.data_manager, process_function)(filename)
print(' Found %s, %s' %
(data_manager_type[a.file_type], filename),
file=self.logger)
printed_something = True
else:
unused_files.append(filename)
# show unrecognized files
if (len(unused_files) > 0):
if (printed_something):
print('', file=self.logger)
print(' Files not used by program:', file=self.logger)
print(' --------------------------', file=self.logger)
for filename in unused_files:
print(' %s' % filename, file=self.logger)
printed_something = True
# process PHIL files for DataManager scope in order from command-line
# files are appended and the default is not overridden
# files from the command-line take precedence
phil_names = self.data_manager.get_phil_names()
for name in phil_names:
phil = self.data_manager.get_phil(name)
if (hasattr(phil.extract(), 'data_manager')):
self.data_manager.load_phil_scope(phil)
if (not printed_something):
print(' No files found', file=self.logger)
print('', file=self.logger)
# ---------------------------------------------------------------------------
def process_phil(self, phil_list):
''''
Process PHIL arguments
Also checks PHIL arguments (command line and files) for parameters that
specify files (.type = path)
'''
print('Processing PHIL parameters:', file=self.logger)
print('-' * self.text_width, file=self.logger)
print('', file=self.logger)
printed_something = False
data_sources = list()
sources = list()
unused_phil = list()
# PHIL files are processed in order from command-line
if (self.data_manager.has_phils()):
phil_names = self.data_manager.get_phil_names()
phil = list()
print(' Adding PHIL files:', file=self.logger)
print(' ------------------', file=self.logger)
for name in phil_names:
# remove DataManager scope since input files are already loaded
phil_scope = self.data_manager.get_phil(name)
for phil_object in phil_scope.objects:
if (phil_object.name == 'data_manager'):
phil_scope.objects.remove(phil_object)
phil.append(phil_scope)
print(' %s' % name, file=self.logger)
data_sources.extend(phil)
print('', file=self.logger)
printed_something = True
# command-line PHIL arguments override any previous settings and are
# processed in given order
def custom_processor(arg):
unused_phil.append(arg)
return True
if (len(phil_list) > 0):
interpreter = self.master_phil.command_line_argument_interpreter(
home_scope='')
print(' Adding command-line PHIL:', file=self.logger)
print(' -------------------------', file=self.logger)
for phil in phil_list:
print(' %s' % phil, file=self.logger)
print('', file=self.logger)
printed_something = True
working = interpreter.process_args(
phil_list, custom_processor=custom_processor)
if (len(working) > 0):
sources.extend(working)
if (self.namespace.overwrite): # override overwrite if True
sources.append(iotbx.phil.parse('output.overwrite=True'))
if ((len(data_sources) + len(sources)) > 0):
self.working_phil, more_unused_phil = self.master_phil.fetch(
sources=data_sources + sources, track_unused_definitions=True)
unused_phil.extend(more_unused_phil)
else:
self.working_phil = self.master_phil.fetch()
# show unrecognized parameters and abort
if (len(unused_phil) > 0):
print(' Unrecognized PHIL parameters:', file=self.logger)
print(' -----------------------------', file=self.logger)
for phil in unused_phil:
print(' %s' % phil, file=self.logger)
print('', file=self.logger)
error_message = 'Some PHIL parameters are not recognized by %s.\n' % \
self.prog
error_message += wordwrap(
'Please run this program with the --show-defaults option to see what parameters are available.',
max_chars=self.text_width) + '\n'
error_message += wordwrap(
'PHIL parameters in files should be fully specified (e.g. "output.overwrite" instead of just "overwrite")',
max_chars=self.text_width) + '\n'
raise Sorry(error_message)
# process input phil for file/directory defintions and add to DataManager
# Note: if a PHIL file is input as a PHIL parameter, the contents of the
# file will NOT be parsed and validated. The PHIL file should be provided
# as a command-line argument. This is mostly for finding data files
# defined by PHIL parameters that should be added to the DataManager
diff_phil = self.master_phil.fetch_diff(self.working_phil)
paths = self.check_phil_for_paths(diff_phil)
if (len(paths) > 0):
files = list()
dirs = list()
for path in paths:
if (path is not None):
if (os.path.isfile(path)):
files.append(path)
elif (os.path.isdir(path)):
dirs.append(path)
if (self.parse_files):
self.process_files(files,
message='Processing files from PHIL:')
if (self.parse_dir):
self.process_dir(dirs,
message='Processing directories from PHIL:')
if (not printed_something):
print(' No PHIL parameters found', file=self.logger)
print('', file=self.logger)
# ---------------------------------------------------------------------------
def process_dir(self, dir_list, message='Processing directories:'):
'''
'''
print(message, file=self.logger)
print('-' * self.text_width, file=self.logger)
print('', file=self.logger)
# ---------------------------------------------------------------------------
def check_phil_for_paths(self, phil_scope):
'''
Recursively check PHIL scope if there is a 'path' type.
Returns the paths (empty list means no paths were found)
'''
paths = list()
if (phil_scope.is_definition):
if (phil_scope.type.phil_type == 'path'):
if phil_scope.style is not None and 'new_file' in phil_scope.style:
pass
else:
paths.append(phil_scope.extract())
elif (phil_scope.is_scope):
for phil_object in phil_scope.objects:
paths.extend(self.check_phil_for_paths(phil_object))
return paths
# ---------------------------------------------------------------------------
def post_process(self):
'''
Post processing of inputs after all arguments are parsed
'''
working_phil_extract = self.working_phil.extract()
# update default model with program pdb interpretation scope
if (self.data_manager.supports('model')
and (self.data_manager.get_default_model_name() is not None)):
self.data_manager.update_pdb_interpretation_for_model(
self.data_manager.get_default_model_name(),
working_phil_extract)
# ---------------------------------------------------------------------------
def show_phil_summary(self):
'''
Show final, modified PHIL parameters after all processing is complete
Also, write phil scopes based on command-line flags
'''
overwrite = (self.namespace.overwrite or \
self.working_phil.extract().output.overwrite)
# check for any remaining unknown arguments
if (len(self.namespace.unknown) > 0):
error_message = 'The following arguments are not recognized:\n'
for value in self.namespace.unknown:
error_message += ' %s\n' % value
raise Sorry(error_message)
# get differences
try:
data_diff = self.data_manager.master_phil.fetch_diff(
self.data_manager.export_phil_scope())
except RuntimeError as err:
raise Sorry(err)
try:
phil_diff = self.master_phil.fetch_diff(self.working_phil)
except RuntimeError as err:
raise Sorry(err)
data_is_different = (len(data_diff.as_str()) > 0)
phil_is_different = (len(phil_diff.as_str()) > 0)
is_different = data_is_different or phil_is_different
# show final processed phil scope
print('Final processed PHIL parameters:', file=self.logger)
print('-' * self.text_width, file=self.logger)
if (is_different):
data_diff.show(prefix=' ', out=self.logger)
phil_diff.show(prefix=' ', out=self.logger)
else:
print(' All parameters are set to their defaults',
file=self.logger)
print('', file=self.logger)
# write scopes if requested
if (self.namespace.write_data or self.namespace.write_modified
or self.namespace.write_all):
print('Writing program PHIL file(s):', file=self.logger)
# write DataManager scope
if (self.namespace.write_data):
if (data_is_different):
self.data_manager.write_phil_file(
self.data_manager.export_phil_scope().as_str(),
filename=self.data_filename,
overwrite=overwrite)
print(' Input file PHIL written to %s.' % self.data_filename,
file=self.logger)
else:
print(' No input file PHIL to write', file=self.logger)
# write differences
if (self.namespace.write_modified):
if (phil_is_different):
self.data_manager.write_phil_file(
phil_diff.as_str(),
filename=self.modified_filename,
overwrite=overwrite)
print(' Modified PHIL parameters written to %s.' %
self.modified_filename,
file=self.logger)
else:
print(' No PHIL modifications to write', file=self.logger)
# write all parameters (DataManager + Program)
if (self.namespace.write_all):
all_phil = self.data_manager.export_phil_scope().as_str()
all_phil += self.working_phil.as_str(expert_level=3)
self.data_manager.write_phil_file(all_phil,
filename=self.all_filename,
overwrite=overwrite)
print(' All PHIL parameters written to %s.' % self.all_filename,
file=self.logger)
print('', file=self.logger)
# ---------------------------------------------------------------------------
def show_citations(self):
# build list of program-specific citations
program_citations = list()
if (self.program_class.citations is not None):
class_citations = citations.master_citation_phil.fetch(
source=self.program_class.citations).extract()
for citation in class_citations.citation:
program_citations.append(citation)
for article_id in self.program_class.known_article_ids:
citation = citations.citations_db.get(article_id)
if (citation is not None):
program_citations.append(citation)
else:
raise Sorry('"%s" not found citations database' % article_id)
# show program-specific citations and general citation for CCTBX
if (len(program_citations) > 0):
print('Citation(s) for %s:' % self.prog, file=self.logger)
print('-' * self.text_width, file=self.logger)
print('', file=self.logger)
for citation in program_citations:
citations.show_citation(citation,
out=self.logger,
format=self.namespace.citations)
self.program_class.show_template_citation(
text_width=self.text_width,
logger=self.logger,
citation_format=self.namespace.citations)
def test_miller_array_datatype():
data_dir = os.path.dirname(os.path.abspath(__file__))
data_mtz = os.path.join(data_dir, 'data',
'insulin_unmerged_cutted_from_ccp4.mtz')
dm = DataManager(['miller_array', 'phil'])
dm.process_miller_array_file(data_mtz)
# test labels
labels = ['M_ISYM', 'BATCH', 'I,SIGI,merged', 'IPR,SIGIPR,merged',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART',
'FLAG', 'BGPKRATIOS']
for label in dm.get_miller_array_labels():
assert(label in labels)
assert(len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels()))
# test access by label
label = dm.get_miller_array_labels()[3]
new_label = dm.get_miller_arrays(labels=[label])[0].info().label_string()
assert(label == new_label)
# test custom PHIL
dm.write_phil_file('test.phil', dm.export_phil_scope().as_str(), True)
loaded_phil = iotbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['miller_array', 'phil'])
new_dm.load_phil_scope(loaded_phil)
assert(data_mtz == new_dm.get_default_miller_array_name())
for label in new_dm.get_miller_array_labels():
assert(label in labels)
os.remove('test.phil')
# test type
assert(dm.get_miller_array_type() == 'x_ray')
dm.set_miller_array_type(data_mtz, 'electron')
assert(dm.get_miller_array_type() == 'electron')
dm.write_phil_file('test_phil', dm.export_phil_scope().as_str(), True)
loaded_phil = iotbx.phil.parse(file_name='test_phil')
new_dm.load_phil_scope(loaded_phil)
assert(new_dm.get_miller_array_type() == 'electron')
new_dm = DataManager(['miller_array'])
try:
new_dm.set_default_miller_array_type('q')
except Sorry:
pass
new_dm.set_default_miller_array_type('neutron')
new_dm.process_miller_array_file(data_mtz)
assert(new_dm.get_miller_array_type() == 'neutron')
# test file server
fs1 = dm.get_reflection_file_server()
fs2 = dm.get_reflection_file_server([data_mtz, data_mtz])
assert(2*len(fs1.miller_arrays) == len(fs2.miller_arrays))
cs = crystal.symmetry(
unit_cell=dm.get_miller_arrays()[0].crystal_symmetry().unit_cell(),
space_group_symbol='P1')
fs = dm.get_reflection_file_server(crystal_symmetry=cs)
assert(fs.crystal_symmetry.is_similar_symmetry(cs))
assert(not fs.crystal_symmetry.is_similar_symmetry(
dm.get_miller_arrays()[0].crystal_symmetry()))
fs = dm.get_reflection_file_server(labels=['I,SIGI,merged'])
assert(len(fs.get_miller_arrays(None)) == 1)
miller_array = fs.get_amplitudes(None,None,True,None,None)
assert(miller_array.info().label_string() == 'I,as_amplitude_array,merged')
fs = dm.get_reflection_file_server(array_type='x_ray')
assert(len(fs.get_miller_arrays(None)) == 0)
fs = dm.get_reflection_file_server(array_type='electron')
assert(len(fs.get_miller_arrays(None)) == 13)
fs = dm.get_reflection_file_server(filenames=[data_mtz],
labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']], array_type='neutron')
assert(len(fs.get_miller_arrays(None)) == 0)
dm.set_miller_array_type(data_mtz, 'x_ray')
fs = dm.get_reflection_file_server(filenames=[data_mtz],
labels=[['I,SIGI,merged', 'IPR,SIGIPR,merged']], array_type='x_ray')
assert(len(fs.get_miller_arrays(data_mtz)) == 2)
def test_miller_array_datatype():
data_dir = os.path.dirname(os.path.abspath(__file__))
data_mtz = os.path.join(data_dir, 'data',
'insulin_unmerged_cutted_from_ccp4.mtz')
dm = DataManager(['miller_array', 'phil'])
dm.process_miller_array_file(data_mtz)
# test labels
labels = [
'M_ISYM', 'BATCH', 'I,SIGI,merged', 'IPR,SIGIPR,merged',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG',
'BGPKRATIOS'
]
for label in dm.get_miller_array_labels():
assert (label in labels)
assert (len(dm.get_miller_arrays()) == len(dm.get_miller_array_labels()))
# test access by label
label = dm.get_miller_array_labels()[3]
new_label = dm.get_miller_arrays(labels=[label])[0].info().label_string()
assert (label == new_label)
# test custom PHIL
dm.write_phil_file('test.phil',
dm.export_phil_scope().as_str(),
overwrite=True)
loaded_phil = libtbx.phil.parse(file_name='test.phil')
new_dm = DataManager(['miller_array', 'phil'])
new_dm.load_phil_scope(loaded_phil)
assert (data_mtz == new_dm.get_default_miller_array_name())
for label in new_dm.get_miller_array_labels():
assert (label in labels)
os.remove('test.phil')
# test file server
fs1 = dm.get_reflection_file_server()
fs2 = dm.get_reflection_file_server([data_mtz, data_mtz])
assert (2 * len(fs1.miller_arrays) == len(fs2.miller_arrays))
cs = crystal.symmetry(
unit_cell=dm.get_miller_arrays()[0].crystal_symmetry().unit_cell(),
space_group_symbol='P1')
fs = dm.get_reflection_file_server(crystal_symmetry=cs)
assert (fs.crystal_symmetry.is_similar_symmetry(cs))
assert (not fs.crystal_symmetry.is_similar_symmetry(
dm.get_miller_arrays()[0].crystal_symmetry()))
fs = dm.get_reflection_file_server(labels=['I,SIGI,merged'])
assert (len(fs.get_miller_arrays(None)) == 1)
miller_array = fs.get_amplitudes(None, None, True, None, None)
assert (
miller_array.info().label_string() == 'I,as_amplitude_array,merged')
master_phil_str = '''
include scope iotbx.data_manager.miller_array.miller_array_phil_str
'''
master_phil = libtbx.phil.parse(master_phil_str, process_includes=True)
master_extract = master_phil.extract()
master_extract.data[0].file_name = data_mtz
master_extract.data[0].labels = 'IPR,SIGIPR,merged'
fs = get_reflection_file_server(dm, master_extract)
assert (len(fs.get_miller_arrays(None)) == 1)
master_extract.data[0].type = 'neutron'
fs = get_reflection_file_server(dm, master_extract)
assert (fs is None)
fs = get_reflection_file_server(dm, master_extract, datatype='neutron')
assert (len(fs.get_miller_arrays(None)) == 1)
