def get_map_and_model(params=None,
map_data=None,
crystal_symmetry=None,
pdb_inp=None,
ncs_obj=None,
half_map_data_list=None,
map_coords_inside_cell=True,
out=sys.stdout):
acc = None # accessor used to shift map back to original location if desired
origin_frac = (0, 0, 0)
if map_data and crystal_symmetry:
original_crystal_symmetry = crystal_symmetry
original_unit_cell_grid = None
elif params.input_files.map_file:
print >> out, "\nReading map from %s\n" % (params.input_files.map_file)
from cctbx.maptbx.segment_and_split_map import get_map_object
map_data,space_group,unit_cell,crystal_symmetry,origin_frac,acc,\
original_crystal_symmetry,original_unit_cell_grid=\
get_map_object(file_name=params.input_files.map_file,out=out)
map_data = map_data.as_double()
if origin_frac != (0, 0, 0) and acc is None:
print >>out,"\nWARNING: Unable to place output map at position of "+\
"input map though input map has non-zero origin at %s\n" %(
str(origin_frac))
elif params.input_files.map_coeffs_file:
map_coeffs = get_map_coeffs_from_file(
map_coeffs_file=params.input_files.map_coeffs_file,
map_coeffs_labels=params.input_files.map_coeffs_labels)
if not map_coeffs:
raise Sorry("Could not get map coeffs from %s with labels %s" %
(params.input_files.map_coeffs_file,
params.input_files.map_coeffs_labels))
print >> out, "Map coefficients read from %s with labels %s" % (
params.input_files.map_coeffs_file,
str(params.input_files.map_coeffs_labels))
crystal_symmetry = map_coeffs.crystal_symmetry()
from cctbx.maptbx.segment_and_split_map import get_map_from_map_coeffs
map_data = get_map_from_map_coeffs(map_coeffs=map_coeffs,
crystal_symmetry=crystal_symmetry)
acc = map_data.accessor()
original_crystal_symmetry = crystal_symmetry
original_unit_cell_grid = None
if not params.crystal_info.resolution:
params.crystal_info.resolution = map_coeffs.d_min()
print >> out, "Resolution from map_coeffs is %7.2f A" % (
params.crystal_info.resolution)
else:
raise Sorry("Need ccp4 map or map_coeffs")
if params.input_files.half_map_file:
if len(params.input_files.half_map_file) != 2:
raise Sorry("Please supply zero or two half_map files")
half_map_data_list = []
from cctbx.maptbx.segment_and_split_map import get_map_object
for file_name in params.input_files.half_map_file:
print >> out, "\nReading half-map from %s\n" % (file_name)
half_map_data,half_map_space_group,half_map_unit_cell,\
half_map_crystal_symmetry,half_map_origin_frac,half_map_acc,\
half_map_original_crystal_symmetry,half_map_original_unit_cell_grid=\
get_map_object(file_name=file_name,out=out)
half_map_data = half_map_data.as_double()
assert half_map_crystal_symmetry.is_similar_symmetry(
crystal_symmetry)
half_map_data_list.append(half_map_data)
if params.crystal_info.resolution is None:
raise Sorry("Need resolution if map is supplied")
if params.crystal_info.resolution >= 10:
print >>out,"\n** WARNING: auto_sharpen is designed for maps at a "+\
"resolution of about 4.5 A\nor better. Sharpening may be"+\
"poor at %7.0f A" %(params.crystal_info.resolution)
if params.input_files.pdb_file and not pdb_inp: # get model
model_file = params.input_files.pdb_file
if not os.path.isfile(model_file):
raise Sorry("Missing the model file: %s" % (model_file))
pdb_inp = iotbx.pdb.input(file_name=model_file)
if origin_frac != (0, 0, 0):
print >> out, "Shifting model by %s" % (str(origin_frac))
from cctbx.maptbx.segment_and_split_map import \
apply_shift_to_pdb_hierarchy
origin_shift = crystal_symmetry.unit_cell().orthogonalize(
(-origin_frac[0], -origin_frac[1], -origin_frac[2]))
pdb_inp = apply_shift_to_pdb_hierarchy(
origin_shift=origin_shift,
crystal_symmetry=crystal_symmetry,
pdb_hierarchy=pdb_inp.construct_hierarchy(),
out=out).as_pdb_input()
if map_coords_inside_cell:
# put inside (0,1)
pdb_inp = map_inside_cell(pdb_inp,
crystal_symmetry=crystal_symmetry)
if params.input_files.ncs_file and not ncs_obj: # NCS
from cctbx.maptbx.segment_and_split_map import get_ncs
ncs_obj, dummy_tracking_data = get_ncs(params, out=out)
if origin_frac != (0, 0, 0):
origin_shift = crystal_symmetry.unit_cell().orthogonalize(
(-origin_frac[0], -origin_frac[1], -origin_frac[2]))
print >> out, "Shifting NCS by (%7.2f,%7.2f,%7.2f) " % (
(origin_shift))
from scitbx.math import matrix
ncs_obj = ncs_obj.coordinate_offset(
coordinate_offset=matrix.col(origin_shift))
return pdb_inp,map_data,half_map_data_list,ncs_obj,crystal_symmetry,acc,\
original_crystal_symmetry,original_unit_cell_grid
def run(args,
command_name="phenix.cif_as_mtz",
out=sys.stdout,
return_as_miller_arrays=False):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_cif_file] [options]" % command_name,
description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent' %
command_name
).enable_symmetry_comprehensive().option(
None,
"--output_file_name",
action="store",
default=False,
type="string",
help="Output mtz file name."
).option(
None,
"--wavelength_id",
action="store",
default=None,
type="int",
help="Extract data set with given wavelength_id."
).option(
None,
"--crystal_id",
action="store",
default=None,
type="int",
help="Extract data set with given crystal_id."
).option(
None,
"--output_r_free_label",
action="store",
default="R-free-flags",
type="string",
help=
"MTZ column label to use for R-free flags (default: R-free-flags)"
).option(
None,
"--merge",
action="store_true",
help="Merge non-unique data where present."
).option(
None,
"--incompatible_flags_to_work_set",
action="store_true",
help=
"When merging place reflections with incompatible flags into the "
"working set."
).option(
None,
"--remove_systematic_absences",
action="store_true",
help="Remove systematic absent reflections."
).option(
None,
"--map_to_asu",
action="store_true",
help="Map to asymmetric unit."
).option(
"--show_details_if_error",
action="store_true",
help="Show data details for some errors."
).option(
"--show_log",
action="store_true",
help="Show some output."
).option(
"--ignore_bad_sigmas",
action="store_true",
help=
"Set sigmas to None instead of raising an error when bad sigmas "
"are present."
).option(
"--extend_flags",
action="store_true",
help="Extend R-free flags to cover all reflections if necessary.")
).process(args=args)
except Exception as e:
if (str(e) != "0"): print(str(e))
sys.exit(0)
crystal_symmetry = command_line.symmetry
if (len(command_line.args) > 1):
print("%d arguments are given from the command line:"% \
len(command_line.args), command_line.args, file=out)
raise Sorry("Please specify one reflection cif file.")
file_name = command_line.args[0]
if (not os.path.isfile(file_name)):
raise Sorry("File is not found: %s" % file_name)
output_r_free_label = command_line.options.output_r_free_label
if ((not output_r_free_label[0] in string.ascii_uppercase)
or (re.search("[^a-zA-Z0-9_\-]", output_r_free_label))):
raise Sorry((
"%s is not a suitable column label. MTZ format requires " +
"an uppercase letter as the first character, and only alphanumeric "
+ "characters or hyphens in the rest of the string.") %
output_r_free_label)
result = process_files(
file_name=file_name,
crystal_symmetry=crystal_symmetry,
output_file_name=command_line.options.output_file_name,
wavelength_id=command_line.options.wavelength_id,
crystal_id=command_line.options.crystal_id,
show_details_if_error=command_line.options.show_details_if_error,
output_r_free_label=command_line.options.output_r_free_label,
merge_non_unique_under_symmetry=command_line.options.merge,
map_to_asu=command_line.options.map_to_asu,
remove_systematic_absences=command_line.options.
remove_systematic_absences,
incompatible_flags_to_work_set=command_line.options.
incompatible_flags_to_work_set,
return_as_miller_arrays=return_as_miller_arrays,
ignore_bad_sigmas=command_line.options.ignore_bad_sigmas,
extend_flags=command_line.options.extend_flags,
log=out)
if return_as_miller_arrays:
return result
def __init__(self, fo2, fc,
scale_factor=None,
outlier_cutoff_factor=None,
probability_plot_slope=None):
self.probability_plot_slope = probability_plot_slope
assert fo2.is_xray_intensity_array()
assert fc.is_complex_array()
assert not fo2.space_group().is_centric()
if scale_factor is None:
scale_factor = fo2.scale_factor(fc)
fc2 = fc.as_intensity_array()
self.delta_fc2 = fc2.anomalous_differences()
self.delta_fo2 = fo2.anomalous_differences()
self.n_bijvoet_pairs = self.delta_fo2.size()
if outlier_cutoff_factor is not None:
cutoff_sel = flex.abs(self.delta_fo2.data()) > (
outlier_cutoff_factor * scale_factor) * flex.max(
flex.abs(self.delta_fc2.data()))
self.delta_fo2 = self.delta_fo2.select(~cutoff_sel)
self.delta_fc2 = self.delta_fc2.select(~cutoff_sel)
self.delta_fc2 = self.delta_fc2.customized_copy(
data=self.delta_fc2.data() * scale_factor)
if not self.delta_fo2.size():
raise Sorry("Absolute structure could not be determined")
min_gamma = -10
max_gamma = 10
# quick and dirty to find better min, max gammas
max_log_p_obs = -1e100
while True:
# search for the maximum
width = max_gamma - min_gamma
if width < 0.0001:
break
middle = (min_gamma + max_gamma)/2
a = middle - width/4
b = middle + width/4
value_a = self.log_p_obs_given_gamma(a)
value_b = self.log_p_obs_given_gamma(b)
if value_a > value_b:
max_gamma = middle
elif value_a == value_b:
min_gamma = a
max_gamma = b
else:
min_gamma = middle
max_log_p_obs = max([max_log_p_obs, value_a, value_b])
while True:
# search for where the curve becomes close to zero on the left
min_gamma = middle - width/2
if (width > 100 or
self.log_p_obs_given_gamma(min_gamma) - max_log_p_obs < -10):
break
width *= 2
width = max_gamma - min_gamma
while True:
# search for where the curve becomes close to zero on the right
max_gamma = middle + width/2
if (width > 100 or
self.log_p_obs_given_gamma(max_gamma) - max_log_p_obs < -10):
break
width *= 2
n_steps = 500
d_gamma = (max_gamma - min_gamma)/n_steps
# now do it properly
log_p_obs_given_gammas = flex.double()
for gamma in xfrange(min_gamma, max_gamma, d_gamma):
log_p_obs_given_gammas.append(self.log_p_obs_given_gamma(gamma))
max_log_p_obs = flex.max(log_p_obs_given_gammas)
G_numerator = 0
G_denominator = 0
p_u_gammas = flex.double()
# Numerical integration using trapezoidal rule
for i, gamma in enumerate(xfrange(min_gamma, max_gamma, d_gamma)):
p_u_gamma = math.exp(log_p_obs_given_gammas[i] - max_log_p_obs)
p_u_gammas.append(p_u_gamma)
if i == 0: continue
G_numerator += 0.5 * d_gamma * (
(gamma-d_gamma) * p_u_gammas[-2] + gamma * p_u_gammas[-1])
G_denominator += 0.5 * (p_u_gammas[-2] + p_u_gammas[-1]) * d_gamma
self.G = G_numerator/G_denominator
sigma_squared_G_numerator = 0
# Numerical integration using trapezoidal rule
next = None
for i, gamma in enumerate(xfrange(min_gamma, max_gamma, d_gamma)):
previous = next
next = math.pow((gamma - self.G), 2) * p_u_gammas[i] * d_gamma
if i == 0: continue
sigma_squared_G_numerator += 0.5 * (previous + next)
self.hooft_y = (1-self.G)/2
self.sigma_G = math.sqrt(sigma_squared_G_numerator/G_denominator)
self.sigma_y = self.sigma_G/2
# Now calculate P2, P3 values
log_p_obs_given_gamma_is_minus_1 = self.log_p_obs_given_gamma(-1)
log_p_obs_given_gamma_is_0 = self.log_p_obs_given_gamma(0)
log_p_obs_given_gamma_is_1 = self.log_p_obs_given_gamma(1)
max_log_p_obs = max([log_p_obs_given_gamma_is_minus_1,
log_p_obs_given_gamma_is_0,
log_p_obs_given_gamma_is_1])
# all values normalised by max_log_p_obs for numerical stability
log_p_obs_given_gamma_is_minus_1 -= max_log_p_obs
log_p_obs_given_gamma_is_0 -= max_log_p_obs
log_p_obs_given_gamma_is_1 -= max_log_p_obs
p2_denominator = math.exp(log_p_obs_given_gamma_is_1) \
+ math.exp(log_p_obs_given_gamma_is_minus_1)
p3_denominator = math.exp(log_p_obs_given_gamma_is_1) \
+ math.exp(log_p_obs_given_gamma_is_minus_1) \
+ math.exp(log_p_obs_given_gamma_is_0)
#
if p2_denominator == 0: self.p2_true = self.p2_false = None
else:
self.p2_true = (
math.exp(log_p_obs_given_gamma_is_1)) / p2_denominator
self.p2_false = (
math.exp(log_p_obs_given_gamma_is_minus_1)) / p2_denominator
self.p3_true = (
math.exp(log_p_obs_given_gamma_is_1)) / p3_denominator
self.p3_false = (
math.exp(log_p_obs_given_gamma_is_minus_1)) / p3_denominator
self.p3_racemic_twin = (
math.exp(log_p_obs_given_gamma_is_0)) / p3_denominator
def _target_function05b(args, kwds, connection):
raise Sorry("_target_function05b")
def find_data_arrays(mtz_file, log=None, merge_anomalous=False,
preferred_labels=None, crystal_symmetry=None):
"""
Guess an appropriate data array to use for refinement, plus optional
Hendrickson-Lattman coefficients and R-free flags if present.
"""
from iotbx import reflection_file_utils
from iotbx.file_reader import any_file
if (log is None) : log = sys.stdout
phases = data = flags = flag_value = None
hkl_in = any_file(mtz_file, force_type="hkl")
hkl_server = hkl_in.file_server
# always use anomalous data if available! also, prefer amplitudes over
# intensities if possible, as they may already be on an absolute scale
data_arrays = hkl_server.get_xray_data(
file_name = None,
labels = None,
ignore_all_zeros = False,
parameter_scope = "",
return_all_valid_arrays = True,
minimum_score = 4,
prefer_amplitudes = True,
prefer_anomalous = True)
if (len(data_arrays) == 0):
raise Sorry("No data arrays found in %s." % mtz_file)
data = data_arrays[0]
if (preferred_labels is not None):
for array in data_arrays :
array_labels = array.info().label_string()
if (array_label== preferred_labels):
data = array
break
else :
raise Sorry("Can't find label string '%s'!" % preferred_labels)
else :
print("Defaulting to using %s" % data.info().label_string(), file=log)
hl_arrays = hkl_server.get_experimental_phases(
file_name = None,
labels = None,
ignore_all_zeros = True,
parameter_scope = "",
return_all_valid_arrays = True,
minimum_score = 1)
if (len(hl_arrays) > 0):
phases = hl_arrays[0]
flags_and_values = hkl_server.get_r_free_flags(
file_name=None,
label=None,
test_flag_value=None,
disable_suitability_test=False,
parameter_scope="",
return_all_valid_arrays=True,
minimum_score=1)
if (len(flags_and_values) > 0):
flags, flag_value = flags_and_values[0]
if (crystal_symmetry is not None):
data = data.customized_copy(crystal_symmetry=crystal_symmetry)
if (flags is not None):
flags = flags.customized_copy(crystal_symmetry=crystal_symmetry)
if (phases is not None):
phases = phases.customized_copy(crystal_symmetry=crystal_symmetry)
return reflection_file_utils.process_raw_data(
obs=data,
r_free_flags=flags,
test_flag_value=flag_value,
phases=phases,
log=log,
merge_anomalous=merge_anomalous)
def parse_phil_args(master_phil,
args,
blank_arg_prepend=None,
home_scope=None):
if blank_arg_prepend is None:
pass
elif isinstance(blank_arg_prepend, dict):
for item in blank_arg_prepend.values():
assert '=' in item
elif isinstance(blank_arg_prepend, str):
assert '=' in blank_arg_prepend
else:
raise Exception('blank_arg_prepend must be str or dict')
# Copy the args so that we can remove items from the list without affecting args etc
args = copy.copy(args)
# Construct interpreter
cmd_interpr = master_phil.command_line_argument_interpreter(
home_scope=home_scope)
# Process any args that are eff files
eff_files = [
f for f in args if ((f.endswith('.eff') or f.endswith('.def')) and (
not f.count('=')) and os.path.isfile(f))
]
# Remove them from the original lists
[args.remove(f) for f in eff_files]
# Parse the 'eff' files - these should contain phils
#eff_sources = [libtbx.phil.parse(open(f, 'r').read()) for f in eff_files]
eff_sources = [cmd_interpr.process(open(f, 'r').read()) for f in eff_files]
# Process input arguments
arg_sources = []
for arg in args:
try:
# Prepend if blank
if '=' not in arg:
if isinstance(blank_arg_prepend, dict):
found_key = False
for key in blank_arg_prepend.keys():
if key is None:
continue
if arg.endswith(key):
arg = blank_arg_prepend[key] + arg
found_key = True
break
if (found_key == False) and (None
in blank_arg_prepend.keys()):
arg = blank_arg_prepend[None] + arg
elif isinstance(blank_arg_prepend, str):
arg = blank_arg_prepend + arg
# Attempt to process arg
cmd_line_args = cmd_interpr.process(arg=arg)
except KeyboardInterrupt:
raise
except Exception:
raise Sorry("Unknown file or keyword: %s" % arg)
else:
arg_sources.append(cmd_line_args)
# Extract Scope object (putting eff sources first so that they're overridden if double-defined)
working_phil = master_phil.fetch(sources=eff_sources + arg_sources)
return working_phil
def run(params):
log = Log(log_file=params.output.log_file, verbose=True)
# Process MTZs
if params.input.mtz:
log.heading('Processing {} MTZ Files'.format(len(params.input.mtz)))
if params.input.file_label=='filename': labels = [os.path.basename(os.path.splitext(f)[0]) for f in params.input.mtz]
elif params.input.file_label=='foldername': labels = [os.path.basename(os.path.dirname(f)) for f in params.input.mtz]
else: raise Exception('MTZ labelling function not supported: {}'.format(params.input.file_label))
log.bar()
log('Grouping {} mtz files by space group'.format(len(params.input.mtz)))
crystal_groups = CrystalGroup.by_space_group(crystals=[CrystalSummary.from_mtz(mtz_file=f, id=lab) for f,lab in zip(params.input.mtz, labels)])
log('> Clustered into {} space group(s)'.format(len(crystal_groups)))
log.bar()
for cg in crystal_groups:
log.subheading('Space group {} - {} datasets'.format(','.join(cg.space_groups), len(cg.crystals)))
error = False
for c in cg.crystals:
for label in params.check_for.column_label:
if label is None: continue
if label not in c.column_labels:
log('Checking: column "{}" not in diffraction data of {}. columns present are {}'.format(label, c.mtz_file, c.column_labels))
for label in params.summary.column_label:
if label is None: continue
if label not in c.column_labels:
log('Required: column "{}" not in diffraction data of {}. columns present are {}'.format(label, c.mtz_file, c.column_labels))
error = True
if error is True: raise Sorry('There are datasets that do not contain the right columns.')
log(crystal_statistics('Wavelength', cg.crystals, value_func=lambda c: c.mtz_object().crystals()[1].datasets()[0].wavelength(), header=True))
log(crystal_statistics('Resolution (high)', cg.crystals, value_func=lambda c: c.high_res, header=False))
log(crystal_statistics('Resolution (low)', cg.crystals, value_func=lambda c: c.low_res, header=False))
log(crystal_statistics('Unit cell - vol', cg.crystals, value_func=lambda c: c.unit_cell.volume(), header=False))
log(crystal_statistics('Unit cell - a', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[0], header=False))
log(crystal_statistics('Unit cell - b', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[1], header=False))
log(crystal_statistics('Unit cell - c', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[2], header=False))
log(crystal_statistics('Unit cell - alpha', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[3], header=False))
log(crystal_statistics('Unit cell - beta', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[4], header=False))
log(crystal_statistics('Unit cell - gamma', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[5], header=False, footer=True))
for label in params.summary.column_label:
if label is None: continue
log(crystal_statistics('Column: {}'.format(label), cg.crystals, value_func=lambda c: c.mtz_object().get_column(label).n_valid_values(), header=False, footer=True))
log.bar(True, False)
log('Smallest + Largest Values')
log.bar()
log(crystal_min_max('Resolution', cg.crystals, value_func=lambda c: c.high_res))
# Process PDBs
if params.input.pdb:
log.heading('Processing {} PDB Files'.format(len(params.input.pdb)))
if params.input.file_label=='filename': labels = [os.path.basename(os.path.splitext(f)[0]) for f in params.input.pdb]
elif params.input.file_label=='foldername': labels = [os.path.basename(os.path.dirname(f)) for f in params.input.pdb]
else: raise Exception('PDB labelling function not supported: {}'.format(params.input.file_label))
log.bar()
log('Grouping {} pdb files by space group'.format(len(params.input.pdb)))
crystal_groups = CrystalGroup.by_space_group(crystals=[CrystalSummary.from_pdb(pdb_file=f, id=lab) for f,lab in zip(params.input.pdb, labels)])
log('> Clustered into {} space group(s)'.format(len(crystal_groups)))
for cg in crystal_groups:
log.subheading('Space group: {} - {} datasets'.format(','.join(cg.space_groups), len(cg.crystals)))
log(crystal_statistics('R-work', cg.crystals, value_func=lambda c: c.pdb_input().get_r_rfree_sigma().r_work, header=True))
log(crystal_statistics('R-free', cg.crystals, value_func=lambda c: c.pdb_input().get_r_rfree_sigma().r_free, header=False, footer=True))
log.bar(True, False)
log('Smallest + Largest Values')
log.bar()
log(crystal_min_max('R-free', cg.crystals, value_func=lambda c: c.pdb_input().get_r_rfree_sigma().r_free))
log.heading('finished')
def as_miller_arrays(
self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None,
assume_shelx_observation_type_is=None,
enforce_positive_sigmas=False,
anomalous=None,
):
"""
Convert the contents of the reflection file into a list of
:py:class:`cctbx.miller.array` objects, each of which may contain multiple
columns of data from the underlying file. By default this will
automatically merge redundant observations to obtain a unique set under
symmetry.
:param crystal_symmetry: :py:class:`cctbx.crystal.symmetry` object
(defaults to using internally specified symmetry, if any)
:param force_symmetry: TODO
:param merge_equivalents: merge redundant obervations (default=True)
:param base_array_info: :py:class:`cctbx.miller.array_info` object
containing basic information to be propagated to the arrays
:param assume_shelx_observation_type_is: if specified, instead of raising
an exception if the SHELX file type is not known from the file name plus
data type tag, the function will force the specified data type.
"""
assert (assume_shelx_observation_type_is
in [None, "amplitudes", "intensities"])
if (self._file_type is None):
return []
info_source = self._file_name
if (info_source.startswith("./") or info_source.startswith(".\\")):
info_source = info_source[2:]
if (base_array_info is None):
base_array_info = miller.array_info(source=info_source,
source_type=self._file_type)
if (self._file_type == "cctbx.miller.array"):
result = []
for miller_array in self._file_content:
info = miller_array.info()
if (info is None or not isinstance(info, miller.array_info)):
info = base_array_info
info.source = info_source
info.crystal_symmetry_from_file = crystal.symmetry(
unit_cell=miller_array.unit_cell(),
space_group_info=miller_array.space_group_info(),
raise_sorry_if_incompatible_unit_cell=True)
result.append(
miller_array.customized_copy(
crystal_symmetry=miller_array.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry,
raise_sorry_if_incompatible_unit_cell=True)).
set_info(info).set_observation_type(
miller_array.observation_type()))
return result
if ((crystal_symmetry is None or crystal_symmetry.unit_cell() is None)
and self._observation_type == 'hklf+ins/res'):
name, ext = os.path.splitext(self._file_name)
if ext != '.hkl': # it may be compressed: name.hkl.gz
name, ext = os.path.splitext(name)
for shelx_file_name in ('%s.ins' % name, '%s.res' % name):
try:
shelx_file = open(shelx_file_name)
break
except IOError:
continue
else:
raise Sorry("Can't open files %s.ins or %s.res"
"required by the option hklf+ins/res" %
((name, ) * 2))
crystal_symmetry = crystal_symmetry_from_ins.extract_from(
file=shelx_file)
shelx_file.seek(0)
remaining = shelx_file.read()
shelx_file.close()
m = re.search(r"^HKLF\s*(\d)", remaining, re.X | re.M | re.S)
if m is None:
raise Sorry(
"%s does not contain the mandatory HKLF instruction" %
shelx_file.name)
if m.group(1) == "4":
self._observation_type = "intensities"
elif m.group(1) == "3":
self._observation_type = "amplitudes"
else:
raise Sorry("HKLF %s not supported" % m.group(1))
result = self._file_content.as_miller_arrays(
crystal_symmetry=crystal_symmetry,
force_symmetry=force_symmetry,
merge_equivalents=merge_equivalents,
base_array_info=base_array_info,
anomalous=anomalous,
)
if (self.file_type() == "shelx_hklf"):
if ((self._observation_type == "intensities")
or (assume_shelx_observation_type_is == "intensities")):
result[0].set_info(result[0].info().customized_copy(
labels=["Iobs", "SigIobs"]))
result[0].set_observation_type_xray_intensity()
elif ((self._observation_type == "amplitudes")
or (assume_shelx_observation_type_is == "amplitudes")):
result[0].set_info(result[0].info().customized_copy(
labels=["Fobs", "SigFobs"]))
result[0].set_observation_type_xray_amplitude()
else:
raise Sorry(
"Unresolved amplitude/intensity ambiguity: %s\n"
" SHELX reflection files may contain amplitudes or intensities.\n"
" Please append =amplitudes\n"
" or =hklf3\n"
" or =intensities\n"
" or =hklf4\n"
" to the file name argument or parameter to resolve the"
" ambiguity.\n"
" If a corresponding .ins file is available, look for the"
" HKLF codeword.\n"
" Alternatively, run the phenix.reflection_statistics"
" command twice,\n"
" once with =amplitudes and once with =intensities. Inspect"
" the <I^2>/(<I>)^2\n"
" statistics. For acentric structures the values should"
" fluctuate around\n"
" 2.0, for centric structures around 3.0. If the statistics"
" are not conclusive\n"
" it will be best to recover the original reflection data, such"
" as SCALEPACK,\n"
" SCALA MTZ, XDS, or d*TREK files." % self._file_name)
# discard reflections where sigma <= 0
# XXX note that this will happen after data merging, so for unmerged data
# it is better to specify merge_equivalents=False!
if (enforce_positive_sigmas):
result_ = []
for array in result:
result_.append(array.enforce_positive_sigmas())
result = result_
return result
def check_if_atoms_superposed(self, r1, r2, i1, i2):
if abs((r1 - r2).length()) < 0.001:
sorry_str = '''Atoms %s and %s are superposed or very close.
Fix your model before proceeding.''' % (self.site_labels[i1],
self.site_labels[i2])
raise Sorry(sorry_str)
def run(args, command_name="phenix.fest"):
print_banner(command_name=command_name)
scenarios = [ "SAD",#0
"SIR",#1
"RIP",#2
"SIRAS",#3
"RIPAS",#4
"MIR",#5
"MIRAS",#6
"2WMAD",#7
"3WMAD",#8
"4WMAD",#9
"2WMAD+NAT",#10
"3WMAD+NAT",#11
"4WMAD+NAT",#12
"2WSAD",#13
"3WSAD",#14
"4WSAD" ]#15
if len(args)==0:
print()
print("usage: %s <EXPERIMENT TYPE> <FLAGS and/or PARAMETER FILE>" \
% command_name)
print()
raise Sorry("No instructions received")
if args[0] in scenarios:
print("experiment: " , args[0])
else:
print("Unknown experiment type")
print(" Choose from the following list:")
for method in scenarios:
print(method)
raise Sorry("Unmknown experimenttype")
## this is a rather ugly enumeration of all cases
if args[0]==scenarios[0]:
sad_scale.run( args[1:] )
elif args[0]==scenarios[1]:
sir_scale.run( args[1:] )
elif args[0]==scenarios[2]:
rip_scale.run( args[1:] )
elif args[0]==scenarios[3]:
siras_scale.run( args[1:] )
elif args[0]==scenarios[4]:
siras_scale.run( args[1:] )
elif args[0]==scenarios[7]:
twmad_scale.run( args[1:] )
else:
print()
print("Sorry, no time. This method has not yet been implemented.")
print("Currently, only SAD,SIR,RIP and 2WMAD are supported.")
print()
def run(args, out=sys.stdout, quiet=False):
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=get_master_phil(),
pdb_file_def="model",
usage_string=usage_string)
params = cmdline.work.extract()
if (params.model is None and params.model_list is None):
raise Usage(usage_string)
if params.model:
models = [params.model]
elif params.model_list:
if os.path.isfile(params.model_list):
with open(params.model_list, 'r') as f:
models = f.read().split('\n')
models = [m for m in models if m != ""]
else:
models = params.model_list.split(',')
params.verbose = False
params.model = models[0]
results = []
for model in models:
if not os.path.isfile(model) and params.model_list:
print "Cannot find '%s', skipping." % model
continue
pdb_in = cmdline.get_file(model, force_type="pdb")
hierarchy = pdb_in.file_object.hierarchy
hierarchy.atoms().reset_i_seq()
result = mmtbx.validation.ramalyze.ramalyze(
pdb_hierarchy=hierarchy,
show_errors=None,
outliers_only=params.outliers_only,
out=out,
quiet=quiet)
results.append(result)
if params.model_list:
print '\nmodel : %s' % model
result.show_summary()
# combine
result = results[0]
for i in range(1, len(results)):
result += results[i]
if params.verbose:
result.show_old_output(out=out, verbose=True)
if params.plot:
plot_file_base = params.output_prefix
if plot_file_base is None:
plot_file_base = os.path.splitext(os.path.basename(
params.model))[0]
result.write_plots(plot_file_base=plot_file_base,
out=out,
show_labels=params.show_labels,
point_style=params.point_style)
if params.wxplot:
try:
import wxtbx.app
except ImportError, e:
raise Sorry("wxPython not available.")
else:
app = wxtbx.app.CCTBXApp(0)
result.display_wx_plots()
app.MainLoop()
def split_for_scan_range(self, experiments, reference, scan_range):
''' Update experiments when scan range is set. '''
from dxtbx.model.experiment_list import ExperimentList
from dxtbx.model.experiment_list import Experiment
from dials.array_family import flex
# Only do anything is the scan range is set
if scan_range is not None and len(scan_range) > 0:
# Ensure that all experiments have the same imageset and scan
iset = [e.imageset for e in experiments]
scan = [e.scan for e in experiments]
assert (all(x == iset[0] for x in iset))
assert (all(x == scan[0] for x in scan))
# Get the imageset and scan
iset = experiments[0].imageset
scan = experiments[0].scan
# Get the array range
if scan is not None:
frames_start, frames_end = scan.get_array_range()
assert (scan.get_num_images() == len(iset))
else:
frames_start, frames_end = (0, len(iset))
# Create the new lists
new_experiments = ExperimentList()
new_reference_all = reference.split_by_experiment_id()
new_reference = flex.reflection_table()
for i in range(len(new_reference_all) - len(experiments)):
new_reference_all.append(flex.reflection_table())
assert (len(new_reference_all) == len(experiments))
# Loop through all the scan ranges and create a new experiment list with
# the requested scan ranges.
for scan_start, scan_end in scan_range:
# Validate the requested scan range
if scan_end == scan_start:
raise Sorry(
"Scan range end must be higher than start; pass {},{} for single image"
.format(scan_start, scan_start + 1))
if scan_end < scan_start:
raise Sorry("Scan range must be in ascending order")
elif scan_start < frames_start or scan_end > frames_end:
raise Sorry(
"Scan range must be within image range {}..{}".format(
frames_start, frames_end))
assert (scan_end > scan_start)
assert (scan_start >= frames_start)
assert (scan_end <= frames_end)
index_start = scan_start - frames_start
index_end = index_start + (scan_end - scan_start)
assert (index_start < index_end)
assert (index_start >= 0)
assert (index_end <= len(iset))
new_iset = iset[index_start:index_end]
if scan is None:
new_scan = None
else:
new_scan = scan[index_start:index_end]
for i, e1 in enumerate(experiments):
e2 = Experiment()
e2.beam = e1.beam
e2.detector = e1.detector
e2.goniometer = e1.goniometer
e2.crystal = e1.crystal
e2.profile = e1.profile
e2.imageset = new_iset
e2.scan = new_scan
new_reference_all[i]['id'] = flex.int(
len(new_reference_all[i]), len(new_experiments))
new_reference.extend(new_reference_all[i])
new_experiments.append(e2)
experiments = new_experiments
reference = new_reference
# Print some information
logger.info(
'Modified experiment list to integrate over requested scan range'
)
for scan_start, scan_end in scan_range:
logger.info(' scan_range = %d -> %d' % (scan_start, scan_end))
logger.info('')
# Return the experiments
return experiments, reference
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry('more than 1 reflection file was given')
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry('no experiment list was specified')
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise Sorry('Shoeboxes must be saved to integration intermediates to apply an absorption correction. '\
+'Set integration.debug.output=True, integration.debug.separate_files=False and '\
+'integration.debug.delete_shoeboxes=True to temporarily store shoeboxes.')
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info('Using external mask: %s' % mask.filename)
for tile in mask.data:
logger.info(' Mask has %d pixels masked' %
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.debug("Models for experiment %d" % i)
logger.debug("")
logger.debug(str(exp.beam))
logger.debug(str(exp.detector))
if exp.goniometer:
logger.debug(str(exp.goniometer))
if exp.scan:
logger.debug(str(exp.scan))
logger.debug(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
# Check pixels don't belong to neighbours
if reference is not None:
if exp.goniometer is not None and exp.scan is not None:
self.filter_reference_pixels(reference, experiments)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Modify experiment list if exclude images is set
experiments = self.exclude_images(experiments, params.exclude_images)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(unmatched)))
logger.info('*' * 80)
logger.info('')
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src['id'] == eid)
tmp['id'] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference['id'] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
experiments = ProfileModelFactory.create(params, experiments)
for expr in experiments:
if expr.profile is None:
raise Sorry('No profile information in experiment list')
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(params)
refls = sig_filter(experiments, reflections)
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(reflections) - len(refls), len(reflections)))
reflections = refls
# Delete the shoeboxes used for intermediate calculations, if requested
if params.integration.debug.delete_shoeboxes and 'shoebox' in reflections:
del reflections['shoebox']
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
# This code is distributed under the BSD license, a copy of which is
# included in the root directory of this package.
#
import libtbx
from libtbx.utils import Sorry
from scitbx.array_family import flex
import logging
logger = logging.getLogger(__name__)
from dials.algorithms.refinement.engine import DisableMPmixin
try:
from scitbx.examples.bevington import non_linear_ls_eigen_wrapper
except ImportError, e:
raise Sorry(
"""Eigen package is not available. Please untar the Eigen source package
(http://eigen.tuxfamily.org) and place a link to it (eigen--> Eigen source dir) in
the modules directory of your developer install; then recompile.
""")
from dials.algorithms.refinement.engine import AdaptLstbx as AdaptLstbxBase
class AdaptLstbxSparse(DisableMPmixin, AdaptLstbxBase,
non_linear_ls_eigen_wrapper):
"""Adapt the base class for Eigen"""
def __init__(self,
target,
prediction_parameterisation,
constraints_manager=None,
log=None,
verbosity=0,
def integration_concept(self,image_number=0,cb_op_to_primitive=None,verbose=False,**kwargs):
self.image_number = image_number
NEAR = 10
pxlsz = self.pixel_size
self.get_predictions_accounting_for_centering(cb_op_to_primitive,**kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) )
refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0]
self.inputpd["symmetry"].show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = self.inputai.getOrientation().unit_cell()
query = flex.double()
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0]/pxlsz)
query.append(pred[1]/pxlsz)
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
spots = self.get_observations_with_outlier_removal()
assert len(spots)>NEAR# Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
IS_adapt.query(query)
print("Calculate correction vectors for %d observations & %d predictions"%(len(spots),len(self.predicted)))
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
idx_cutoff = float(min(self.mask_focus[image_number]))
if verbose:
print("idx_cutoff distance in pixels",idx_cutoff)
if not self.horizons_phil.integration.enable_one_to_one_safeguard:
# legacy code, no safeguard against many-to-one predicted-to-observation mapping
for i in range(len(self.predicted)): # loop over predicteds
#for n in range(NEAR): # loop over near spotfinder spots
for n in range(1): # only consider the nearest spotfinder spot
Match = dict(spot=IS_adapt.nn[i*NEAR+n],pred=i)
if n==0 and math.sqrt(IS_adapt.distances[i*NEAR+n]) < idx_cutoff:
indexed_pairs_provisional.append(Match)
vector = matrix.col(
[spots[Match["spot"]].ctr_mass_x() - self.predicted[Match["pred"]][0]/pxlsz,
spots[Match["spot"]].ctr_mass_y() - self.predicted[Match["pred"]][1]/pxlsz])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
else:
one_to_one = {}
for i in range(len(self.predicted)): # loop over predicteds
annresultidx = i*NEAR
obsidx = IS_adapt.nn[annresultidx]
this_distancesq = IS_adapt.distances[annresultidx]
if obsidx not in one_to_one or \
this_distancesq < one_to_one[obsidx]["distancesq"]:
if math.sqrt(this_distancesq) < idx_cutoff:
one_to_one[obsidx] = dict(spot=obsidx,pred=i,distancesq=this_distancesq)
for key,value in one_to_one.items():
indexed_pairs_provisional.append(value)
vector = matrix.col(
[spots[value["spot"]].ctr_mass_x() - self.predicted[value["pred"]][0]/pxlsz,
spots[value["spot"]].ctr_mass_y() - self.predicted[value["pred"]][1]/pxlsz])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
print("... %d provisional matches"%len(correction_vectors_provisional), end=' ')
print("r.m.s.d. in pixels: %5.2f"%(math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]],[-cv[0]],"b.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
fig = plt.figure()
for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional):
plt.plot([spots[match["spot"]].ctr_mass_y()],[-spots[match["spot"]].ctr_mass_x()],"r.")
plt.plot([self.predicted[match["pred"]][1]/pxlsz],[-self.predicted[match["pred"]][0]/pxlsz],"g.")
plt.plot([spots[match["spot"]].ctr_mass_y(), spots[match["spot"]].ctr_mass_y() + 10.*cv[1]],
[-spots[match["spot"]].ctr_mass_x(), -spots[match["spot"]].ctr_mass_x() - 10.*cv[0]],'b-')
plt.xlim([0,float(self.inputpd["size2"])])
plt.ylim([-float(self.inputpd["size1"]),0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"map")
plt.close()
# insert code here to remove correction length outliers...
# they are causing terrible
# problems for finding legitimate correction vectors (print out the list)
# also remove outliers for the purpose of reporting RMS
outlier_rejection = True
cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False)
if outlier_rejection:
correction_lengths = flex.double([v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
ACCEPTABLE_LIMIT = 2
limit = int(0.33 * len(sorted_cl)) # best 1/3 of data are assumed to be correctly modeled.
if (limit <= ACCEPTABLE_LIMIT):
raise Sorry("Not enough indexed spots to reject outliers; have %d need >%d" % (limit, ACCEPTABLE_LIMIT))
y_data = flex.double(len(sorted_cl))
for i in range(len(y_data)):
y_data[i] = float(i)/float(len(y_data))
# ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616.
from rstbx.outlier_spots.fit_distribution import fit_cdf,rayleigh
fitted_rayleigh = fit_cdf(x_data = sorted_cl[0:limit],
y_data = y_data[0:limit],
distribution=rayleigh)
inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data]
#print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in range(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
if (sorted_cl[icand]-inv_cdf[icand])/fitted_rayleigh.distribution.sigma > 1.0:
break
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append(
spots[indexed_pairs[-1]["spot"]])
if kwargs.get("verbose_cv")==True:
print("CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%(
float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2.,
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz), end=' ')
print("OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%(
spots[indexed_pairs[-1]["spot"]].ctr_mass_x(),
spots[indexed_pairs[-1]["spot"]].ctr_mass_y(),
self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz,
self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz), end=' ')
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print("HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id, end=' ')
radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)
print("RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal))
# Store a list of correction vectors in self.
radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz),
obsspot=(spots[indexed_pairs[-1]['spot']].ctr_mass_x(),
spots[indexed_pairs[-1]['spot']].ctr_mass_y()),
predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz,
self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
print("After outlier rejection %d indexed spotfinder spots remain."%len(indexed_pairs))
if False:
rayleigh_cdf = [
fitted_rayleigh.distribution.cdf(x=sorted_cl[c]) for c in range(len(sorted_cl))]
from matplotlib import pyplot as plt
plt.plot(sorted_cl,y_data,"r+")
#plt.plot(sorted_cl,rayleigh_cdf,"g.")
plt.plot(inv_cdf,y_data,"b.")
plt.show()
else:
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### finished with outlier rejection
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
is_triclinic = (self.setting_id==1)
if is_triclinic:
self.triclinic_pairs = [ dict(pred=self.hkllist[a["pred"]],spot=a["spot"])
for a in indexed_pairs ]
if self.horizons_phil.integration.model == "user_supplied":
if kwargs.get("user-reentrant",None)==None:
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs)
return
########### finished with user-supplied code
if self.horizons_phil.integration.spot_shape_verbose:
from rstbx.new_horizons.spot_shape import spot_shape_verbose
spot_shape_verbose(rawdata = self.imagefiles.images[self.image_number].linearintdata,
beam_center_pix = matrix.col((self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)),
indexed_pairs = indexed_pairs,
spotfinder_observations = spots,
distance_mm = self.inputai.distance(),
mm_per_pixel = pxlsz,
hkllist = self.hkllist,
unit_cell = self.cell,
wavelength_ang = self.inputai.wavelength
)
#Other checks to be implemented (future):
# spot is within active area of detector on a circular detector such as the Mar IP
# integration masks do not overlap; or deconvolute
correction_lengths=flex.double([v.length() for v in correction_vectors])
if verbose:
print("average correction %5.2f over %d vectors"%(flex.mean(correction_lengths),
len(correction_lengths)), end=' ')
print("or %5.2f mm."%(pxlsz*flex.mean(correction_lengths)))
self.r_residual = pxlsz*flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
#self.null_correction_mapping( predicted=self.predicted,
# correction_vectors = correction_vectors,
# IS_adapt = IS_adapt,
# spots = spots)
self.positional_correction_mapping( predicted=self.predicted,
correction_vectors = correction_vectors,
PS_adapt = PS_adapt,
IS_adapt = IS_adapt,
spots = spots)
# which spots are close enough to interfere with background?
MAXOVER=6
OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration")
nbr_cutoff = 2.0* max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff/2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0]);flex_sorted.append(item[1]);
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id);
else:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted);
for i in range(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in range(0,len(keys),2):
B_S_mask[(keys[k],keys[k+1])]=True
self.BSmasks.append(B_S_mask)
#print "Done"
return
.multiple = True
apply_detector_distance = False
.type = bool
.help = If True, copy detector distance
""")
if (__name__ == "__main__"):
user_phil = []
for arg in sys.argv[1:]:
if (os.path.isfile(arg)):
user_phil.append(libtbx.phil.parse("""dest_cbf=\"%s\"""" % arg))
else:
try:
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError, e:
raise Sorry("Unrecognized argument '%s' (error: %s)" %
(arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if (params.source_cbf is None) or not os.path.isfile(params.source_cbf):
master_phil.show()
raise Usage("source_cbf must be a file")
if (params.source_cbf is None):
master_phil.show()
raise Usage(
"dest_cbf must be a file (either dest_cbf=XXX, or the file path(s) alone)."
)
print "Source file:", params.source_cbf
print "Destination file(s):",
for path in params.dest_cbf:
print path,
def get_predictions_accounting_for_centering(self,cb_op_to_primitive=None,**kwargs):
# interface requires this function to set current_orientation
# in the actual setting used for Miller index calculation
if (self.horizons_phil.known_setting is None or self.horizons_phil.known_setting == self.setting_id ) and \
self.horizons_phil.integration.model in ["use_case_3_simulated_annealing",
"use_case_3_simulated_annealing_7",
"use_case_3_simulated_annealing_9"]:
if cb_op_to_primitive==None:
raise Sorry("Can't use model_3 simulated annealing for non-primitive cells, contact authors.")
if self.horizons_phil.integration.model=="use_case_3_simulated_annealing":
self.best_params = dict(zip(("half_mosaicity_deg","wave_HE_ang","wave_LE_ang",
"reserve_orientation","rotation100_rad","rotation010_rad","rotation001_rad"),
self.use_case_3_simulated_annealing(self.horizons_phil.integration.use_subpixel_translations))
)
elif self.horizons_phil.integration.model=="use_case_3_simulated_annealing_7":
self.best_params = dict(zip(("half_mosaicity_deg","wave_HE_ang","wave_LE_ang",
"reserve_orientation","rotation100_rad","rotation010_rad","rotation001_rad",
"domain_size_ang"),
self.use_case_3_simulated_annealing_7(self.horizons_phil.integration.use_subpixel_translations))
)
elif self.horizons_phil.integration.model=="use_case_3_simulated_annealing_9":
self.best_params = dict(zip(("half_mosaicity_deg","wave_HE_ang","wave_LE_ang",
"reserve_orientation","rotation100_rad","rotation010_rad","rotation001_rad",
"domain_size_ang","ab_factor","c_factor"),
self.use_case_3_simulated_annealing_9(self.horizons_phil.integration.use_subpixel_translations))
)
self.current_orientation = self.best_params["reserve_orientation"]
self.current_cb_op_to_primitive = cb_op_to_primitive
BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format()
BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls()
self.predicted,self.hkllist = BPpredicted, BPhkllist
self.partialities = dict(indices=BPhkllist.deep_copy(),
data=self.bp3_wrapper.ucbp3.selected_partialities())
#self.hi = self.bp3_wrapper.ucbp3.selected_hi_predictions()
#self.lo = self.bp3_wrapper.ucbp3.selected_lo_predictions()
#print list(self.partialities)
#for x in range(len(self.hi)):
# print "%4d %4d %4d %7.1f %7.1f %7.1f %7.1f PARTIAL %7.2f"%(
# self.hkllist[x][0], self.hkllist[x][1],self.hkllist[x][2],
# self.hi[x][0], self.hi[x][1],
# self.lo[x][0], self.lo[x][1], self.partialities[x])
if self.inputai.active_areas != None:
self.predicted,self.hkllist = self.inputai.active_areas(
self.predicted,self.hkllist,self.pixel_size)
return
if self.horizons_phil.integration.model == "user_supplied":
lower_limit_domain_size = math.pow(
self.inputai.getOrientation().unit_cell().volume(),
1./3.)*self.horizons_phil.integration.mosaic.domain_size_lower_limit # default 10-unit cell block size minimum reasonable domain
actual_used_domain_size = kwargs.get("domain_size_ang",lower_limit_domain_size)
if cb_op_to_primitive==None:
from cxi_user import pre_get_predictions
self.bp3_wrapper = pre_get_predictions(self.inputai, self.horizons_phil,
raw_image = self.imagefiles.images[self.image_number],
imageindex = self.frame_numbers[self.image_number],
spotfinder = self.spotfinder,
limiting_resolution = self.limiting_resolution,
domain_size_ang = actual_used_domain_size)
self.current_orientation = self.inputai.getOrientation()
self.current_cb_op_to_primitive = cb_op_to_primitive
BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format()
BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls()
self.predicted,self.hkllist = BPpredicted, BPhkllist
if self.inputai.active_areas != None:
self.predicted,self.hkllist = self.inputai.active_areas(
self.predicted,self.hkllist,self.pixel_size)
return
else:
self.block_counter+=1
rot_mat = matrix.sqr(cb_op_to_primitive.c().r().as_double()).transpose()
centered_orientation = self.inputai.getOrientation()
self.current_orientation = centered_orientation
self.current_cb_op_to_primitive = cb_op_to_primitive
primitive_orientation = centered_orientation.change_basis(rot_mat)
self.inputai.setOrientation(primitive_orientation)
from cxi_user import pre_get_predictions
if self.block_counter < 2:
KLUDGE = self.horizons_phil.integration.mosaic.kludge1 # bugfix 1 of 2 for protocol 6, equation 2
self.inputai.setMosaicity(KLUDGE*self.inputai.getMosaicity())
self.bp3_wrapper = pre_get_predictions(self.inputai, self.horizons_phil,
raw_image = self.imagefiles.images[self.image_number],
imageindex = self.frame_numbers[self.image_number],
spotfinder = self.spotfinder,
limiting_resolution = self.limiting_resolution,
domain_size_ang = actual_used_domain_size)
BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format()
BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls()
self.actual = actual_used_domain_size
self.predicted = BPpredicted
primitive_hkllist = BPhkllist
#not sure if matrix needs to be transposed first for outputting HKL's???:
self.hkllist = cb_op_to_primitive.inverse().apply(primitive_hkllist)
self.inputai.setOrientation(centered_orientation)
if self.inputai.active_areas != None:
self.predicted,self.hkllist = self.inputai.active_areas(
self.predicted,self.hkllist,self.pixel_size)
if self.block_counter < 2:
down = self.inputai.getMosaicity()/KLUDGE
print("Readjusting mosaicity back down to ",down)
self.inputai.setMosaicity(down)
return
if cb_op_to_primitive==None:
predicted = self.inputai.predict_all(
self.image_centers[self.image_number],self.limiting_resolution)
self.predicted = predicted.vec3() #only good for integrating one frame...
self.hkllist = predicted.hkl()
self.current_orientation = self.inputai.getOrientation()
from cctbx import sgtbx
self.cb_op_to_primitive = sgtbx.change_of_basis_op()
else:
rot_mat = matrix.sqr(cb_op_to_primitive.c().r().as_double()).transpose()
centered_orientation = self.inputai.getOrientation()
self.current_orientation = centered_orientation
self.current_cb_op_to_primitive = cb_op_to_primitive
primitive_orientation = centered_orientation.change_basis(rot_mat)
self.inputai.setOrientation(primitive_orientation)
predicted = self.inputai.predict_all(
self.image_centers[self.image_number],self.limiting_resolution)
self.predicted = predicted.vec3() #only good for integrating one frame...
primitive_hkllist = predicted.hkl()
#not sure if matrix needs to be transposed first for outputting HKL's???:
self.hkllist = cb_op_to_primitive.inverse().apply(primitive_hkllist)
self.inputai.setOrientation(centered_orientation)
if self.inputai.active_areas != None:
self.predicted,self.hkllist = self.inputai.active_areas(
self.predicted,self.hkllist,self.pixel_size)
if False: #development only; compare the two methods:
from matplotlib import pyplot as plt
plt.plot([i[0] for i in BPpredicted],[i[1] for i in BPpredicted],"r.")
plt.plot([i[0] for i in predicted],[i[1] for i in predicted],"b.")
plt.show()
def write_map(
self,
file_name=None, # Name of file to be written
verbose=None,
):
'''
Simple version of write
file_name is output file name
map_data is map_data object with 3D values for map. If not supplied,
use self.map_data()
Normally call with file_name (file to be written)
Output labels are generated from existing self.labels,
self.program_name, and self.limitations
'''
if not file_name:
raise Sorry("Need file_name for write_map")
if not self.map_data():
raise Sorry("Need map_data for write_map")
map_data = self.map_data()
from iotbx.mrcfile import create_output_labels
labels = create_output_labels(program_name=self.program_name,
input_file_name=self.input_file_name,
input_labels=self.labels,
limitations=self.limitations)
crystal_symmetry = self.unit_cell_crystal_symmetry()
unit_cell_grid = self.unit_cell_grid
origin_shift_grid_units = self.origin_shift_grid_units
if map_data.origin() == (0, 0, 0): # Usual
self._print(
"Writing map with origin at %s and size of %s to %s" %
(str(origin_shift_grid_units), str(map_data.all()), file_name))
write_ccp4_map(
file_name=file_name,
crystal_symmetry=crystal_symmetry, # unit cell and space group
map_data=map_data,
unit_cell_grid=
unit_cell_grid, # optional gridding of full unit cell
origin_shift_grid_units=
origin_shift_grid_units, # optional origin shift
labels=labels,
verbose=verbose)
else: # map_data has not been shifted to (0,0,0). Shift it and then write
self._print("Writing map after shifting origin")
if self.origin_shift_grid_units and origin_shift_grid_units != (
0, 0, 0):
self._print(
"WARNING: map_data has origin at %s " %
(str(map_data.origin())),
" and this map_manager will apply additional origin shift of %s "
% (str(self.origin_shift_grid_units)))
new_mm = self.deep_copy()
new_mm.set_log(self.log)
new_mm.shift_origin()
new_mm.write_map(file_name=file_name)
def __init__(
self,
model,
pdb_hierarchy=None, # keep for mmtbx.validation_summary (multiple models)
fmodel=None,
fmodel_neutron=None,
sequences=None,
flags=None,
header_info=None,
raw_data=None,
unmerged_data=None,
keep_hydrogens=True,
nuclear=False,
save_probe_unformatted_file=None,
show_hydrogen_outliers=False,
min_cc_two_fofc=0.8,
n_bins_data=10,
count_anomalous_pairs_separately=False,
use_internal_variance=True,
outliers_only=True,
use_pdb_header_resolution_cutoffs=False,
file_name=None,
ligand_selection=None,
rotamer_library="8000",
map_params=None):
assert rotamer_library == "8000", "data_version given to RotamerEval not recognized."
for name in self.__slots__:
setattr(self, name, None)
# use objects from model
self.model = model
if (self.model is not None):
pdb_hierarchy = self.model.get_hierarchy()
xray_structure = self.model.get_xray_structure()
geometry_restraints_manager = self.model.get_restraints_manager(
).geometry
crystal_symmetry = self.model.crystal_symmetry()
all_chain_proxies = self.model.all_chain_proxies
else:
assert (pdb_hierarchy is not None)
xray_structure = None
geometry_restraints_manager = None
crystal_symmetry = None
all_chain_proxies = None
# very important - the i_seq attributes may be extracted later
pdb_hierarchy.atoms().reset_i_seq()
self.pdb_hierarchy = pdb_hierarchy
if (xray_structure is None):
if (fmodel is not None):
xray_structure = fmodel.xray_structure
elif (crystal_symmetry is not None):
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry=crystal_symmetry)
self.crystal_symmetry = crystal_symmetry
if (crystal_symmetry is None) and (fmodel is not None):
self.crystal_symmetry = fmodel.f_obs().crystal_symmetry()
# use maps (fmodel is not used)
# run earlier since pdb_hierarchy gets modified
use_maps = False
if (map_params is not None):
use_maps = ((map_params.input.maps.map_file_name) or
((map_params.input.maps.map_coefficients_file_name) and
(map_params.input.maps.map_coefficients_label)))
if (use_maps):
if (flags.real_space):
self.real_space = experimental.real_space(
fmodel=None,
model=self.model,
cc_min=min_cc_two_fofc,
molprobity_map_params=map_params.input.maps)
if (flags.waters):
self.waters = waters.waters(
pdb_hierarchy=pdb_hierarchy,
xray_structure=xray_structure,
fmodel=None,
collect_all=True,
molprobity_map_params=map_params.input.maps)
self.header_info = header_info
if (flags is None):
flags = molprobity_flags()
import mmtbx.model.statistics
self.model_statistics_geometry = mmtbx.model.statistics.geometry(
pdb_hierarchy=pdb_hierarchy,
geometry_restraints_manager=geometry_restraints_manager,
use_hydrogens=keep_hydrogens,
use_nuclear=nuclear)
self.model_statistics_geometry_result = \
self.model_statistics_geometry.result()
self.ramalyze = self.model_statistics_geometry_result.ramachandran.ramalyze
self.omegalyze = self.model_statistics_geometry_result.omega.omegalyze
self.rotalyze = self.model_statistics_geometry_result.rotamer.rotalyze
self.cbetadev = self.model_statistics_geometry_result.c_beta.cbetadev
self.clashes = self.model_statistics_geometry_result.clash.clashes
if pdb_hierarchy.contains_protein():
self.find_missing_atoms(out=null_out())
if (flags.nqh):
self.nqh_flips = clashscore.nqh_flips(
pdb_hierarchy=pdb_hierarchy)
if (pdb_hierarchy.contains_rna() and flags.rna
and libtbx.env.has_module(name="suitename")):
if (geometry_restraints_manager is not None):
self.rna = rna_validate.rna_validation(
pdb_hierarchy=pdb_hierarchy,
geometry_restraints_manager=geometry_restraints_manager,
outliers_only=outliers_only,
params=None)
if (flags.model_stats) and (xray_structure is not None):
self.model_stats = model_properties.model_statistics(
pdb_hierarchy=pdb_hierarchy,
xray_structure=xray_structure,
all_chain_proxies=all_chain_proxies,
ignore_hd=(not nuclear),
ligand_selection=ligand_selection)
if (geometry_restraints_manager is not None) and (flags.restraints):
assert (xray_structure is not None)
self.restraints = restraints.combined(
pdb_hierarchy=pdb_hierarchy,
xray_structure=xray_structure,
geometry_restraints_manager=geometry_restraints_manager,
ignore_hd=(not nuclear),
cdl=getattr(all_chain_proxies, "use_cdl", None))
if (sequences is not None) and (flags.seq):
self.sequence = sequence.validation(
pdb_hierarchy=pdb_hierarchy,
sequences=sequences,
log=null_out(),
include_secondary_structure=True,
extract_coordinates=True)
if (fmodel is not None):
if (use_pdb_header_resolution_cutoffs) and (header_info
is not None):
fmodel = fmodel.resolution_filter(d_min=header_info.d_min,
d_max=header_info.d_max)
if (flags.rfactors):
self.data_stats = experimental.data_statistics(
fmodel,
raw_data=raw_data,
n_bins=n_bins_data,
count_anomalous_pairs_separately=
count_anomalous_pairs_separately)
if (not use_maps): # if maps are used, keep previous results
if (flags.real_space):
self.real_space = experimental.real_space(
model=model, fmodel=fmodel, cc_min=min_cc_two_fofc)
if (flags.waters):
self.waters = waters.waters(pdb_hierarchy=pdb_hierarchy,
xray_structure=xray_structure,
fmodel=fmodel,
collect_all=True)
if (unmerged_data is not None):
self.merging = experimental.merging_and_model_statistics(
f_obs=fmodel.f_obs(),
f_model=fmodel.f_model(),
r_free_flags=fmodel.r_free_flags(),
unmerged_i_obs=unmerged_data,
anomalous=count_anomalous_pairs_separately,
use_internal_variance=use_internal_variance,
n_bins=n_bins_data)
if (flags.xtriage):
import mmtbx.scaling.xtriage
f_model = abs(
fmodel.f_model()).set_observation_type_xray_amplitude()
if (raw_data is not None):
f_model, obs = f_model.common_sets(other=raw_data)
else:
obs = fmodel.f_obs()
self.xtriage = mmtbx.scaling.xtriage.xtriage_analyses(
miller_obs=obs,
miller_calc=f_model,
unmerged_obs=unmerged_data, # XXX some redundancy here...
text_out=null_out())
if (fmodel_neutron is not None) and (flags.rfactors):
self.neutron_stats = experimental.data_statistics(
fmodel_neutron,
n_bins=n_bins_data,
count_anomalous_pairs_separately=False)
if (pdb_hierarchy.models_size() == 1):
self._multi_criterion = multi_criterion_view(pdb_hierarchy)
# wilson B
self.wilson_b = None
if (fmodel is not None):
self.wilson_b = fmodel.wilson_b()
elif (fmodel_neutron is not None):
self.wilson_b = fmodel_neutron.wilson_b()
# validate hydrogens
self.hydrogens = None
if self.model is not None and self.model.has_hd():
# import here to avoid circular import issues
from mmtbx.hydrogens.validate_H import validate_H, validate_H_results
hydrogens = validate_H(model, nuclear)
hydrogens.validate_inputs()
hydrogens.run()
self.hydrogens = validate_H_results(hydrogens.get_results())
# write probe file if needed (CLI and GUI)
if (save_probe_unformatted_file is not None):
pcm = self.clashes.probe_clashscore_manager
try:
with open(save_probe_unformatted_file, 'w') as f:
f.write(pcm.probe_unformatted)
self.clashes.probe_file = save_probe_unformatted_file
except IOError as err:
raise Sorry('%s could not be written correctly.\n%s' %
(save_probe_unformatted_file, err))
def run(args):
if len(args) == 0 or '-h' in args or '--help' in args or '-c' in args:
print("Usage: %s [-p] files" % libtbx.env.dispatcher_name)
phil_scope.show(attributes_level=2)
return
files = [arg for arg in args if os.path.isfile(arg)]
arguments = [arg for arg in args if not os.path.isfile(arg)]
user_phil = []
for arg in arguments:
if arg == '-p':
user_phil.append(parse("show_plots=True"))
else:
try:
user_phil.append(parse(arg))
except Exception as e:
raise Sorry("Unrecognized argument: %s" % arg)
params = phil_scope.fetch(sources=user_phil).extract()
for file in files:
message = """Based on the file %s,
this program will compute incremental quadrant translations to circularize
powder rings on the inner four sensors. The algorithm treats each quadrant
independently and scores based on self-correlation upon 45-degree rotation.
""" % file
print(message)
image = dxtbx.load(file)
detector = image.get_detector()
beam = image.get_beam()
from xfel.metrology.quadrant import one_panel
ccs = flex.double()
for i_quad, quad in enumerate(detector.hierarchy()):
# find panel closest to the beam center
panels = []
def recursive_get_panels(group):
if group.is_group():
for child in group:
recursive_get_panels(child)
else:
panels.append(group)
recursive_get_panels(quad)
smallest_dist = float("inf")
for panel in panels:
p_w, p_h = panel.get_image_size()
c = center([
col(panel.get_pixel_lab_coord((0, 0))),
col(panel.get_pixel_lab_coord((p_w - 1, 0))),
col(panel.get_pixel_lab_coord((p_w - 1, p_h - 1))),
col(panel.get_pixel_lab_coord((0, p_h - 1)))
])
beam_center = col(panel.get_beam_centre_lab(beam.get_s0()))
dist = (c - beam_center).length()
if dist < smallest_dist:
smallest_dist = dist
key_panel = panel
print("Doing cross-correlation on panel", key_panel.get_name())
Q = one_panel(image, key_panel, i_quad, quad, params.show_plots,
params.multi_angle, params.plot_range,
params.pdf_file is None)
delta = panel.pixel_to_millimeter((Q.coordmax[0], Q.coordmax[1]))
quad.set_frame(
quad.get_fast_axis(), quad.get_slow_axis(),
col(quad.get_origin()) - col((delta[0], delta[1], 0)))
ccs.append(Q.ccmax)
print("Average CC: %7.4f" % flex.mean(ccs))
if params.pdf_file is not None:
print("Saving plots to", params.pdf_file)
from matplotlib import pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
pp = PdfPages(params.pdf_file)
for i in plt.get_fignums():
pp.savefig(plt.figure(i), dpi=300)
pp.close()
if params.save_cbf:
import pycbf
image.sync_detector_to_cbf()
dest_path = os.path.splitext(file)[0] + "_cc.cbf"
print("Saving result to", dest_path)
image._cbf_handle.write_widefile(dest_path,pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K,0)
def __init__(self,
i_obs,
crystal_symmetry=None,
d_min=None,
d_max=None,
anomalous=False,
n_bins=10,
binning_method='volume',
debug=False,
file_name=None,
model_arrays=None,
sigma_filtering=Auto,
use_internal_variance=True,
eliminate_sys_absent=True,
d_min_tolerance=1.e-6,
extend_d_max_min=False,
cc_one_half_significance_level=None,
cc_one_half_method='half_dataset',
assert_is_not_unique_set_under_symmetry=True,
log=None):
self.file_name = file_name
if (log is None) : log = null_out()
assert (i_obs.sigmas() is not None)
info = i_obs.info()
sigma_filtering = get_filtering_convention(i_obs, sigma_filtering)
if (crystal_symmetry is None):
assert (i_obs.space_group() is not None)
crystal_symmetry = i_obs.crystal_symmetry()
self.crystal_symmetry = crystal_symmetry
i_obs = i_obs.customized_copy(
crystal_symmetry=crystal_symmetry).set_info(info)
if (assert_is_not_unique_set_under_symmetry and i_obs.is_unique_set_under_symmetry()):
raise Sorry(("The data in %s are already merged. Only unmerged (but "+
"scaled) data may be used in this program.")%
i_obs.info().label_string())
d_min_cutoff = d_min
d_max_cutoff = d_max
if (d_min is not None):
d_min_cutoff *= (1-d_min_tolerance)
if (d_max is not None):
assert (d_max > d_min)
if (d_max is not None):
d_max_cutoff *= 1+d_min_tolerance
i_obs = i_obs.resolution_filter(
d_min=d_min_cutoff,
d_max=d_max_cutoff).set_info(info)
if (i_obs.size() == 0):
raise Sorry("No reflections left after applying resolution cutoffs.")
i_obs.show_summary(f=log)
self.anom_extra = ""
if (not anomalous):
i_obs = i_obs.customized_copy(anomalous_flag=False).set_info(info)
self.anom_extra = " (non-anomalous)"
overall_d_max_min = None
# eliminate_sys_absent() before setting up binner to ensure consistency
# between reported overall d_min/max and d_min/max for resolution bins"
if eliminate_sys_absent:
i_obs = i_obs.eliminate_sys_absent()
if extend_d_max_min :
i_obs.setup_binner(
n_bins=n_bins,
d_max=d_max_cutoff,
d_min=d_min_cutoff)
overall_d_max_min = d_max_cutoff, d_min_cutoff
else :
if binning_method == 'volume':
i_obs.setup_binner(n_bins=n_bins)
elif binning_method == 'counting_sorted':
i_obs.setup_binner_counting_sorted(n_bins=n_bins)
self.overall = merging_stats(i_obs,
d_max_min=overall_d_max_min,
model_arrays=model_arrays,
anomalous=anomalous,
debug=debug,
sigma_filtering=sigma_filtering,
use_internal_variance=use_internal_variance,
cc_one_half_significance_level=cc_one_half_significance_level,
cc_one_half_method=cc_one_half_method)
self.bins = []
title = "Intensity merging statistics"
column_labels = ["1/d**2","N(obs)","N(unique)","Redundancy","Completeness",
"Mean(I)", "Mean(I/sigma)", "R-merge", "R-meas", "R-pim", "CC1/2",
"CC(anom)"]
graph_names = ["Reflection counts", "Redundancy", "Completeness",
"Mean(I)", "Mean(I/sigma)", "R-factors", "CC1/2", "CC(anom)"]
graph_columns = [[0,1,2],[0,3],[0,4],[0,5],[0,6],[0,7,8,9],[0,10],[0,13]]
if cc_one_half_significance_level is not None:
column_labels.extend(["CC1/2 significance", "CC1/2 critical value"])
graph_names.extend(["CC1/2 significance", "CC1/2 critical value"])
graph_columns[-2] = [0,10,12]
#--- CC* mode
if (model_arrays is not None):
title = "Model quality and intensity merging statistics"
column_labels.extend(["CC*", "CC(work)", "CC(free)", "R-work", "R-free"])
graph_names.extend(["CC*", "Model R-factors"])
graph_columns.extend([[0,11,14,15],[0,16,17]])
#---
self.table = data_plots.table_data(
title=title,
column_labels=column_labels,
graph_names=graph_names,
graph_columns=graph_columns,
x_is_inverse_d_min=True,
force_exact_x_labels=True)
last_bin = None
for bin in i_obs.binner().range_used():
sele_unmerged = i_obs.binner().selection(bin)
bin_stats = merging_stats(i_obs.select(sele_unmerged),
d_max_min=i_obs.binner().bin_d_range(bin),
model_arrays=model_arrays,
anomalous=anomalous,
debug=debug,
sigma_filtering=sigma_filtering,
use_internal_variance=use_internal_variance,
cc_one_half_significance_level=cc_one_half_significance_level,
cc_one_half_method=cc_one_half_method)
self.bins.append(bin_stats)
self.table.add_row(bin_stats.table_data())
from scitbx.array_family import flex
self.cc_one_half_overall = flex.mean_weighted(
flex.double(b.cc_one_half for b in self.bins),
flex.double(b.cc_one_half_n_refl for b in self.bins))
self.cc_one_half_sigma_tau_overall = flex.mean_weighted(
flex.double(b.cc_one_half_sigma_tau for b in self.bins),
flex.double(b.cc_one_half_sigma_tau_n_refl for b in self.bins))
def run(args,
crystal_symmetry=None,
ncs_object=None,
pdb_hierarchy=None,
map_data=None,
half_map_data_list=None,
half_map_labels_list=None,
lower_bounds=None,
upper_bounds=None,
write_output_files=True,
log=None):
h = "phenix.map_box: extract box with model and map around selected atoms"
if (log is None): log = sys.stdout
print_statistics.make_header(h, out=log)
default_message = """\
%s.
Usage:
phenix.map_box model.pdb map_coefficients.mtz selection="chain A and resseq 1:10"
or
phenix.map_box map.ccp4 density_select=True
Parameters:""" % h
if (len(args) == 0 and not pdb_hierarchy):
print(default_message)
master_phil.show(prefix=" ")
return
# Process inputs ignoring symmetry conflicts just to get the value of
# ignore_symmetry_conflicts...
inputs = mmtbx.utils.process_command_line_args(
args=args,
cmd_cs=crystal_symmetry,
master_params=master_phil,
suppress_symmetry_related_errors=True)
params = inputs.params.extract()
# Now process inputs for real and write a nice error message if necessary.
try:
inputs = mmtbx.utils.process_command_line_args(
args=args,
cmd_cs=crystal_symmetry,
master_params=master_phil,
suppress_symmetry_related_errors=params.ignore_symmetry_conflicts)
except Exception as e:
if str(e).find("symmetry mismatch ") > 1:
raise Sorry(str(e) + "\nTry 'ignore_symmetry_conflicts=True'")
else:
raise e
params = inputs.params.extract()
master_phil.format(python_object=params).show(out=log)
# Overwrite params with parameters in call if available
if lower_bounds:
params.lower_bounds = lower_bounds
if upper_bounds:
params.upper_bounds = upper_bounds
# PDB file
if params.pdb_file and not inputs.pdb_file_names and not pdb_hierarchy:
inputs.pdb_file_names = [params.pdb_file]
if (len(inputs.pdb_file_names) != 1 and not params.density_select
and not params.mask_select and not pdb_hierarchy
and not params.keep_map_size and not params.upper_bounds
and not params.extract_unique
and not params.bounds_match_this_file):
raise Sorry(
"PDB file is needed unless extract_unique, " +
"density_select, mask_select, keep_map_size \nor bounds are set .")
if (len(inputs.pdb_file_names)!=1 and not pdb_hierarchy and \
(params.mask_atoms )):
raise Sorry("PDB file is needed for mask_atoms")
if params.soft_mask and (not params.resolution) and \
(len(inputs.pdb_file_names)!=1 and not pdb_hierarchy):
raise Sorry("Need resolution for soft_mask without PDB file")
if ((params.density_select or params.mask_select)
and params.keep_map_size):
raise Sorry(
"Cannot set both density_select/mask_select and keep_map_size")
if ((params.density_select or params.mask_select) and params.upper_bounds):
raise Sorry("Cannot set both density_select/mask_select and bounds")
if (params.keep_map_size and params.upper_bounds):
raise Sorry("Cannot set both keep_map_size and bounds")
if (params.upper_bounds and not params.lower_bounds):
raise Sorry("Please set lower_bounds if you set upper_bounds")
if (params.extract_unique):
if (not params.resolution):
raise Sorry("Please set resolution for extract_unique")
if (not params.symmetry) and (not params.symmetry_file) and \
(not ncs_object):
raise Sorry(
"Please supply a symmetry file or symmetry for extract_unique (you "
+
"\ncan try symmetry=ALL if you do not know your symmetry or " +
"symmetry=C1 if \nthere is none)")
from mmtbx.ncs.ncs import ncs
ncs_object = ncs()
ncs_object.set_unit_ncs()
if params.keep_input_unit_cell_and_grid and (
(params.output_unit_cell_grid is not None) or
(params.output_unit_cell is not None)):
raise Sorry("If you set keep_input_unit_cell_and_grid then you cannot "+\
"set \noutput_unit_cell_grid or output_unit_cell")
if (write_output_files) and ("mtz" in params.output_format) and (
(params.keep_origin) and (not params.keep_map_size)):
print("\nNOTE: Skipping write of mtz file as keep_origin=True and \n"+\
"keep_map_size is False\n")
params.output_format = remove_element(params.output_format,
element='mtz')
if (write_output_files) and ("mtz" in params.output_format) and (
(params.extract_unique)):
print("\nNOTE: Skipping write of mtz file as extract_unique=True\n")
params.output_format = remove_element(params.output_format,
element='mtz')
if params.output_origin_match_this_file or params.bounds_match_this_file:
if params.output_origin_match_this_file:
fn = params.output_origin_match_this_file
if params.bounds_match_this_file:
raise Sorry("Cannot match origin and bounds at same time")
else:
fn = params.bounds_match_this_file
if not params.ccp4_map_file:
raise Sorry(
"Need to specify your input file with ccp4_map_file=xxx if you use "
+
"output_origin_match_this_file=xxxx or bounds_match_this_file=xxxx"
)
af = any_file(fn)
if (af.file_type == 'ccp4_map'):
origin = af.file_content.data.origin()
if params.output_origin_match_this_file:
params.output_origin_grid_units = origin
print("Origin of (%s,%s,%s) taken from %s" %
(origin[0], origin[1], origin[2], fn))
else:
all = af.file_content.data.all()
params.lower_bounds = origin
print("Lower bounds of (%s,%s,%s) taken from %s" %
(params.lower_bounds[0], params.lower_bounds[1],
params.lower_bounds[2], fn))
params.upper_bounds = list(
col(origin) + col(all) - col((1, 1, 1)))
print("upper bounds of (%s,%s,%s) taken from %s" %
(params.upper_bounds[0], params.upper_bounds[1],
params.upper_bounds[2], fn))
params.bounds_are_absolute = True
else:
raise Sorry("Unable to interpret %s as map file" % (fn))
if params.output_origin_grid_units is not None and params.keep_origin:
params.keep_origin = False
print("Setting keep_origin=False as output_origin_grid_units is set")
print_statistics.make_sub_header("pdb model", out=log)
if len(inputs.pdb_file_names) > 0:
pdb_inp = iotbx.pdb.input(file_name=inputs.pdb_file_names[0])
pdb_hierarchy = pdb_inp.construct_hierarchy()
if pdb_hierarchy:
pdb_atoms = pdb_hierarchy.atoms()
pdb_atoms.reset_i_seq()
else:
pdb_hierarchy = None
# Map or map coefficients
map_coeff = None
input_unit_cell_grid = None
input_unit_cell = None
input_map_labels = None
if (not map_data):
# read first mtz file
if ((len(inputs.reflection_file_names) > 0)
or (params.map_coefficients_file is not None)):
# file in phil takes precedent
if (params.map_coefficients_file is not None):
if (len(inputs.reflection_file_names) == 0):
inputs.reflection_file_names.append(
params.map_coefficients_file)
else:
inputs.reflection_file_names[
0] = params.map_coefficients_file
map_coeff = reflection_file_utils.extract_miller_array_from_file(
file_name=inputs.reflection_file_names[0],
label=params.label,
type="complex",
log=log)
if not crystal_symmetry:
crystal_symmetry = map_coeff.crystal_symmetry()
fft_map = map_coeff.fft_map(
resolution_factor=params.resolution_factor)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
map_or_map_coeffs_prefix = os.path.basename(
inputs.reflection_file_names[0][:-4])
# or read CCP4 map
elif ((inputs.ccp4_map is not None)
or (params.ccp4_map_file is not None)):
if (params.ccp4_map_file is not None):
af = any_file(params.ccp4_map_file)
if (af.file_type == 'ccp4_map'):
inputs.ccp4_map = af.file_content
inputs.ccp4_map_file_name = params.ccp4_map_file
print_statistics.make_sub_header("CCP4 map", out=log)
ccp4_map = inputs.ccp4_map
ccp4_map.show_summary(prefix=" ", out=log)
if not crystal_symmetry:
crystal_symmetry = ccp4_map.crystal_symmetry()
map_data = ccp4_map.data #map_data()
input_unit_cell_grid = ccp4_map.unit_cell_grid
input_unit_cell = ccp4_map.unit_cell_parameters
input_map_labels = ccp4_map.get_labels()
if inputs.ccp4_map_file_name.endswith(".ccp4"):
map_or_map_coeffs_prefix = os.path.basename(
inputs.ccp4_map_file_name[:-5])
else:
map_or_map_coeffs_prefix = os.path.basename(
inputs.ccp4_map_file_name[:-4])
else: # have map_data
map_or_map_coeffs_prefix = None
if params.half_map_list and (not half_map_data_list):
if not params.extract_unique:
raise Sorry("Can only use half_map_with extract_unique")
print("Reading half-maps", params.half_map_list)
half_map_data_list = []
half_map_labels_list = []
for fn in params.half_map_list:
print("Reading half map from %s" % (fn), file=log)
af = any_file(fn)
print_statistics.make_sub_header("CCP4 map", out=log)
h_ccp4_map = af.file_content
h_ccp4_map.show_summary(prefix=" ", out=log)
h_map_data = h_ccp4_map.data
half_map_data_list.append(h_map_data)
half_map_labels_list.append(h_ccp4_map.get_labels())
if params.map_scale_factor:
print("Applying scale factor of %s to map data on read-in" %
(params.map_scale_factor))
map_data = map_data * params.map_scale_factor
if params.output_origin_grid_units is not None:
origin_to_match = tuple(params.output_origin_grid_units)
else:
origin_to_match = None
if origin_to_match:
sc = []
for x, o, a in zip(crystal_symmetry.unit_cell().parameters()[:3],
origin_to_match, map_data.all()):
sc.append(-x * o / a)
shift_cart_for_origin_to_match = tuple(sc)
else:
origin_to_match = None
shift_cart_for_origin_to_match = None
if crystal_symmetry and not inputs.crystal_symmetry:
inputs.crystal_symmetry = crystal_symmetry
# final check that map_data exists
if (map_data is None):
raise Sorry("Map or map coefficients file is needed.")
if len(inputs.pdb_file_names) > 0:
output_prefix = os.path.basename(inputs.pdb_file_names[0])[:-4]
else:
output_prefix = map_or_map_coeffs_prefix
if not pdb_hierarchy: # get an empty hierarchy
from cctbx.array_family import flex
pdb_hierarchy = iotbx.pdb.input(
source_info='', lines=flex.split_lines('')).construct_hierarchy()
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry=inputs.crystal_symmetry)
xray_structure.show_summary(f=log)
#
if not params.selection: params.selection = "all"
selection = pdb_hierarchy.atom_selection_cache().selection(
string=params.selection)
if selection.size():
print_statistics.make_sub_header("atom selection", out=log)
print("Selection string: selection='%s'" % params.selection, file=log)
print(" selects %d atoms from total %d atoms." %
(selection.count(True), selection.size()),
file=log)
sites_cart_all = xray_structure.sites_cart()
sites_cart = sites_cart_all.select(selection)
selection = xray_structure.selection_within(radius=params.selection_radius,
selection=selection)
if not ncs_object:
from mmtbx.ncs.ncs import ncs
ncs_object = ncs()
if params.symmetry_file:
ncs_object.read_ncs(params.symmetry_file, log=log)
print("Total of %s operators read" % (ncs_object.max_operators()),
file=log)
if not ncs_object or ncs_object.max_operators() < 1:
print("No symmetry available", file=log)
if ncs_object:
n_ops = max(1, ncs_object.max_operators())
else:
n_ops = 1
# Get sequence if extract_unique is set
sequence = None
if params.extract_unique or params.mask_select:
if params.sequence_file:
if n_ops > 1: # get unique part of sequence
remove_duplicates = True
else:
remove_duplicates = False
from iotbx.bioinformatics import get_sequences
sequence = (" ".join(
get_sequences(file_name=params.sequence_file,
remove_duplicates=remove_duplicates)))
if params.chain_type in ['None', None]: params.chain_type = None
if sequence and not params.molecular_mass:
# get molecular mass from sequence
from iotbx.bioinformatics import text_from_chains_matching_chain_type
if params.chain_type in [None, 'PROTEIN']:
n_protein = len(
text_from_chains_matching_chain_type(text=sequence,
chain_type='PROTEIN'))
else:
n_protein = 0
if params.chain_type in [None, 'RNA']:
n_rna = len(
text_from_chains_matching_chain_type(text=sequence,
chain_type='RNA'))
else:
n_rna = 0
if params.chain_type in [None, 'DNA']:
n_dna = len(
text_from_chains_matching_chain_type(text=sequence,
chain_type='DNA'))
else:
n_dna = 0
params.molecular_mass = n_ops * (n_protein * 110 +
(n_rna + n_dna) * 330)
print("\nEstimate of molecular mass is %.0f " %
(params.molecular_mass),
file=log)
if params.density_select or params.mask_select:
print_statistics.make_sub_header(
"Extracting box around selected density and writing output files",
out=log)
else:
print_statistics.make_sub_header(
"Extracting box around selected atoms and writing output files",
out=log)
#
if params.value_outside_atoms == 'mean':
print("\nValue outside atoms mask will be set to mean inside mask",
file=log)
if params.get_half_height_width and params.density_select:
print("\nHalf width at half height will be used to id boundaries",
file=log)
if params.soft_mask and sites_cart_all.size() > 0:
print("\nSoft mask will be applied to model-based mask", file=log)
elif params.soft_mask:
print("\nSoft mask will be applied to outside of map box", file=log)
if params.keep_map_size:
print("\nEntire map will be kept (not cutting out region)", file=log)
if params.restrict_map_size:
print("\nOutput map will be within input map", file=log)
if params.lower_bounds and params.upper_bounds:
print("Bounds for cut out map are (%s,%s,%s) to (%s,%s,%s)" %
(tuple(list(params.lower_bounds) + list(params.upper_bounds))),
file=log)
box = mmtbx.utils.extract_box_around_model_and_map(
xray_structure=xray_structure,
map_data=map_data.as_double(),
box_cushion=params.box_cushion,
selection=selection,
mask_select=params.mask_select,
density_select=params.density_select,
threshold=params.density_select_threshold,
get_half_height_width=params.get_half_height_width,
mask_atoms=params.mask_atoms,
soft_mask=params.soft_mask,
soft_mask_radius=params.soft_mask_radius,
mask_atoms_atom_radius=params.mask_atoms_atom_radius,
value_outside_atoms=params.value_outside_atoms,
keep_map_size=params.keep_map_size,
restrict_map_size=params.restrict_map_size,
lower_bounds=params.lower_bounds,
upper_bounds=params.upper_bounds,
bounds_are_absolute=params.bounds_are_absolute,
zero_outside_original_map=params.zero_outside_original_map,
extract_unique=params.extract_unique,
target_ncs_au_file=params.target_ncs_au_file,
regions_to_keep=params.regions_to_keep,
box_buffer=params.box_buffer,
soft_mask_extract_unique=params.soft_mask_extract_unique,
mask_expand_ratio=params.mask_expand_ratio,
keep_low_density=params.keep_low_density,
chain_type=params.chain_type,
sequence=sequence,
solvent_content=params.solvent_content,
molecular_mass=params.molecular_mass,
resolution=params.resolution,
ncs_object=ncs_object,
symmetry=params.symmetry,
half_map_data_list=half_map_data_list,
)
ph_box = pdb_hierarchy.select(selection)
ph_box.adopt_xray_structure(box.xray_structure_box)
box.hierarchy = ph_box
if params.mask_select:
print("\nSolvent content used in mask_select: %.3f " %
(box.get_solvent_content()),
file=log)
if (inputs and inputs.crystal_symmetry and inputs.ccp4_map
and inputs.crystal_symmetry.unit_cell().parameters()
and inputs.ccp4_map.unit_cell_parameters) and (
inputs.crystal_symmetry.unit_cell().parameters() !=
inputs.ccp4_map.unit_cell_parameters):
print("\nNOTE: Input CCP4 map is only part of unit cell:", file=log)
print("Full unit cell ('unit cell parameters'): "+\
"(%.1f, %.1f, %.1f, %.1f, %.1f, %.1f) A" %tuple(
inputs.ccp4_map.unit_cell_parameters), file=log)
print("Size of CCP4 map 'map unit cell': "+\
"(%.1f, %.1f, %.1f, %.1f, %.1f, %.1f) A" %tuple(
inputs.crystal_symmetry.unit_cell().parameters()), file=log)
print("Full unit cell as grid units: (%s, %s, %s)" %
(inputs.ccp4_map.unit_cell_grid),
file=log)
print("Map unit cell as grid units: (%s, %s, %s)" % (map_data.all()),
file=log)
box.unit_cell_parameters_from_ccp4_map = inputs.ccp4_map.unit_cell_parameters
box.unit_cell_parameters_deduced_from_map_grid=\
inputs.crystal_symmetry.unit_cell().parameters()
else:
box.unit_cell_parameters_from_ccp4_map = None
box.unit_cell_parameters_deduced_from_map_grid = None
if box.pdb_outside_box_msg:
print(box.pdb_outside_box_msg, file=log)
# NOTE: box object is always shifted to place origin at (0,0,0)
# NOTE ON ORIGIN SHIFTS: The shifts are applied locally here. The box
# object is not affected and always has the origin at (0,0,0)
# output_box is copy of box with shift_cart corresponding to the output
# files. Normally this is the same as the original shift_cart. However
# if user has specified a new output origin it will differ.
# For output files ONLY:
# keep_origin==False leave origin at (0,0,0)
# keep_origin==True: we shift everything back to where it was,
# output_origin_grid_units=10,10,10: output origin is at (10,10,10)
# ncs_object is original
# box.ncs_object is shifted by shift_cart
# output_box.ncs_object is shifted back by -new shift_cart
# Additional note on output unit_cell and grid_units.
# The ccp4-style output map can specify the unit cell and grid units
# corresponding to that cell. This can be separate from the origin and
# number of grid points in the map as written. If specified, write these
# out to the output ccp4 map and also use this unit cell for writing
# any output PDB files
from copy import deepcopy
output_box = deepcopy(box) # won't use box below here except to return it
print("\nBox cell dimensions: (%.2f, %.2f, %.2f) A" %
(box.box_crystal_symmetry.unit_cell().parameters()[:3]),
file=log)
if box.shift_cart:
print("Working origin moved from grid position of"+\
": (%d, %d, %d) to (0,0,0) " %(
tuple(box.origin_shift_grid_units(reverse=True))), file=log)
print("Working origin moved from coordinates of:"+\
" (%.2f, %.2f, %.2f) A to (0,0,0)\n" %(
tuple(-col(box.shift_cart))), file=log)
if (params.keep_origin):
print("\nRestoring original position for output files", file=log)
print("Origin will be at grid position of"+\
": (%d, %d, %d) " %(
tuple(box.origin_shift_grid_units(reverse=True))), file=log)
print("\nOutput files will be in same location as original",
end=' ',
file=log)
if not params.keep_map_size:
print("just cut out.", file=log)
else:
print("keeping entire map", file=log)
print("Note that output maps are only valid in the cut out region.\n",
file=log)
else:
if origin_to_match:
output_box.shift_cart = shift_cart_for_origin_to_match
if params.output_origin_grid_units:
print("Output map origin to be shifted to match target",
file=log)
print("Placing origin at grid point (%s, %s, %s)" %(
origin_to_match)+"\n"+ \
"Final coordinate shift for output files: (%.2f,%.2f,%.2f) A\n" %(
tuple(col(output_box.shift_cart)-col(box.shift_cart))), file=log)
elif box.shift_cart:
output_box.shift_cart = (0, 0, 0) # not shifting back
print("Final origin will be at (0,0,0)", file=log)
print(
"Final coordinate shift for output files: (%.2f,%.2f,%.2f) A\n"
% (tuple(col(output_box.shift_cart) - col(box.shift_cart))),
file=log)
else:
print("\nOutput files are in same location as original and origin "+\
"is at (0,0,0)\n", file=log)
print("\nBox grid: (%s, %s, %s) " % (output_box.map_box.all()))
ph_output_box_output_location = ph_box.deep_copy()
if output_box.shift_cart: # shift coordinates and NCS back by shift_cart
# NOTE output_box.shift_cart could be different than box.shift_cart if
# there is a target position for the origin and it is not the same as the
# original origin.
sites_cart = output_box.shift_sites_cart_back(
output_box.xray_structure_box.sites_cart())
xrs_offset = ph_output_box_output_location.extract_xray_structure(
crystal_symmetry=output_box.xray_structure_box.crystal_symmetry(
)).replace_sites_cart(new_sites=sites_cart)
ph_output_box_output_location.adopt_xray_structure(xrs_offset)
if output_box.ncs_object:
output_box.ncs_object = output_box.ncs_object.coordinate_offset(
tuple(-col(output_box.shift_cart)))
shift_back = True
else:
shift_back = False
if params.keep_input_unit_cell_and_grid and \
(input_unit_cell_grid is not None) and \
(input_unit_cell is not None):
params.output_unit_cell = input_unit_cell
params.output_unit_cell_grid = input_unit_cell_grid
print("Setting output unit cell parameters and unit cell grid to"+\
" match\ninput map file", file=log)
if params.output_unit_cell: # Set output unit cell parameters
from cctbx import crystal
output_crystal_symmetry = crystal.symmetry(
unit_cell=params.output_unit_cell, space_group="P1")
output_unit_cell = output_crystal_symmetry.unit_cell()
print("Output unit cell set to: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f)" %
tuple(output_crystal_symmetry.unit_cell().parameters()),
file=log)
else:
output_crystal_symmetry = None
# ============= Check/set output unit cell grid and cell parameters =======
if params.output_unit_cell_grid or output_crystal_symmetry:
if params.output_unit_cell_grid:
output_unit_cell_grid = params.output_unit_cell_grid
else:
output_unit_cell_grid = output_box.map_box.all()
print("Output unit cell grid set to: (%s, %s, %s)" %
tuple(output_unit_cell_grid),
file=log)
expected_output_abc = []
box_spacing = []
output_spacing = []
box_abc=output_box.xray_structure_box.\
crystal_symmetry().unit_cell().parameters()[:3]
if output_crystal_symmetry:
output_abc = output_crystal_symmetry.unit_cell().parameters()[:3]
else:
output_abc = [None, None, None]
for a_box, a_output, n_box, n_output in zip(box_abc, output_abc,
output_box.map_box.all(),
output_unit_cell_grid):
expected_output_abc.append(a_box * n_output / n_box)
box_spacing.append(a_box / n_box)
if output_crystal_symmetry:
output_spacing.append(a_output / n_output)
else:
output_spacing.append(a_box / n_box)
if output_crystal_symmetry: # make sure it is compatible...
r0 = expected_output_abc[0] / output_abc[0]
r1 = expected_output_abc[1] / output_abc[1]
r2 = expected_output_abc[2] / output_abc[2]
from libtbx.test_utils import approx_equal
if not approx_equal(r0, r1, eps=0.001) or not approx_equal(
r0, r2, eps=0.001):
print("WARNING: output_unit_cell and cell_grid will "+\
"change ratio of grid spacing.\nOld spacings: "+\
"(%.2f, %.2f, %.2f) A " %(tuple(box_spacing))+\
"\nNew spacings: (%.2f, %.2f, %.2f) A \n" %(tuple(output_spacing)), file=log)
else:
output_abc = expected_output_abc
from cctbx import crystal
output_crystal_symmetry = crystal.symmetry(unit_cell=list(output_abc) +
[90, 90, 90],
space_group="P1")
print(
"Output unit cell will be: (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f)\n"
% (tuple(output_crystal_symmetry.unit_cell().parameters())),
file=log)
else:
output_unit_cell_grid = output_box.map_box.all()
output_crystal_symmetry = output_box.xray_structure_box.crystal_symmetry(
)
# ========== Done check/set output unit cell grid and cell parameters =====
if write_output_files:
# Write PDB file
if ph_box.overall_counts().n_residues > 0:
if (params.output_file_name_prefix is None):
file_name = "%s_box.pdb" % output_prefix
else:
file_name = "%s.pdb" % params.output_file_name_prefix
ph_output_box_output_location.write_pdb_file(
file_name=file_name, crystal_symmetry=output_crystal_symmetry)
print("Writing boxed PDB with box unit cell to %s" % (file_name),
file=log)
# Write NCS file if NCS
if output_box.ncs_object and output_box.ncs_object.max_operators() > 0:
if (params.output_file_name_prefix is None):
output_symmetry_file = "%s_box.ncs_spec" % output_prefix
else:
output_symmetry_file = "%s.ncs_spec" % params.output_file_name_prefix
output_box.ncs_object.format_all_for_group_specification(
file_name=output_symmetry_file)
print("\nWriting symmetry to %s" % (output_symmetry_file),
file=log)
# Write ccp4 map.
if ("ccp4" in params.output_format):
if (params.output_file_name_prefix is None):
file_name = "%s_box.ccp4" % output_prefix
else:
file_name = "%s.ccp4" % params.output_file_name_prefix
from iotbx.mrcfile import create_output_labels
if params.extract_unique:
program_name = 'map_box using extract_unique'
limitations = ["extract_unique"]
else:
program_name = 'map_box'
limitations = []
labels = create_output_labels(
program_name=program_name,
input_file_name=inputs.ccp4_map_file_name,
input_labels=input_map_labels,
limitations=limitations,
output_labels=params.output_map_labels)
output_box.write_ccp4_map(
file_name=file_name,
output_crystal_symmetry=output_crystal_symmetry,
output_mean=params.output_ccp4_map_mean,
output_sd=params.output_ccp4_map_sd,
output_unit_cell_grid=output_unit_cell_grid,
shift_back=shift_back,
output_map_labels=labels,
output_external_origin=params.output_external_origin)
print("Writing boxed map "+\
"to CCP4 formatted file: %s"%file_name, file=log)
if not params.half_map_list:
params.half_map_list = []
if not output_box.map_box_half_map_list:
output_box.map_box_half_map_list = []
if not half_map_labels_list:
half_map_labels_list = len(
output_box.map_box_half_map_list) * [None]
for hm, labels, fn in zip(
output_box.map_box_half_map_list, half_map_labels_list,
params.half_map_list): # half maps matching
labels = create_output_labels(
program_name=program_name,
input_file_name=fn,
input_labels=labels,
limitations=limitations,
output_labels=params.output_map_labels)
hm_fn = "%s_box.ccp4" % (".".join(
os.path.basename(fn).split(".")[:-1]))
output_box.write_ccp4_map(
file_name=hm_fn,
map_data=hm,
output_crystal_symmetry=output_crystal_symmetry,
output_mean=params.output_ccp4_map_mean,
output_sd=params.output_ccp4_map_sd,
output_unit_cell_grid=output_unit_cell_grid,
shift_back=shift_back,
output_map_labels=labels,
output_external_origin=params.output_external_origin)
print("Writing boxed half map to: %s " % (hm_fn), file=log)
# Write xplor map. Shift back if keep_origin=True
if ("xplor" in params.output_format):
if (params.output_file_name_prefix is None):
file_name = "%s_box.xplor" % output_prefix
else:
file_name = "%s.xplor" % params.output_file_name_prefix
output_box.write_xplor_map(
file_name=file_name,
output_crystal_symmetry=output_crystal_symmetry,
output_unit_cell_grid=output_unit_cell_grid,
shift_back=shift_back,
)
print("Writing boxed map "+\
"to X-plor formatted file: %s"%file_name, file=log)
# Write mtz map coeffs. Shift back if keep_origin=True
if ("mtz" in params.output_format):
if (params.output_file_name_prefix is None):
file_name = "%s_box.mtz" % output_prefix
else:
file_name = "%s.mtz" % params.output_file_name_prefix
print("Writing map coefficients "+\
"to MTZ file: %s"%file_name, file=log)
if (map_coeff is not None):
d_min = map_coeff.d_min()
elif params.resolution is not None:
d_min = params.resolution
else:
d_min = maptbx.d_min_from_map(
map_data=output_box.map_box,
unit_cell=output_box.xray_structure_box.unit_cell())
output_box.map_coefficients(
d_min=d_min,
scale_max=params.scale_max,
resolution_factor=params.resolution_factor,
file_name=file_name,
shift_back=shift_back)
print(file=log)
return box
def __init__(self,
pdb_file,
output_file=None,
log=None,
quiet=False,
set_se_occ=True,
remove_atoms_with_zero_occupancy=False):
from iotbx.file_reader import any_file
import iotbx.pdb
if (log is None):
log = null_out()
pdb_in = any_file(pdb_file, force_type="pdb")
pdb_in.assert_file_type("pdb")
hierarchy = pdb_in.file_object.hierarchy
if (len(hierarchy.models()) > 1):
raise Sorry("Multi-MODEL PDB files are not supported.")
n_unknown = 0
all_atoms = hierarchy.atoms()
cache = hierarchy.atom_selection_cache()
# resname UNK is now okay (with some restrictions)
known_sel = cache.selection("not (element X or resname UNX or resname UNL)")
semet_sel = cache.selection("element SE and resname MSE")
zero_occ_sel = all_atoms.extract_occ() == 0
self.n_unknown = known_sel.count(False)
self.n_semet = semet_sel.count(True)
self.n_zero_occ = zero_occ_sel.count(True)
keep_sel = known_sel
modified = False
if ((self.n_unknown > 0) or
((self.n_semet > 0) and (set_se_occ)) or
(self.n_zero_occ > 0) and (remove_atoms_with_zero_occupancy)):
modified = True
if (output_file is None):
output_file = pdb_file
if (self.n_unknown > 0) and (not quiet):
print("Warning: %d unknown atoms or ligands removed:" % \
self.n_unknown, file=log)
for i_seq in (~known_sel).iselection():
print(" %s" % all_atoms[i_seq].id_str(), file=log)
if (self.n_zero_occ > 0):
msg = "Warning: %d atoms with zero occupancy present in structure:"
if (remove_atoms_with_zero_occupancy):
msg = "Warning: %d atoms with zero occupancy removed:"
keep_sel &= ~zero_occ_sel
if (not quiet):
print(msg % self.n_zero_occ, file=log)
for i_seq in zero_occ_sel.iselection():
print(" %s" % all_atoms[i_seq].id_str(), file=log)
hierarchy_filtered = hierarchy.select(keep_sel)
if (self.n_semet > 0) and (set_se_occ):
for atom in hierarchy_filtered.atoms():
if (atom.element == "SE") and (atom.fetch_labels().resname == "MSE"):
if (atom.occ == 1.0):
if (not quiet):
print("Set occupancy of %s to 0.99" % atom.id_str(), file=log)
atom.occ = 0.99 # just enough to trigger occupancy refinement
if (modified):
f = open(output_file, "w")
# if the input file is actually from the PDB, we need to preserve the
# header information for downstream code.
print("\n".join(pdb_in.file_object.input.title_section()), file=f)
print("\n".join(pdb_in.file_object.input.remark_section()), file=f)
print(iotbx.pdb.format_cryst1_record(
crystal_symmetry=pdb_in.file_object.crystal_symmetry()), file=f)
print(hierarchy_filtered.as_pdb_string(), file=f)
f.close()
def __setitem__(self, key, value):
assert isinstance(value, block)
if not re.match(tag_re, '_' + key):
raise Sorry("%s is not a valid data block name" % key)
self.blocks[key] = value
self.keys_lower[key.lower()] = key
def extract(file_name,
crystal_symmetry,
wavelength_id,
crystal_id,
show_details_if_error,
output_r_free_label,
merge_non_unique_under_symmetry,
map_to_asu,
remove_systematic_absences,
all_miller_arrays=None,
incompatible_flags_to_work_set=False,
ignore_bad_sigmas=False,
extend_flags=False,
return_as_miller_arrays=False,
log=sys.stdout):
import iotbx.cif
from cctbx import miller
if all_miller_arrays is None:
base_array_info = miller.array_info(
crystal_symmetry_from_file=crystal_symmetry)
all_miller_arrays = iotbx.cif.reader(
file_path=file_name).build_miller_arrays(
base_array_info=base_array_info)
if (len(all_miller_arrays) == 0):
raise Sorry(
"No data arrays were found in this CIF file. Please make " +
"sure that the file contains reflection data, rather than the refined "
+ "model.")
column_labels = set()
if (extend_flags):
map_to_asu = True
# TODO: is all_mille_arrays a dict ? If not change back
for (data_name, miller_arrays) in six.iteritems(all_miller_arrays):
for ma in miller_arrays.values():
other_symmetry = crystal_symmetry
try:
crystal_symmetry = other_symmetry.join_symmetry(
other_symmetry=ma.crystal_symmetry(), force=True)
except AssertionError as e:
str_e = str(e)
from six.moves import cStringIO as StringIO
s = StringIO()
if "Space group is incompatible with unit cell parameters." in str_e:
other_symmetry.show_summary(f=s)
ma.crystal_symmetry().show_summary(f=s)
str_e += "\n%s" % (s.getvalue())
raise Sorry(str_e)
else:
raise
if (crystal_symmetry.unit_cell() is None
or crystal_symmetry.space_group_info() is None):
raise Sorry(
"Crystal symmetry is not defined. Please use the --symmetry option."
)
mtz_object = iotbx.mtz.object() \
.set_title(title="phenix.cif_as_mtz") \
.set_space_group_info(space_group_info=crystal_symmetry.space_group_info())
unit_cell = crystal_symmetry.unit_cell()
mtz_crystals = {}
mtz_object.set_hkl_base(unit_cell=unit_cell)
from iotbx.reflection_file_utils import cif_status_flags_as_int_r_free_flags
# generate list of all reflections (for checking R-free flags)
from iotbx.reflection_file_utils import make_joined_set
all_arrays = []
for (data_name, miller_arrays) in six.iteritems(all_miller_arrays):
for ma in miller_arrays.values():
all_arrays.append(ma)
complete_set = make_joined_set(all_arrays)
if return_as_miller_arrays:
miller_array_list = []
for i, (data_name,
miller_arrays) in enumerate(six.iteritems(all_miller_arrays)):
for ma in miller_arrays.values():
ma = ma.customized_copy(
crystal_symmetry=crystal_symmetry).set_info(ma.info())
labels = ma.info().labels
label = get_label(miller_array=ma,
output_r_free_label=output_r_free_label)
if label is None:
print("Can't determine output label for %s - skipping." % \
ma.info().label_string(), file=log)
continue
elif label.startswith(output_r_free_label):
ma, _ = cif_status_flags_as_int_r_free_flags(
ma, test_flag_value="f")
if isinstance(ma.data(), flex.double):
data_int = ma.data().iround()
assert data_int.as_double().all_eq(ma.data())
ma = ma.customized_copy(data=data_int).set_info(ma.info())
elif (
(ma.is_xray_amplitude_array() or ma.is_xray_intensity_array())
and isinstance(ma.data(), flex.int)):
ma = ma.customized_copy(data=ma.data().as_double()).set_info(
ma.info())
crys_id = 0
for l in labels:
if 'crystal_id' in l:
crys_id = int(l.split('=')[-1])
break
if crys_id > 0 and crystal_id is None:
label += "%i" % crys_id
if crystal_id is not None and crys_id > 0 and crys_id != crystal_id:
continue
if crys_id not in mtz_crystals:
mtz_crystals[crys_id] = (mtz_object.add_crystal(
name="crystal_%i" % crys_id,
project_name="project",
unit_cell=unit_cell), {})
crystal, datasets = mtz_crystals[crys_id]
w_id = 0
for l in labels:
if 'wavelength_id' in l:
w_id = int(l.split('=')[-1])
break
if wavelength_id is not None and w_id > 0 and w_id != wavelength_id:
continue
if w_id > 1 and wavelength_id is None:
if (label in column_labels):
label += "%i" % w_id
#print "label is", label
if w_id not in datasets:
wavelength = ma.info().wavelength
if (wavelength is None):
wavelength = 0
datasets[w_id] = crystal.add_dataset(name="dataset",
wavelength=wavelength)
dataset = datasets[w_id]
# if all sigmas for an array are set to zero either raise an error, or set sigmas to None
if ma.sigmas() is not None and (ma.sigmas()
== 0).count(False) == 0:
if ignore_bad_sigmas:
print("Warning: bad sigmas, setting sigmas to None.",
file=log)
ma.set_sigmas(None)
else:
raise Sorry("""Bad sigmas: all sigmas are equal to zero.
Add --ignore_bad_sigmas to command arguments to leave out sigmas from mtz file."""
)
if not ma.is_unique_set_under_symmetry():
if merge_non_unique_under_symmetry:
print("Warning: merging non-unique data", file=log)
if (label.startswith(output_r_free_label)
and incompatible_flags_to_work_set):
merging = ma.merge_equivalents(
incompatible_flags_replacement=0)
if merging.n_incompatible_flags > 0:
print("Warning: %i reflections were placed in the working set " \
"because of incompatible flags between equivalents." %(
merging.n_incompatible_flags), file=log)
else:
try:
merging = ma.merge_equivalents()
except Sorry as e:
if ("merge_equivalents_exact: incompatible"
in str(e)):
raise Sorry(
str(e) + " for %s" % ma.info().labels[-1] +
"\n" +
"Add --incompatible_flags_to_work_set to command line "
"arguments to place incompatible flags to working set."
)
raise
ma = merging.array().customized_copy(
crystal_symmetry=ma).set_info(ma.info())
elif return_as_miller_arrays: # allow non-unique set
pass
else:
n_all = ma.indices().size()
sel_unique = ma.unique_under_symmetry_selection()
sel_dup = ~flex.bool(n_all, sel_unique)
n_duplicate = sel_dup.count(True)
n_uus = sel_unique.size()
msg = (
"Miller indices not unique under symmetry: " + file_name + \
"(%d redundant indices out of %d)" % (n_all-n_uus, n_all) +
"Add --merge to command arguments to force merging data.")
if (show_details_if_error):
print(msg)
ma.show_comprehensive_summary(prefix=" ")
ma.map_to_asu().sort().show_array(prefix=" ")
raise Sorry(msg)
if (map_to_asu):
ma = ma.map_to_asu().set_info(ma.info())
if (remove_systematic_absences):
ma = ma.remove_systematic_absences()
if (label.startswith(output_r_free_label)
and complete_set is not None):
n_missing = len(complete_set.lone_set(other=ma).indices())
if (n_missing > 0):
if (extend_flags):
from cctbx import r_free_utils
# determine flag values
fvals = list(set(ma.data()))
print("fvals", fvals)
fval = None
if (len(fvals) == 1):
fval = fvals[0]
elif (len(fvals) == 2):
f1 = (ma.data()
== fvals[0]).count(True) / ma.data().size()
f2 = (ma.data()
== fvals[1]).count(True) / ma.data().size()
if (f1 < f2): fval = fvals[0]
else: fval = fvals[1]
elif (len(fvals) == 0):
fval = None
else:
fval = 0
if (not fval in fvals):
raise Sorry(
"Cannot determine free-R flag value.")
#
if (fval is not None):
ma = r_free_utils.extend_flags(
r_free_flags=ma,
test_flag_value=fval,
array_label=label,
complete_set=complete_set,
preserve_input_values=True,
allow_uniform_flags=True,
log=sys.stdout)
else:
ma = None
else:
libtbx.warn((
"%d reflections do not have R-free flags in the " +
"array '%s' - this may " +
"cause problems if you try to use the MTZ file for refinement "
+
"or map calculation. We recommend that you extend the flags "
+
"to cover all reflections (--extend_flags on the command line)."
) % (n_missing, label))
# Get rid of fake (0,0,0) reflection in some CIFs
if (ma is not None):
ma = ma.select_indices(indices=flex.miller_index(
((0, 0, 0), )),
negate=True).set_info(ma.info())
if return_as_miller_arrays:
miller_array_list.append(ma)
continue # don't make a dataset
dec = None
if ("FWT" in label):
dec = iotbx.mtz.ccp4_label_decorator()
column_types = None
if ("PHI" in label or "PHWT" in label) and (ma.is_real_array()):
column_types = "P"
elif (label.startswith("DANO") and ma.is_real_array()):
if (ma.sigmas() is not None):
column_types = "DQ"
else:
column_types = "D"
label_base = label
i = 1
while label in column_labels:
label = label_base + "-%i" % (i)
i += 1
if (ma is not None):
column_labels.add(label)
dataset.add_miller_array(ma,
column_root_label=label,
label_decorator=dec,
column_types=column_types)
if return_as_miller_arrays:
return miller_array_list
else:
return mtz_object
def run(args):
import libtbx.load_env
usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
read_reflections=True,
read_datablocks=True,
read_experiments=True,
phil=phil_scope,
check_format=False,
epilog=help_message)
from libtbx.utils import Sorry
params, options = parser.parse_args(show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
if not any([reflections, experiments, datablocks]):
parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(datablocks) != 1:
if experiments:
if len(experiments.imagesets()) != 1:
raise Sorry('exactly 1 datablock must be specified')
imageset = experiments.imagesets()[0]
else:
raise Sorry('exactly 1 datablock must be specified')
else:
imageset = datablocks[0].extract_imagesets()[0]
reflections = reflections[0]
if params.id is not None:
reflections = reflections.select(reflections['id'] == params.id)
stats = per_image_analysis.stats_imageset(
imageset,
reflections,
resolution_analysis=params.resolution_analysis,
plot=params.individual_plots)
per_image_analysis.print_table(stats)
from libtbx import table_utils
overall_stats = per_image_analysis.stats_single_image(
imageset, reflections, resolution_analysis=params.resolution_analysis)
rows = [
("Overall statistics", ""),
("#spots", "%i" % overall_stats.n_spots_total),
("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice),
("d_min", "%.2f" % overall_stats.estimated_d_min),
("d_min (distl method 1)",
"%.2f (%.2f)" % (overall_stats.d_min_distl_method_1,
overall_stats.noisiness_method_1)),
("d_min (distl method 2)",
"%.2f (%.2f)" % (overall_stats.d_min_distl_method_1,
overall_stats.noisiness_method_1)),
]
print(table_utils.format(rows, has_header=True, prefix="| ", postfix=" |"))
if params.json:
import json
if params.split_json:
for k in stats.__dict__:
start, end = params.json.split('.')
with open('%s_%s.%s' % (start, k, end), 'wb') as fp:
json.dump(stats.__dict__[k], fp)
if params.joint_json:
with open(params.json, 'wb') as fp:
json.dump(stats.__dict__, fp)
if params.plot:
per_image_analysis.plot_stats(stats, filename=params.plot)
class MainWindow(wx.Frame):
''' Frame housing the entire app; all windows open from this one '''
def __init__(self, parent, id, title):
wx.Frame.__init__(self, parent, id, title, size=(800, 500))
# TODO: Allow GUI to be loaded with command line args, parse into PHIL
self.iota_phil = inp.master_phil
self.prefs_phil = None
self.target_phil = None
self.term_file = None
# Create some defaults on startup
# Figure out temp folder
self.gparams = self.iota_phil.extract()
tmp_folder = '/tmp/{}_{}'.format(user, pid)
self.gparams.advanced.temporary_output_folder = tmp_folder
self.iota_phil = self.iota_phil.format(python_object=self.gparams)
# Menu bar
menubar = wx.MenuBar()
# Status bar
self.sb = self.CreateStatusBar()
# Help menu item with the about dialog
m_help = wx.Menu()
m_file = wx.Menu()
self.mb_load_script = m_file.Append(wx.ID_OPEN, '&Load Script...')
self.mb_save_script = m_file.Append(wx.ID_SAVE, '&Save Script...')
m_file.AppendSeparator()
self.mb_reset = m_file.Append(wx.ID_ANY, '&Reset Settings')
self.mb_about = m_help.Append(wx.ID_ANY, '&About')
menubar.Append(m_file, '&File')
menubar.Append(m_help, '&Help')
self.SetMenuBar(menubar)
self.main_sizer = wx.BoxSizer(wx.VERTICAL)
self.SetSizer(self.main_sizer)
# Toolbar
self.toolbar = self.CreateToolBar(style=wx.TB_3DBUTTONS | wx.TB_TEXT)
quit_bmp = bitmaps.fetch_icon_bitmap('actions', 'exit')
self.tb_btn_quit = self.toolbar.AddLabelTool(wx.ID_EXIT,
label='Quit',
bitmap=quit_bmp,
shortHelp='Quit',
longHelp='Quit IOTA')
pref_bmp = bitmaps.fetch_icon_bitmap('apps', 'advancedsettings')
self.tb_btn_prefs = self.toolbar.AddLabelTool(
wx.ID_ANY,
label='Preferences',
bitmap=pref_bmp,
shortHelp='Preferences',
longHelp='IOTA Preferences')
self.toolbar.AddSeparator()
load_bmp = bitmaps.fetch_icon_bitmap('actions', 'open')
self.tb_btn_load = self.toolbar.AddLabelTool(
wx.ID_ANY,
label='Load Script',
bitmap=load_bmp,
shortHelp='Load Script',
longHelp='Load IOTA Script')
save_bmp = bitmaps.fetch_icon_bitmap('actions', 'save')
self.tb_btn_save = self.toolbar.AddLabelTool(
wx.ID_ANY,
label='Save Script',
bitmap=save_bmp,
shortHelp='Save Script',
longHelp='Save IOTA Script')
reset_bmp = bitmaps.fetch_icon_bitmap('actions', 'reload')
self.tb_btn_reset = self.toolbar.AddLabelTool(
wx.ID_ANY,
label='Reset',
bitmap=reset_bmp,
shortHelp='Reset Settings',
longHelp='Reset IOTA settings with defaults')
self.toolbar.AddSeparator()
analyze_bmp = bitmaps.fetch_icon_bitmap('mimetypes',
'text-x-generic-2')
self.tb_btn_analysis = self.toolbar.AddLabelTool(
wx.ID_ANY,
label='Recover',
bitmap=analyze_bmp,
shortHelp='Recover',
longHelp='Recover run, show statistics and restart if aborted ')
run_bmp = bitmaps.fetch_icon_bitmap('actions', 'run')
self.tb_btn_run = self.toolbar.AddLabelTool(
wx.ID_ANY,
label='Run',
bitmap=run_bmp,
shortHelp='Run',
longHelp='Run all stages of refinement')
# Test buttons for test windows - comment out when not needed
# self.toolbar.AddSeparator()
test_bmp = bitmaps.fetch_icon_bitmap('actions', 'utilities')
self.tb_btn_test = self.toolbar.AddLabelTool(wx.ID_ANY,
label='Test',
bitmap=test_bmp)
self.Bind(wx.EVT_TOOL, self.onRun, self.tb_btn_test)
self.toolbar.RemoveTool(self.tb_btn_test.GetId())
# These buttons will be disabled until input path is provided
self.toolbar.EnableTool(self.tb_btn_run.GetId(), False)
self.toolbar.Realize()
# Instantiate windows
self.input_window = frm.InputWindow(self, phil=self.iota_phil)
# Single input window
self.main_sizer.Add(self.input_window,
1,
flag=wx.ALL | wx.EXPAND,
border=10)
self.main_sizer.Add((-1, 20))
# button bindings
self.Bind(wx.EVT_TOOL, self.onQuit, self.tb_btn_quit)
self.Bind(wx.EVT_TOOL, self.onPreferences, self.tb_btn_prefs)
self.Bind(wx.EVT_TOOL, self.onRun, self.tb_btn_run)
self.Bind(wx.EVT_TOOL, self.onRecovery, self.tb_btn_analysis)
self.Bind(wx.EVT_TOOL, self.onLoadScript, self.tb_btn_load)
self.Bind(wx.EVT_TOOL, self.onOutputScript, self.tb_btn_save)
self.Bind(wx.EVT_TOOL, self.onReset, self.tb_btn_reset)
# Menubar button bindings
self.Bind(wx.EVT_MENU, self.OnAboutBox, self.mb_about)
self.Bind(wx.EVT_MENU, self.onOutputScript, self.mb_save_script)
self.Bind(wx.EVT_MENU, self.onLoadScript, self.mb_load_script)
self.Bind(wx.EVT_MENU, self.onReset, self.mb_reset)
# Bindings to Input Window
self.Bind(wx.EVT_BUTTON, self.onImportOptions,
self.input_window.opt_btn_import)
self.Bind(wx.EVT_BUTTON, self.onProcessOptions,
self.input_window.opt_btn_process)
self.Bind(wx.EVT_BUTTON, self.onAnalysisOptions,
self.input_window.opt_btn_analysis)
# File list control bindings
self.Bind(ulc.EVT_LIST_INSERT_ITEM, self.onItemInserted,
self.input_window.input)
def read_command_line_options(self):
help_message = '''This command will run the IOTA GUI '''
self.args, self.phil_args = parse_command_args('').parse_known_args()
if self.args.path is not None and len(self.args.path) > 0:
for carg in self.args.path:
if os.path.exists(carg):
if os.path.isfile(carg) and os.path.basename(
carg).endswith('.param'):
self.load_script(filepath=carg,
update_input_window=False)
else:
self.input_window.input.add_item(os.path.abspath(carg))
if self.args.watch > 0:
self.gparams.advanced.monitor_mode = True
self.gparams.advanced.monitor_mode_timeout = True
self.gparams.advanced.monitor_mode_timeout_length = self.args.watch[
0]
if self.args.random > 0:
self.gparams.advanced.random_sample.flag_on = True
self.gparams.advanced.random_sample.number = self.args.random[0]
if self.args.tmp is not None:
self.gparams.advanced.temporary_output_folder = self.args.tmp[0]
if self.args.nproc is not None:
self.gparams.n_processors = self.args.nproc
self.iota_phil = self.iota_phil.format(python_object=self.gparams)
# Parse in-line params into phil
argument_interpreter = argint(master_phil=self.iota_phil)
consume = []
for arg in self.phil_args:
try:
command_line_params = argument_interpreter.process(arg=arg)
self.iota_phil = self.iota_phil.fetch(sources=[
command_line_params,
])
consume.append(arg)
except Sorry, e:
pass
for item in consume:
self.phil_args.remove(item)
if len(self.phil_args) > 0:
raise Sorry("Not all arguments processed, remaining: {}".format(
self.phil_args))
self.gparams = self.iota_phil.extract()
self.update_input_window()
def set_default_model_type(self, model_type):
if (model_type not in self._possible_model_types):
raise Sorry(
'Unrecognized model type, "%s," possible choices are %s.' %
(model_type, ', '.join(self._possible_model_types)))
self._default_model_type = model_type
def process_input_array(self, arr):
array = arr.deep_copy()
work_array = arr
multiplicities = None
try:
if self.merge_equivalents :
array, multiplicities, merge = MergeData(array, self.settings.show_anomalous_pairs)
settings = self.settings
data = array.data()
#import code, traceback; code.interact(local=locals(), banner="".join( traceback.format_stack(limit=10) ) )
self.missing_set = oop.null()
#if (array.is_xray_intensity_array()):
# data.set_selected(data < 0, flex.double(data.size(), 0.))
if (array.is_unique_set_under_symmetry()) and (settings.map_to_asu):
array = array.map_to_asu()
if (multiplicities is not None):
multiplicities = multiplicities.map_to_asu()
if (settings.d_min is not None):
array = array.resolution_filter(d_min=settings.d_min)
if (multiplicities is not None):
multiplicities = multiplicities.resolution_filter(
d_min=settings.d_min)
self.filtered_array = array.deep_copy()
if (settings.expand_anomalous):
if not array.is_unique_set_under_symmetry():
raise Sorry("Error! Cannot generate bijvoet mates of unmerged reflections.")
array = array.generate_bijvoet_mates()
original_symmetry = array.crystal_symmetry()
if (multiplicities is not None):
multiplicities = multiplicities.generate_bijvoet_mates()
if (self.settings.show_missing):
self.missing_set = array.complete_set().lone_set(array)
if self.settings.show_anomalous_pairs:
self.missing_set = self.missing_set.select(
self.missing_set.centric_flags().data(), negate=True)
if (settings.expand_to_p1):
if not array.is_unique_set_under_symmetry():
raise Sorry("Error! Cannot expand unmerged reflections to P1.")
original_symmetry = array.crystal_symmetry()
array = array.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
#array = array.niggli_cell().expand_to_p1()
#self.missing_set = self.missing_set.niggli_cell().expand_to_p1()
self.missing_set = self.missing_set.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
if (multiplicities is not None):
multiplicities = multiplicities.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
data = array.data()
self.r_free_mode = False
self.phases = flex.double(data.size(), float('nan'))
self.radians = flex.double(data.size(), float('nan'))
self.ampl = flex.double(data.size(), float('nan'))
self.sigmas = None
if isinstance(data, flex.bool):
self.r_free_mode = True
data_as_float = flex.double(data.size(), 0.0)
data_as_float.set_selected(data==True, flex.double(data.size(), 1.0))
data = data_as_float
self.data = data #.deep_copy()
else :
if isinstance(data, flex.double):
self.data = data #.deep_copy()
elif isinstance(data, flex.complex_double):
self.data = data #.deep_copy()
self.ampl = flex.abs(data)
self.phases = flex.arg(data) * 180.0/math.pi
# purge nan values from array to avoid crash in fmod_positive()
#b = flex.bool([bool(math.isnan(e)) for e in self.phases])
b = graphics_utils.IsNansArray( self.phases )
# replace the nan values with an arbitrary float value
self.phases = self.phases.set_selected(b, 42.4242)
# Cast negative degrees to equivalent positive degrees
self.phases = flex.fmod_positive(self.phases, 360.0)
self.radians = flex.arg(data)
# replace the nan values with an arbitrary float value
self.radians = self.radians.set_selected(b, 0.424242)
elif hasattr(array.data(), "as_double"):
self.data = data
else:
raise RuntimeError("Unexpected data type: %r" % data)
if (settings.show_data_over_sigma):
if (array.sigmas() is None):
raise Sorry("sigmas not defined.")
sigmas = array.sigmas()
non_zero_sel = sigmas != 0
array = array.select(non_zero_sel)
array = array.customized_copy(data=array.data()/array.sigmas())
self.data = array.data()
if (multiplicities is not None):
multiplicities = multiplicities.select(non_zero_sel)
if array.sigmas() is not None:
self.sigmas = array.sigmas()
else:
self.sigmas = None
work_array = array
except Exception as e:
print(to_str(e) + "".join(traceback.format_stack(limit=10)))
raise e
return None, None
work_array.set_info(arr.info() )
multiplicities = multiplicities
return work_array, multiplicities
def main(self):
# FIXME import simulation code
import cPickle as pickle
import math
from dials.util.command_line import Importer
from dials.algorithms.integration import ReflectionPredictor
from libtbx.utils import Sorry
# Parse the command line
params, options, args = self.parser.parse_args()
importer = Importer(args)
if len(importer.imagesets) == 0 and len(importer.crystals) == 0:
self.config().print_help()
return
if len(importer.imagesets) != 1:
raise Sorry('need 1 sweep: %d given' % len(importer.imagesets))
if len(importer.crystals) != 1:
raise Sorry('need 1 crystal: %d given' % len(importer.crystals))
sweep = importer.imagesets[0]
crystal = importer.crystals[0]
# generate predictions for possible reflections => generate a
# reflection list
predict = ReflectionPredictor()
predicted = predict(sweep, crystal)
# sort with James's reflection table: should this not go somewhere central?
from dials.scratch.jmp.container.reflection_table import ReflectionTable
# calculate shoebox sizes: take parameters from params & transform
# from reciprocal space to image space to decide how big a shoe box to use
table = ReflectionTable()
table['miller_index'] = predicted.miller_index()
indexer = table.index_map('miller_index')
candidates = []
unique = sorted(indexer)
for h, k, l in unique:
try:
for _h in h - 1, h + 1:
if not indexer[(_h, k, l)]:
raise ValueError('missing')
for _k in k - 1, k + 1:
if not indexer[(h, _k, l)]:
raise ValueError('missing')
for _l in l - 1, l + 1:
if not indexer[(h, k, _l)]:
raise ValueError('missing')
candidates.append((h, k, l))
except ValueError:
continue
from dials.algorithms.simulation.utils import build_prediction_matrix
from dials.algorithms.simulation.generate_test_reflections import \
master_phil
from libtbx.phil import command_line
cmd = command_line.argument_interpreter(master_params=master_phil)
working_phil = cmd.process_and_fetch(args=args[2:])
params = working_phil.extract()
node_size = params.rs_node_size
window_size = params.rs_window_size
reference = params.integrated_data_file
scale = params.integrated_data_file_scale
if reference:
counts_database = {}
from iotbx import mtz
m = mtz.object(reference)
mi = m.extract_miller_indices()
i = m.extract_reals('IMEAN').data
s = m.space_group().build_derived_point_group()
for j in range(len(mi)):
for op in s.all_ops():
hkl = tuple(map(int, op * mi[j]))
counts = max(0, int(math.floor(i[j] * scale)))
counts_database[hkl] = counts
counts_database[(-hkl[0], -hkl[1], -hkl[2])] = counts
else:
def constant_factory(value):
import itertools
return itertools.repeat(value).next
from collections import defaultdict
counts_database = defaultdict(constant_factory(params.counts))
from dials.model.data import ReflectionList
useful = ReflectionList()
d_matrices = []
for h, k, l in candidates:
hkl = predicted[indexer[(h, k, l)][0]]
_x = hkl.image_coord_px[0]
_y = hkl.image_coord_px[1]
_z = hkl.frame_number
# build prediction matrix
mhkl = predicted[indexer[(h - 1, k, l)][0]]
phkl = predicted[indexer[(h + 1, k, l)][0]]
hmkl = predicted[indexer[(h, k - 1, l)][0]]
hpkl = predicted[indexer[(h, k + 1, l)][0]]
hkml = predicted[indexer[(h, k, l - 1)][0]]
hkpl = predicted[indexer[(h, k, l + 1)][0]]
d = build_prediction_matrix(hkl, mhkl, phkl, hmkl, hpkl, hkml,
hkpl)
d_matrices.append(d)
# construct the shoebox parameters: outline the ellipsoid
x, y, z = [], [], []
for dh in (1, 0, 0), (0, 1, 0), (0, 0, 1):
dxyz = -1 * window_size * d * dh
x.append(dxyz[0] + _x)
y.append(dxyz[1] + _y)
z.append(dxyz[2] + _z)
dxyz = window_size * d * dh
x.append(dxyz[0] + _x)
y.append(dxyz[1] + _y)
z.append(dxyz[2] + _z)
hkl.bounding_box = (int(math.floor(min(x))),
int(math.floor(max(x)) + 1),
int(math.floor(min(y))),
int(math.floor(max(y)) + 1),
int(math.floor(min(z))),
int(math.floor(max(z)) + 1))
try:
counts = counts_database[hkl.miller_index]
useful.append(hkl)
except KeyError:
continue
from dials.algorithms import shoebox
shoebox.allocate(useful)
from dials.util.command_line import ProgressBar
p = ProgressBar(title='Generating shoeboxes')
# now for each reflection perform the simulation
for j, refl in enumerate(useful):
p.update(j * 100.0 / len(useful))
d = d_matrices[j]
from scitbx.random import variate, normal_distribution
g = variate(normal_distribution(mean=0, sigma=node_size))
counts = counts_database[refl.miller_index]
dhs = g(counts)
dks = g(counts)
dls = g(counts)
self.map_to_image_space(refl, d, dhs, dks, dls)
p.finished('Generated %d shoeboxes' % len(useful))
# now for each reflection add background
from dials.algorithms.simulation.generate_test_reflections import \
random_background_plane
p = ProgressBar(title='Generating background')
for j, refl in enumerate(useful):
p.update(j * 100.0 / len(useful))
if params.background:
random_background_plane(refl.shoebox, params.background, 0.0,
0.0, 0.0)
else:
random_background_plane(refl.shoebox, params.background_a,
params.background_b,
params.background_c,
params.background_d)
p.finished('Generated %d backgrounds' % len(useful))
if params.output.all:
pickle.dump(useful, open(params.output.all, 'w'))
return
