def per_image(self):
"""Set one block per image for all experiments"""
self._create_block_columns()
# get observed phi in radians
phi_obs = self._reflections["xyzobs.mm.value"].parts()[2]
for iexp, exp in enumerate(self._experiments):
sel = self._reflections["id"] == iexp
isel = sel.iselection()
exp_phi = phi_obs.select(isel)
# convert phi to integer frames
frames = exp.scan.get_array_index_from_angle(exp_phi, deg=False)
frames = flex.floor(frames).iround()
start, stop = flex.min(frames), flex.max(frames)
frame_range = range(start, stop + 1)
for f_num, f in enumerate(frame_range):
sub_isel = isel.select(frames == f)
f_cent = f + 0.5
self._reflections["block"].set_selected(sub_isel, f_num)
self._reflections["block_centre"].set_selected(sub_isel, f_cent)
return self._reflections
def per_image(self):
"""Set one block per image for all experiments"""
self._create_block_columns()
# get observed phi in radians
phi_obs = self._reflections['xyzobs.mm.value'].parts()[2]
for iexp, exp in enumerate(self._experiments):
sel = self._reflections['id'] == iexp
isel = sel.iselection()
exp_phi = phi_obs.select(isel)
# convert phi to integer frames
frames = exp.scan.get_array_index_from_angle(exp_phi, deg=False)
frames = flex.floor(frames).iround()
start, stop = flex.min(frames), flex.max(frames)
frame_range = range(start, stop + 1)
for f_num, f in enumerate(frame_range):
sub_isel = isel.select(frames == f)
self._reflections['block'].set_selected(sub_isel, f_num)
self._reflections['block_centre'].set_selected(sub_isel, f_num)
return self._reflections
def read_experiments(experiments, reflection_table):
"""
Get information from experiments and reflections
"""
# Get space group and unit cell
space_group = experiments[0].crystal.get_space_group()
sgno = space_group.type().number()
unit_cell_list = []
for e in experiments:
assert sgno == e.crystal.get_space_group().type().number()
unit_cell_list.append(e.crystal.get_unit_cell())
if len(unit_cell_list) > 1:
# calc mean unit cell
mean_parameters = [0, 0, 0, 0, 0, 0]
for uc in unit_cell_list:
for i in range(6):
mean_parameters[i] += uc.parameters()[i]
for i in range(6):
mean_parameters[i] /= len(unit_cell_list)
mean_unit_cell = uctbx.unit_cell(mean_parameters)
else:
mean_unit_cell = unit_cell_list[0]
# Require a dials scaled experiments file.
filtered_table = filter_reflection_table(reflection_table, ["scale"])
filtered_table["intensity"] = filtered_table["intensity.scale.value"]
filtered_table["variance"] = filtered_table["intensity.scale.variance"]
filtered_table["dataset"] = filtered_table["id"]
filtered_table["image"] = (
flex.floor(filtered_table["xyzobs.px.value"].parts()[2]).iround() +
1)
return filtered_table, mean_unit_cell, space_group
def read_experiments(self, experiments, reflections):
"""
Get information from experiments and reflections
"""
# Get space group and unit cell
space_group = None
unit_cell = []
exp_identifiers = []
for e in experiments:
if space_group is None:
space_group = e.crystal.get_space_group()
else:
assert (space_group.type().number() ==
e.crystal.get_space_group().type().number())
unit_cell.append(e.crystal.get_unit_cell())
exp_identifiers.append(e.identifier)
# get a list of the ids from the reflection table corresponding to exp_ids
identifiers = []
for expit in exp_identifiers:
for k in reflections.experiment_identifiers().keys():
if reflections.experiment_identifiers()[k] == expit:
identifiers.append(k)
break
# Selection of reflections
selection = ~(reflections.get_flags(reflections.flags.bad_for_scaling,
all=False))
outliers = reflections.get_flags(reflections.flags.outlier_in_scaling)
reflections = reflections.select(selection & ~outliers)
# Scale factor
inv_scale_factor = reflections["inverse_scale_factor"]
selection = inv_scale_factor > 0
reflections = reflections.select(selection)
inv_scale_factor = reflections["inverse_scale_factor"]
# Get the reflection data
index = reflections["id"]
miller_index = reflections["miller_index"]
intensity = reflections["intensity.scale.value"] / inv_scale_factor
variance = reflections["intensity.scale.variance"] / inv_scale_factor**2
# calculate image number of observation (e.g 0.0 <= z < 1.0), image = 1
images = flex.floor(
reflections["xyzobs.px.value"].parts()[2]).iround() + 1
# Get the MTZ file
return self.DataRecord(
unit_cell=unit_cell,
space_group=space_group,
miller_index=miller_index,
dataset=index,
intensity=intensity,
variance=variance,
identifiers=identifiers,
images=images,
)
def assign_batches_to_reflections(reflections, batch_offsets):
"""Assign a 'batch' column to the reflection table"""
for batch_offset, refl in zip(batch_offsets, reflections):
xdet, ydet, zdet = [flex.double(x) for x in refl["xyzobs.px.value"].parts()]
# compute BATCH values - floor() to get (fortran) image captured within
# +1 because FORTRAN counting; zdet+1=image_index
# +off because image_index+o=batch
refl["batch"] = (flex.floor(zdet).iround() + 1) + batch_offset
return reflections
def assign_segment_index(reflections):
"""Divide a sphere into 12 segments of equal area."""
seg_idx = flex.int(reflections.size(), -1)
all_theta = flex.bool(reflections.size(), True)
theta_idx_0 = reflections["theta"] < magic_theta
theta_idx_2 = reflections["theta"] > (180.0 - magic_theta)
theta_idx_1 = all_theta & ~(theta_idx_0 | theta_idx_2)
phi_norm = flex.floor(reflections["phi"] / 90.0)
phi_iround = phi_norm.iround()
phi_idx = phi_iround % 4 # phi from -180 to 180
seg_idx.set_selected(theta_idx_0, phi_idx)
phi_idx += 8
seg_idx.set_selected(theta_idx_2, phi_idx)
phi_idx = (flex.floor(
(reflections["phi"] + 45.0) / 90.00001).iround() % 4) + 4
seg_idx.set_selected(theta_idx_1, phi_idx)
assert flex.min(seg_idx) >= 0, flex.min(seg_idx)
assert flex.max(seg_idx) <= 11, flex.max(seg_idx)
reflections["class_index"] = seg_idx
return reflections
def _compute_background(self):
print("Computing background for %d reflections" %
len(self.reflections))
self.reflections.compute_background(self.experiments)
if False:
for r in self.reflections:
d = r["shoebox"].data
b = r["shoebox"].background
m = r["shoebox"].mask
diff = d - b
mask = (diff > 0).as_1d().as_int()
mask.reshape(diff.accessor())
diff = diff * mask.as_double().as_float()
print((flex.floor(diff)).as_numpy_array())
def stats_imageset(imageset,
reflections,
resolution_analysis=True,
plot=False):
n_spots_total = []
n_spots_no_ice = []
n_spots_4A = []
total_intensity = []
estimated_d_min = []
d_min_distl_method_1 = []
d_min_distl_method_2 = []
noisiness_method_1 = []
noisiness_method_2 = []
image_number = reflections["xyzobs.px.value"].parts()[2]
image_number = flex.floor(image_number)
try:
start, end = imageset.get_array_range()
except AttributeError:
start = 0
for i in range(len(imageset)):
stats = stats_single_image(
imageset[i:i + 1],
reflections.select(image_number == i + start),
i=i + start,
resolution_analysis=resolution_analysis,
plot=plot,
)
n_spots_total.append(stats.n_spots_total)
n_spots_no_ice.append(stats.n_spots_no_ice)
n_spots_4A.append(stats.n_spots_4A)
total_intensity.append(stats.total_intensity)
estimated_d_min.append(stats.estimated_d_min)
d_min_distl_method_1.append(stats.d_min_distl_method_1)
noisiness_method_1.append(stats.noisiness_method_1)
d_min_distl_method_2.append(stats.d_min_distl_method_2)
noisiness_method_2.append(stats.noisiness_method_2)
return group_args(
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spots_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2,
)
def stats_per_image(experiment, reflections, resolution_analysis=True):
n_spots_total = []
n_spots_no_ice = []
n_spots_4A = []
total_intensity = []
estimated_d_min = []
d_min_distl_method_1 = []
d_min_distl_method_2 = []
noisiness_method_1 = []
noisiness_method_2 = []
image_number = reflections["xyzobs.px.value"].parts()[2]
image_number = flex.floor(image_number)
try:
start, end = experiment.scan.get_array_range()
except AttributeError:
start, end = 0, 1
for i in range(start, end):
stats = stats_for_reflection_table(
reflections.select(image_number == i),
resolution_analysis=resolution_analysis,
)
n_spots_total.append(stats.n_spots_total)
n_spots_no_ice.append(stats.n_spots_no_ice)
n_spots_4A.append(stats.n_spots_4A)
total_intensity.append(stats.total_intensity)
estimated_d_min.append(stats.estimated_d_min)
d_min_distl_method_1.append(stats.d_min_distl_method_1)
noisiness_method_1.append(stats.noisiness_method_1)
d_min_distl_method_2.append(stats.d_min_distl_method_2)
noisiness_method_2.append(stats.noisiness_method_2)
return StatsMultiImage(
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spots_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2,
)
def stats_imageset(imageset, reflections, resolution_analysis=True, plot=False):
n_spots_total = []
n_spots_no_ice = []
n_spots_4A = []
total_intensity = []
estimated_d_min = []
d_min_distl_method_1 = []
d_min_distl_method_2 = []
noisiness_method_1 = []
noisiness_method_2 = []
image_number = reflections['xyzobs.px.value'].parts()[2]
image_number = flex.floor(image_number)
try:
start, end = imageset.get_array_range()
except AttributeError:
start = 1
for i in range(len(imageset)):
stats = stats_single_image(
imageset[i:i+1],
reflections.select(image_number==i+start), i=i+start,
resolution_analysis=resolution_analysis, plot=plot)
n_spots_total.append(stats.n_spots_total)
n_spots_no_ice.append(stats.n_spots_no_ice)
n_spots_4A.append(stats.n_spots_4A)
total_intensity.append(stats.total_intensity)
estimated_d_min.append(stats.estimated_d_min)
d_min_distl_method_1.append(stats.d_min_distl_method_1)
noisiness_method_1.append(stats.noisiness_method_1)
d_min_distl_method_2.append(stats.d_min_distl_method_2)
noisiness_method_2.append(stats.noisiness_method_2)
return group_args(n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spots_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2)
def export_mtz(
integrated_data,
experiment_list,
hklout,
ignore_panels=False,
include_partials=False,
keep_partials=False,
min_isigi=None,
force_static_model=False,
filter_ice_rings=False,
):
"""Export data from integrated_data corresponding to experiment_list to an
MTZ file hklout."""
from dials.array_family import flex
# for the moment assume (and assert) that we will convert data from exactly
# one lattice...
# FIXME allow for more than one experiment in here: this is fine just add
# multiple MTZ data sets (DIALS1...DIALSN) and multiple batch headers: one
# range of batches for each experiment
assert len(experiment_list) == 1
# select reflections that are assigned to an experiment (i.e. non-negative id)
integrated_data = integrated_data.select(integrated_data["id"] >= 0)
assert max(integrated_data["id"]) == 0
# strip out negative variance reflections: these should not really be there
# FIXME Doing select on summation results. Should do on profile result if
# present? Yes
if "intensity.prf.variance" in integrated_data:
selection = integrated_data.get_flags(integrated_data.flags.integrated, all=True)
else:
selection = integrated_data.get_flags(integrated_data.flags.integrated_sum)
integrated_data = integrated_data.select(selection)
selection = integrated_data["intensity.sum.variance"] <= 0
if selection.count(True) > 0:
integrated_data.del_selected(selection)
logger.info("Removing %d reflections with negative variance" % selection.count(True))
if "intensity.prf.variance" in integrated_data:
selection = integrated_data["intensity.prf.variance"] <= 0
if selection.count(True) > 0:
integrated_data.del_selected(selection)
logger.info("Removing %d profile reflections with negative variance" % selection.count(True))
if filter_ice_rings:
selection = integrated_data.get_flags(integrated_data.flags.in_powder_ring)
integrated_data.del_selected(selection)
logger.info("Removing %d reflections in ice ring resolutions" % selection.count(True))
if min_isigi is not None:
selection = (
integrated_data["intensity.sum.value"] / flex.sqrt(integrated_data["intensity.sum.variance"])
) < min_isigi
integrated_data.del_selected(selection)
logger.info("Removing %d reflections with I/Sig(I) < %s" % (selection.count(True), min_isigi))
if "intensity.prf.variance" in integrated_data:
selection = (
integrated_data["intensity.prf.value"] / flex.sqrt(integrated_data["intensity.prf.variance"])
) < min_isigi
integrated_data.del_selected(selection)
logger.info("Removing %d profile reflections with I/Sig(I) < %s" % (selection.count(True), min_isigi))
# FIXME in here work on including partial reflections => at this stage best
# to split off the partial refections into a different selection & handle
# gracefully... better to work on a short list as will need to "pop" them &
# find matching parts to combine.
if include_partials:
integrated_data = sum_partial_reflections(integrated_data)
integrated_data = scale_partial_reflections(integrated_data)
if "partiality" in integrated_data:
selection = integrated_data["partiality"] < 0.99
if selection.count(True) > 0 and not keep_partials:
integrated_data.del_selected(selection)
logger.info("Removing %d incomplete reflections" % selection.count(True))
# FIXME TODO for more than one experiment into an MTZ file:
#
# - add an epoch (or recover an epoch) from the scan and add this as an extra
# column to the MTZ file for scaling, so we know that the two lattices were
# integrated at the same time
# - decide a sensible BATCH increment to apply to the BATCH value between
# experiments and add this
#
# At the moment this is probably enough to be working on.
experiment = experiment_list[0]
# also only work with one panel(for the moment)
if not ignore_panels:
assert len(experiment.detector) == 1
from scitbx import matrix
if experiment.goniometer:
axis = matrix.col(experiment.goniometer.get_rotation_axis())
else:
axis = 0.0, 0.0, 0.0
s0 = experiment.beam.get_s0()
wavelength = experiment.beam.get_wavelength()
panel = experiment.detector[0]
origin = matrix.col(panel.get_origin())
fast = matrix.col(panel.get_fast_axis())
slow = matrix.col(panel.get_slow_axis())
pixel_size = panel.get_pixel_size()
fast *= pixel_size[0]
slow *= pixel_size[1]
cb_op_to_ref = experiment.crystal.get_space_group().info().change_of_basis_op_to_reference_setting()
experiment.crystal = experiment.crystal.change_basis(cb_op_to_ref)
U = experiment.crystal.get_U()
if experiment.goniometer is not None:
F = matrix.sqr(experiment.goniometer.get_fixed_rotation())
else:
F = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
unit_cell = experiment.crystal.get_unit_cell()
from iotbx import mtz
from scitbx.array_family import flex
from math import floor, sqrt
m = mtz.object()
m.set_title("from dials.export_mtz")
m.set_space_group_info(experiment.crystal.get_space_group().info())
if experiment.scan:
image_range = experiment.scan.get_image_range()
else:
image_range = 1, 1
# pointless (at least) doesn't like batches starting from zero
b_incr = max(image_range[0], 1)
for b in range(image_range[0], image_range[1] + 1):
o = m.add_batch().set_num(b + b_incr).set_nbsetid(1).set_ncryst(1)
o.set_time1(0.0).set_time2(0.0).set_title("Batch %d" % (b + b_incr))
o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0)
o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0)
o.set_divhd(0.0).set_divvd(0.0)
# FIXME hard-coded assumption on indealized beam vector below... this may be
# broken when we come to process data from a non-imgCIF frame
o.set_so(flex.float(s0)).set_source(flex.float((0, 0, -1)))
# these are probably 0, 1 respectively, also flags for how many are set, sd
o.set_bbfac(0.0).set_bscale(1.0)
o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0)
# unit cell (this is fine) and the what-was-refined-flags FIXME hardcoded
# take time-varying parameters from the *end of the frame* unlikely to
# be much different at the end - however only exist if time-varying refinement
# was used
if not force_static_model and experiment.crystal.num_scan_points > 0:
_unit_cell = experiment.crystal.get_unit_cell_at_scan_point(b - image_range[0])
_U = experiment.crystal.get_U_at_scan_point(b - image_range[0])
else:
_unit_cell = unit_cell
_U = U
# apply the fixed rotation to this to unify matrix definitions - F * U
# was what was used in the actual prediction: U appears to be stored
# as the transpose?! At least is for Mosflm...
#
# FIXME Do we need to apply the setting rotation here somehow? i.e. we have
# the U.B. matrix assuming that the axis is equal to S * axis_datum but
# here we are just giving the effective axis so at scan angle 0 this will
# not be correct... FIXME 2 not even sure we can express the stack of
# matrices S * R * F * U * B in MTZ format?...
_U = dials_u_to_mosflm(F * _U, _unit_cell)
# FIXME need to get what was refined and what was constrained from the
# crystal model
o.set_cell(flex.float(_unit_cell.parameters()))
o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1)))
o.set_umat(flex.float(_U.transpose().elems))
# get the mosaic spread though today it may not actually be set
mosaic = experiment.crystal.get_mosaicity()
o.set_crydat(flex.float([mosaic, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))
o.set_lcrflg(0)
o.set_datum(flex.float((0.0, 0.0, 0.0)))
# detector size, distance
o.set_detlm(flex.float([0.0, panel.get_image_size()[0], 0.0, panel.get_image_size()[1], 0, 0, 0, 0]))
o.set_dx(flex.float([panel.get_directed_distance(), 0.0]))
# goniometer axes and names, and scan axis number, and number of axes, missets
o.set_e1(flex.float(axis))
o.set_e2(flex.float((0.0, 0.0, 0.0)))
o.set_e3(flex.float((0.0, 0.0, 0.0)))
o.set_gonlab(flex.std_string(("AXIS", "", "")))
o.set_jsaxs(1)
o.set_ngonax(1)
o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))
# scan ranges, axis
if experiment.scan:
phi_start, phi_range = experiment.scan.get_image_oscillation(b)
else:
phi_start, phi_range = 0.0, 0.0
o.set_phistt(phi_start)
o.set_phirange(phi_range)
o.set_phiend(phi_start + phi_range)
o.set_scanax(flex.float(axis))
# number of misorientation angles
o.set_misflg(0)
# crystal axis closest to rotation axis (why do I want this?)
o.set_jumpax(0)
# type of data - 1; 2D, 2; 3D, 3; Laue
o.set_ldtype(2)
# now create the actual data structures - first keep a track of the columns
# H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
# LP MPART FLAG BGPKRATIOS
from cctbx.array_family import flex as cflex # implicit import
from cctbx.miller import map_to_asu_isym # implicit import
# gather the required information for the reflection file
nref = len(integrated_data["miller_index"])
x_px, y_px, z_px = integrated_data["xyzcal.px"].parts()
xdet = flex.double(x_px)
ydet = flex.double(y_px)
zdet = flex.double(z_px)
# compute ROT values
if experiment.scan:
rot = flex.double([experiment.scan.get_angle_from_image_index(z) for z in zdet])
else:
rot = zdet
# compute BATCH values
batch = flex.floor(zdet).iround() + 1 + b_incr
# we're working with full reflections so...
fractioncalc = flex.double(nref, 1.0)
# now go for it and make an MTZ file...
x = m.add_crystal("XTAL", "DIALS", unit_cell.parameters())
d = x.add_dataset("FROMDIALS", wavelength)
# now add column information...
# FIXME add DIALS_FLAG which can include e.g. was partial etc.
type_table = {
"H": "H",
"K": "H",
"L": "H",
"I": "J",
"SIGI": "Q",
"IPR": "J",
"SIGIPR": "Q",
"BG": "R",
"SIGBG": "R",
"XDET": "R",
"YDET": "R",
"BATCH": "B",
"BGPKRATIOS": "R",
"WIDTH": "R",
"MPART": "I",
"M_ISYM": "Y",
"FLAG": "I",
"LP": "R",
"FRACTIONCALC": "R",
"ROT": "R",
"DQE": "R",
}
# derive index columns from original indices with
#
# from m.replace_original_index_miller_indices
#
# so all that is needed now is to make space for the reflections - fill with
# zeros...
m.adjust_column_array_sizes(nref)
m.set_n_reflections(nref)
# assign H, K, L, M_ISYM space
for column in "H", "K", "L", "M_ISYM":
d.add_column(column, type_table[column]).set_values(flex.double(nref, 0.0).as_float())
m.replace_original_index_miller_indices(cb_op_to_ref.apply(integrated_data["miller_index"]))
d.add_column("BATCH", type_table["BATCH"]).set_values(batch.as_double().as_float())
if "lp" in integrated_data:
lp = integrated_data["lp"]
else:
lp = flex.double(nref, 1.0)
if "dqe" in integrated_data:
dqe = integrated_data["dqe"]
else:
dqe = flex.double(nref, 1.0)
I_profile = None
V_profile = None
I_sum = None
V_sum = None
# FIXME errors in e.g. LP correction need to be propogated here
scl = lp / dqe
if "intensity.prf.value" in integrated_data:
I_profile = integrated_data["intensity.prf.value"] * scl
V_profile = integrated_data["intensity.prf.variance"] * scl * scl
# Trap negative variances
assert V_profile.all_gt(0)
d.add_column("IPR", type_table["I"]).set_values(I_profile.as_float())
d.add_column("SIGIPR", type_table["SIGI"]).set_values(flex.sqrt(V_profile).as_float())
if "intensity.sum.value" in integrated_data:
I_sum = integrated_data["intensity.sum.value"] * scl
V_sum = integrated_data["intensity.sum.variance"] * scl * scl
# Trap negative variances
assert V_sum.all_gt(0)
d.add_column("I", type_table["I"]).set_values(I_sum.as_float())
d.add_column("SIGI", type_table["SIGI"]).set_values(flex.sqrt(V_sum).as_float())
if "background.sum.value" in integrated_data and "background.sum.variance" in integrated_data:
bg = integrated_data["background.sum.value"]
varbg = integrated_data["background.sum.variance"]
assert (varbg >= 0).count(False) == 0
sigbg = flex.sqrt(varbg)
d.add_column("BG", type_table["BG"]).set_values(bg.as_float())
d.add_column("SIGBG", type_table["SIGBG"]).set_values(sigbg.as_float())
d.add_column("FRACTIONCALC", type_table["FRACTIONCALC"]).set_values(fractioncalc.as_float())
d.add_column("XDET", type_table["XDET"]).set_values(xdet.as_float())
d.add_column("YDET", type_table["YDET"]).set_values(ydet.as_float())
d.add_column("ROT", type_table["ROT"]).set_values(rot.as_float())
d.add_column("LP", type_table["LP"]).set_values(lp.as_float())
d.add_column("DQE", type_table["DQE"]).set_values(dqe.as_float())
m.write(hklout)
return m
def _write_columns(mtz_file, dataset, integrated_data, scale_partials,
apply_scales):
"""Write the column definitions AND data for a single dataset."""
# now create the actual data structures - first keep a track of the columns
# H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
# LP MPART FLAG BGPKRATIOS
# gather the required information for the reflection file
nref = len(integrated_data['miller_index'])
# check reflections remain
if nref == 0:
raise Sorry('no reflections for export')
xdet, ydet, zdet = [
flex.double(x) for x in integrated_data['xyzobs.px.value'].parts()
]
# compute BATCH values - floor() to get (fortran) image captured within
# +1 because FORTRAN counting; zdet+1=image_index
# +off because image_index+o=batch
batch = (flex.floor(zdet).iround() + 1) + integrated_data["batch_offset"]
# we're working with full reflections so... #388 no longer guaranteed
if scale_partials:
fractioncalc = flex.double(nref, 1.0)
else:
fractioncalc = integrated_data['partiality']
# now add column information...
# FIXME add DIALS_FLAG which can include e.g. was partial etc.
type_table = {
'H': 'H',
'K': 'H',
'L': 'H',
'I': 'J',
'SIGI': 'Q',
'IPR': 'J',
'SIGIPR': 'Q',
'BG': 'R',
'SIGBG': 'R',
'XDET': 'R',
'YDET': 'R',
'BATCH': 'B',
'BGPKRATIOS': 'R',
'WIDTH': 'R',
'MPART': 'I',
'M_ISYM': 'Y',
'FLAG': 'I',
'LP': 'R',
'FRACTIONCALC': 'R',
'ROT': 'R',
'QE': 'R',
}
# derive index columns from original indices with
#
# from m.replace_original_index_miller_indices
#
# so all that is needed now is to make space for the reflections - fill with
# zeros...
mtz_file.adjust_column_array_sizes(nref)
mtz_file.set_n_reflections(nref)
# assign H, K, L, M_ISYM space
for column in 'H', 'K', 'L', 'M_ISYM':
dataset.add_column(column, type_table[column]).set_values(
flex.double(nref, 0.0).as_float())
mtz_file.replace_original_index_miller_indices(
integrated_data['miller_index_rebase'])
dataset.add_column('BATCH', type_table['BATCH']).set_values(
batch.as_double().as_float())
if 'lp' in integrated_data:
lp = integrated_data['lp']
else:
lp = flex.double(nref, 1.0)
if 'qe' in integrated_data:
qe = integrated_data['qe']
elif 'dqe' in integrated_data:
qe = integrated_data['dqe']
else:
qe = flex.double(nref, 1.0)
I_profile = None
V_profile = None
I_sum = None
V_sum = None
# FIXME errors in e.g. LP correction need to be propagated here
scl = lp / qe
if apply_scales:
scl = scl / integrated_data['inverse_scale_factor']
if 'intensity.prf.value' in integrated_data:
I_profile = integrated_data['intensity.prf.value'] * scl
V_profile = integrated_data['intensity.prf.variance'] * scl * scl
# Trap negative variances
assert V_profile.all_gt(0)
dataset.add_column('IPR',
type_table['I']).set_values(I_profile.as_float())
dataset.add_column('SIGIPR', type_table['SIGI']).set_values(
flex.sqrt(V_profile).as_float())
if 'intensity.sum.value' in integrated_data:
I_sum = integrated_data['intensity.sum.value'] * scl
V_sum = integrated_data['intensity.sum.variance'] * scl * scl
# Trap negative variances
assert V_sum.all_gt(0)
dataset.add_column('I', type_table['I']).set_values(I_sum.as_float())
dataset.add_column('SIGI', type_table['SIGI']).set_values(
flex.sqrt(V_sum).as_float())
if ('background.sum.value' in integrated_data
and 'background.sum.variance' in integrated_data):
bg = integrated_data['background.sum.value']
varbg = integrated_data['background.sum.variance']
assert (varbg >= 0).count(False) == 0
sigbg = flex.sqrt(varbg)
dataset.add_column('BG', type_table['BG']).set_values(bg.as_float())
dataset.add_column('SIGBG',
type_table['SIGBG']).set_values(sigbg.as_float())
dataset.add_column('FRACTIONCALC', type_table['FRACTIONCALC']).set_values(
fractioncalc.as_float())
dataset.add_column('XDET', type_table['XDET']).set_values(xdet.as_float())
dataset.add_column('YDET', type_table['YDET']).set_values(ydet.as_float())
dataset.add_column('ROT', type_table['ROT']).set_values(
integrated_data["ROT"].as_float())
dataset.add_column('LP', type_table['LP']).set_values(lp.as_float())
dataset.add_column('QE', type_table['QE']).set_values(qe.as_float())
def glm2(y):
from math import sqrt, exp, floor, log
from scipy.stats import poisson
from scitbx import matrix
from dials.array_family import flex
y = flex.double(y)
x = flex.double([1.0 for yy in y])
w = flex.double([1.0 for yy in y])
X = matrix.rec(x, (len(x), 1))
c = 1.345
beta = matrix.col([0])
maxiter = 10
accuracy = 1e-3
for iter in range(maxiter):
ni = flex.double([1.0 for xx in x])
sni = flex.sqrt(ni)
eta = flex.double(X * beta)
mu = flex.exp(eta)
dmu_deta = flex.exp(eta)
Vmu = mu
sVF = flex.sqrt(Vmu)
residP = (y - mu) * sni / sVF
phi = 1
sV = sVF * sqrt(phi)
residPS = residP / sqrt(phi)
H = flex.floor(mu * ni - c * sni * sV)
K = flex.floor(mu * ni + c * sni * sV)
# print min(H)
dpH = flex.double([poisson(mui).pmf(Hi) for mui, Hi in zip(mu, H)])
dpH1 = flex.double(
[poisson(mui).pmf(Hi - 1) for mui, Hi in zip(mu, H)])
dpK = flex.double([poisson(mui).pmf(Ki) for mui, Ki in zip(mu, K)])
dpK1 = flex.double(
[poisson(mui).pmf(Ki - 1) for mui, Ki in zip(mu, K)])
pHm1 = flex.double(
[poisson(mui).cdf(Hi - 1) for mui, Hi in zip(mu, H)])
pKm1 = flex.double(
[poisson(mui).cdf(Ki - 1) for mui, Ki in zip(mu, K)])
pH = pHm1 + dpH # = ppois(H,*)
pK = pKm1 + dpK # = ppois(K,*)
E2f = mu * (dpH1 - dpH - dpK1 + dpK) + pKm1 - pHm1
Epsi = c * (1.0 - pK - pH) + (mu / sV) * (dpH - dpK)
Epsi2 = c * c * (pH + 1.0 - pK) + E2f
EpsiS = c * (dpH + dpK) + E2f / sV
psi = flex.double([huber(rr, c) for rr in residPS])
cpsi = psi - Epsi
temp = cpsi * w * sni / sV * dmu_deta
EEqMat = [0] * len(X)
for j in range(X.n_rows()):
for i in range(X.n_columns()):
k = i + j * X.n_columns()
EEqMat[k] = X[k] * temp[j]
EEqMat = matrix.rec(EEqMat, (X.n_rows(), X.n_columns()))
EEq = []
for i in range(EEqMat.n_columns()):
col = []
for j in range(EEqMat.n_rows()):
k = i + j * EEqMat.n_columns()
col.append(EEqMat[k])
EEq.append(sum(col) / len(col))
EEq = matrix.col(EEq)
DiagB = EpsiS / (sni * sV) * w * (ni * dmu_deta)**2
B = matrix.diag(DiagB)
H = (X.transpose() * B * X) / len(y)
dbeta = H.inverse() * EEq
beta_new = beta + dbeta
relE = sqrt(
sum([d * d for d in dbeta]) / max(1e-10, sum([d * d
for d in beta])))
beta = beta_new
# print relE
if relE < accuracy:
break
weights = [min(1, c / abs(r)) for r in residPS]
eta = flex.double(X * beta)
mu = flex.exp(eta)
return beta
def glm2(y):
from math import sqrt, exp, floor, log
from scipy.stats import poisson
from scitbx import matrix
from dials.array_family import flex
y = flex.double(y)
x = flex.double([1.0 for yy in y])
w = flex.double([1.0 for yy in y])
X = matrix.rec(x, (len(x), 1))
c = 1.345
beta = matrix.col([0])
maxiter = 10
accuracy = 1e-3
for iter in range(maxiter):
ni = flex.double([1.0 for xx in x])
sni = flex.sqrt(ni)
eta = flex.double(X * beta)
mu = flex.exp(eta)
dmu_deta = flex.exp(eta)
Vmu = mu
sVF = flex.sqrt(Vmu)
residP = (y - mu) * sni / sVF
phi = 1
sV = sVF * sqrt(phi)
residPS = residP / sqrt(phi)
H = flex.floor(mu * ni - c * sni * sV)
K = flex.floor(mu * ni + c * sni * sV)
# print min(H)
dpH = flex.double([poisson(mui).pmf(Hi) for mui, Hi in zip(mu, H)])
dpH1 = flex.double([poisson(mui).pmf(Hi - 1) for mui, Hi in zip(mu, H)])
dpK = flex.double([poisson(mui).pmf(Ki) for mui, Ki in zip(mu, K)])
dpK1 = flex.double([poisson(mui).pmf(Ki - 1) for mui, Ki in zip(mu, K)])
pHm1 = flex.double([poisson(mui).cdf(Hi - 1) for mui, Hi in zip(mu, H)])
pKm1 = flex.double([poisson(mui).cdf(Ki - 1) for mui, Ki in zip(mu, K)])
pH = pHm1 + dpH # = ppois(H,*)
pK = pKm1 + dpK # = ppois(K,*)
E2f = mu * (dpH1 - dpH - dpK1 + dpK) + pKm1 - pHm1
Epsi = c * (1.0 - pK - pH) + (mu / sV) * (dpH - dpK)
Epsi2 = c * c * (pH + 1.0 - pK) + E2f
EpsiS = c * (dpH + dpK) + E2f / sV
psi = flex.double([huber(rr, c) for rr in residPS])
cpsi = psi - Epsi
temp = cpsi * w * sni / sV * dmu_deta
EEqMat = [0] * len(X)
for j in range(X.n_rows()):
for i in range(X.n_columns()):
k = i + j * X.n_columns()
EEqMat[k] = X[k] * temp[j]
EEqMat = matrix.rec(EEqMat, (X.n_rows(), X.n_columns()))
EEq = []
for i in range(EEqMat.n_columns()):
col = []
for j in range(EEqMat.n_rows()):
k = i + j * EEqMat.n_columns()
col.append(EEqMat[k])
EEq.append(sum(col) / len(col))
EEq = matrix.col(EEq)
DiagB = EpsiS / (sni * sV) * w * (ni * dmu_deta) ** 2
B = matrix.diag(DiagB)
H = (X.transpose() * B * X) / len(y)
dbeta = H.inverse() * EEq
beta_new = beta + dbeta
relE = sqrt(sum([d * d for d in dbeta]) / max(1e-10, sum([d * d for d in beta])))
beta = beta_new
# print relE
if relE < accuracy:
break
weights = [min(1, c / abs(r)) for r in residPS]
eta = flex.double(X * beta)
mu = flex.exp(eta)
return beta
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] experiments.json | integrated.pickle" % (
libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit()
reflections = reflections[0]
cryst = experiments.crystals()[0]
unit_cell = cryst.get_unit_cell()
if params.space_group is not None:
space_group = params.space_group.group()
assert space_group.is_compatible_unit_cell(unit_cell), unit_cell
else:
space_group = cryst.get_space_group()
print space_group.info()
print unit_cell
expt = experiments[0]
from cctbx import miller, crystal
from mmtbx.scaling.data_statistics import wilson_scaling
sel = reflections.get_flags(reflections.flags.integrated_sum)
reflections = reflections.select(sel)
cs = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)
ms = miller.set(cs,
indices=reflections['miller_index'],
anomalous_flag=True)
intensities = miller.array(ms,
data=reflections['intensity.sum.value'],
sigmas=flex.sqrt(
reflections['intensity.sum.variance']))
intensities.set_observation_type_xray_intensity()
d_star_sq = intensities.d_star_sq().data()
n_bins = 20
# binner = intensities.setup_binner_d_star_sq_step(
# d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
binner = intensities.setup_binner_counting_sorted(n_bins=n_bins)
# wilson = intensities.wilson_plot(use_binning=True)
# wilson.show()
# from matplotlib import pyplot
# pyplot.figure()
# pyplot.scatter(wilson.binner.bin_centers(2), wilson.data[1:-1])
# pyplot.show()
intensities = intensities.merge_equivalents().array()
wilson = wilson_scaling(intensities, n_residues=200)
wilson.iso_scale_and_b.show()
from matplotlib import pyplot
pyplot.figure()
pyplot.scatter(wilson.d_star_sq, wilson.mean_I_obs_data, label='Data')
pyplot.plot(wilson.d_star_sq, wilson.mean_I_obs_theory, label='theory')
pyplot.plot(wilson.d_star_sq,
wilson.mean_I_normalisation,
label='smoothed')
pyplot.yscale('log')
pyplot.legend()
pyplot.show()
import copy
import math
# Hack to make the predicter predict reflections outside of the range
# of the scan
expt_input = copy.deepcopy(expt)
scan = expt.scan
image_range = scan.get_image_range()
oscillation = scan.get_oscillation()
scan.set_image_range((1, int(math.ceil(360 / oscillation[1]))))
scan.set_oscillation((0, oscillation[1]))
print scan
# Populate the reflection table with predictions
predicted = flex.reflection_table.from_predictions(
expt, force_static=params.force_static, dmin=params.d_min)
predicted['id'] = flex.int(len(predicted), 0)
print len(predicted)
space_group = space_group.build_derived_reflection_intensity_group(
anomalous_flag=True)
cs = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms,
data=flex.double(ms.size(), 1),
sigmas=flex.double(ms.size(), 1))
d_star_sq = ma.d_star_sq().data()
n_bins = 1
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq) - flex.min(d_star_sq) + 1e-8) /
n_bins)
image_number = predicted['xyzcal.px'].parts()[2]
print flex.min(image_number)
print flex.max(image_number)
#dose = flex.size_t(list(flex.floor(image_number).iround()))
angle_deg = predicted['xyzcal.mm'].parts()[2] * 180 / math.pi
dose = flex.size_t(list(flex.floor(angle_deg).iround()))
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(binner)
n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
ranges_dict = {}
completeness_levels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99]
for c in completeness_levels:
ranges_dict[c] = []
from xia2.Modules.PyChef2 import ChefStatistics
step = 1
for i in range(0, 360, step):
sel = dose < step
dose.set_selected(sel, dose.select(sel) + 360)
dose -= flex.min(dose)
chef_stats = ChefStatistics(ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, n_complete,
binner, ma.space_group(),
ma.anomalous_flag(), n_steps)
ieither_completeness = chef_stats.ieither_completeness()
iboth_completeness = chef_stats.iboth_completeness()
for c in completeness_levels:
ranges_dict[c].append(
min((ieither_completeness > (c / 100)).iselection()))
from matplotlib import pyplot
pyplot.figure()
for c in completeness_levels:
pyplot.plot(ranges_dict[c], label=str(c))
# pyplot.plot(range_for_50)
# pyplot.plot(range_for_99)
# pyplot.scatter(range(iboth_completeness.size()), iboth_completeness)
pyplot.legend()
pyplot.show()
def run(args):
from dials.util.options import OptionParser
import libtbx.load_env
usage = "%s [options] stars.pickle" % (
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
from dials.array_family import flex
import cPickle as pickle
from astrotbx.algorithms.match import matcher, pair_up, compute_Rt
stars = None
for arg in args:
tmp_stars = pickle.load(open(arg))
if not stars:
stars = tmp_stars
else:
stars.extend(tmp_stars)
z = flex.floor(stars['xyzobs.px.value'].parts()[2]).iround()
zs = sorted(set(z))
Rtds = []
Rtds.append({'R': (1, 0, 0, 1), 't': (0, 0), 'd': 0, 'n': 0, 'dt': 0})
from scitbx import matrix
import math
datum0 = stars.select(z == zs[0])
for j in range(len(zs) - 1):
_r = zs[j]
_z = zs[j + 1]
datum = stars.select(z == _r)
move = stars.select(z == _z)
dt = move['timestamp'][0]
# get the moves for this step
R, t, d, n = matcher(datum, move, params)
# compose with previous to map back to datum i.e. 0-th image positions
_R, _t = matrix.sqr(Rtds[-1]['R']), matrix.col(Rtds[-1]['t'])
Rc = matrix.sqr(R) * _R
tc = matrix.sqr(R) * _t + matrix.col(t)
# refine w.r.t. datum positions... seems to drift
rsel, msel = pair_up(datum0, move, params, Rc, tc)
R1, t1, d1, n1 = compute_Rt(datum0.select(rsel), move.select(msel))
# now use this stack to match up with the datum stars i.e. apply to star
# positions, match then use that iselection to derive the full Rt =>
# will involve refactor... N.B. need to do outlier rejection after matching
# but before mapping back / Rt calculation.
# the re-compute the full Rt from datum to this time point
Rtds.append({'R': R1.elems, 't': t1, 'd': d1, 'n': n1, 'dt': dt})
for j, Rtd in enumerate(Rtds):
print('%3d %.4f %3d %4d' % (j, Rtd['d'], Rtd['n'], Rtd['dt']))
import json
json.dump(Rtds, open(params.output, 'w'))