def read_data(self,params):
from os import listdir, path
from libtbx import easy_pickle
from cctbx.crystal_orientation import crystal_orientation # XXX Necessary later?
#directory = "/net/viper/raid1/hattne/L220/merging/05fs"
#directory = "/reg/d/psdm/cxi/cxib8113/scratch/sauter/metrology/008"
#directory = "/reg/d/psdm/xpp/xpp74813/scratch/sauter/metrology/204"
#directory = "/net/viper/raid1/hattne/L220/merging/test"
#directory = "/reg/d/psdm/xpp/xppb4313/scratch/brewster/results/r0243/003/integration"
#directory = "/reg/d/psdm/cxi/cxic0614/scratch/sauter/metrology/004/integration"
#directory = "/reg/d/psdm/cxi/cxic0614/scratch/sauter/metrology/150/integration"
#directory = "/reg/d/psdm/cxi/cxic0614/scratch/sauter/metrology/152/integration"
directory = "/reg/d/psdm/cxi/cxib6714/scratch/sauter/metrology/009/integration"
dir_glob = "/reg/d/psdm/CXI/cxib6714/scratch/sauter/results/r*/009/integration"
dir_glob = "/reg/d/psdm/CXI/cxib6714/scratch/sauter/results/r*/801/integration"
dir_glob = "/reg/d/psdm/xpp/xpp74813/scratch/sauter/r*/216/integration"
dir_glob = "/reg/d/psdm/xpp/xpp74813/ftc/sauter/result/r*/104/integration"
dir_glob = "/reg/d/psdm/cxi/cxid9114/scratch/sauter/metrology/001/integration"
dir_glob = "/reg/d/psdm/CXI/cxid9114/ftc/brewster/results/r00[3-4]*/003/integration"
dir_glob = "/reg/d/psdm/CXI/cxid9114/ftc/sauter/results/r00[3-4]*/004/integration"
dir_glob = "/reg/d/psdm/CXI/cxid9114/ftc/sauter/results/r00[3-4]*/006/integration"
dir_list = ["/reg/d/psdm/CXI/cxid9114/ftc/brewster/results/r%04d/006/integration"%seq for seq in range(95,115)]
dir_list = ["/reg/d/psdm/CXI/cxid9114/ftc/sauter/results/r%04d/018/integration"%seq for seq in range(102,115)]
dir_list = params.data
T = Timer("populate C++ store with register line")
itile = flex.int()
self.spotfx = flex.double()
self.spotfy = flex.double()
self.spotcx = flex.double()
self.spotcy = flex.double()
self.observed_cntr_x = flex.double()
self.observed_cntr_y = flex.double()
self.refined_cntr_x = flex.double()
self.refined_cntr_y = flex.double()
self.HKL = flex.miller_index()
self.radial = flex.double()
self.azimut = flex.double()
self.FRAMES = dict(
frame_id=flex.int(),
wavelength=flex.double(),
beam_x=flex.double(),
beam_y=flex.double(),
distance=flex.double(),
orientation=[],
rotation100_rad=flex.double(),
rotation010_rad=flex.double(),
rotation001_rad=flex.double(),
half_mosaicity_deg=flex.double(),
wave_HE_ang=flex.double(),
wave_LE_ang=flex.double(),
domain_size_ang=flex.double(),
unique_file_name=[]
)
self.frame_id = flex.int()
import glob
#for directory in glob.glob(dir_glob):
for directory in dir_list:
if self.params.max_frames is not None and len(self.FRAMES['frame_id']) >= self.params.max_frames:
break
for entry in listdir(directory):
tttd = d = easy_pickle.load(path.join(directory, entry))
# XXX Hardcoded, should honour the phil! And should be verified
# to be consistent for each correction vector later on!
#import pdb; pdb.set_trace()
setting_id = d['correction_vectors'][0][0]['setting_id']
#if setting_id != 5:
#if setting_id != 12:
if setting_id != self.params.bravais_setting_id:
#if setting_id != 22:
#print "HATTNE BIG SLIPUP 1"
continue
# Assert that effective_tiling is consistent, and a non-zero
# multiple of eight (only whole sensors considered for now--see
# mark10.fit_translation4.print_table()). self.tiles is
# initialised to zero-length in the C++ code. XXX Should now be
# able to retire the "effective_tile_boundaries" parameter.
#
# XXX Other checks from correction_vector plot, such as consistent
# setting?
if hasattr(self, 'tiles') and len(self.tiles) > 0:
assert (self.tiles == d['effective_tiling']).count(False) == 0
else:
assert len(d['effective_tiling']) > 0 \
and len(d['effective_tiling']) % 8 == 0
self.tiles = d['effective_tiling']
if not self.standalone_check(self,setting_id,entry,d,params.diff_cutoff): continue
# Reading the frame data. The frame ID is just the index of the
# image.
self.FRAMES['frame_id'].append(len(self.FRAMES['frame_id']) + 1) # XXX try zero-based here
self.FRAMES['wavelength'].append(d['wavelength'])
self.FRAMES['beam_x'].append(d['xbeam'])
self.FRAMES['beam_y'].append(d['ybeam'])
self.FRAMES['distance'].append(d['distance'])
self.FRAMES['orientation'].append(d['current_orientation'][0])
self.FRAMES['rotation100_rad'].append(0) # XXX FICTION
self.FRAMES['rotation010_rad'].append(0) # XXX FICTION
self.FRAMES['rotation001_rad'].append(0) # XXX FICTION
self.FRAMES['half_mosaicity_deg'].append(0) # XXX FICTION
# self.FRAMES['wave_HE_ang'].append(0.995 * d['wavelength']) # XXX FICTION -- what does Nick use?
# self.FRAMES['wave_LE_ang'].append(1.005 * d['wavelength']) # XXX FICTION
self.FRAMES['wave_HE_ang'].append(d['wavelength'])
self.FRAMES['wave_LE_ang'].append(d['wavelength'])
self.FRAMES['domain_size_ang'].append(5000) # XXX FICTION
self.FRAMES['unique_file_name'].append(path.join(directory, entry))
print "added frame", self.FRAMES['frame_id'][-1],entry
for cv in d['correction_vectors'][0]:
# Try to reproduce every predicition using the model from the
# frame -- skip CV if fail. Could be because of wrong HKL:s?
#
# Copy these two images to test directory to reproduce:
# int-s01-2011-02-20T21:27Z37.392_00000.pickle
# int-s01-2011-02-20T21:27Z37.725_00000.pickle
from rstbx.bandpass import use_case_bp3, parameters_bp3
from scitbx.matrix import col
from math import hypot, pi
indices = flex.miller_index()
indices.append(cv['hkl'])
parameters = parameters_bp3(
indices=indices,
orientation=self.FRAMES['orientation'][-1],
incident_beam=col(self.INCIDENT_BEAM),
packed_tophat=col((1.,1.,0.)),
detector_normal=col(self.DETECTOR_NORMAL),
detector_fast=col((0.,1.,0.)),detector_slow=col((1.,0.,0.)),
pixel_size=col((0.11,0.11,0)), # XXX hardcoded, twice!
pixel_offset=col((0.,0.,0.0)),
distance=self.FRAMES['distance'][-1],
detector_origin=col((-self.FRAMES['beam_x'][-1],
-self.FRAMES['beam_y'][-1],
0))
)
ucbp3 = use_case_bp3(parameters=parameters)
ucbp3.set_active_areas(self.tiles)
integration_signal_penetration=0.5
ucbp3.set_sensor_model(thickness_mm=0.5,
mu_rho=8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration=integration_signal_penetration)
half_mosaicity_rad = self.FRAMES['half_mosaicity_deg'][-1] * pi/180.
ucbp3.set_mosaicity(half_mosaicity_rad)
ucbp3.set_bandpass(self.FRAMES['wave_HE_ang'][-1],
self.FRAMES['wave_LE_ang'][-1])
ucbp3.set_orientation(self.FRAMES['orientation'][-1])
ucbp3.set_domain_size(self.FRAMES['domain_size_ang'][-1])
ucbp3.picture_fast_slow_force()
ucbp3_prediction = 0.5 * (ucbp3.hi_E_limit + ucbp3.lo_E_limit)
diff = hypot(ucbp3_prediction[0][0] - cv['predspot'][1],
ucbp3_prediction[0][1] - cv['predspot'][0])
if diff > self.params.diff_cutoff:
print "HATTNE INDEXING SLIPUP"
continue
# For some reason, the setting_id is recorded for each
# correction vector as well--assert that it is consistent.
#if cv['setting_id'] != setting_id:
# print "HATTNE BIG SLIPUP 2"
assert cv['setting_id'] == setting_id
# For each observed spot, figure out what tile it is on, and
# store in itile. XXX This is probably not necessary here, as
# correction_vector_store::register_line() does the same thing.
obstile = None
for i in range(0, len(self.tiles), 4):
if cv['obsspot'][0] >= self.tiles[i + 0] \
and cv['obsspot'][0] <= self.tiles[i + 2] \
and cv['obsspot'][1] >= self.tiles[i + 1] \
and cv['obsspot'][1] <= self.tiles[i + 3]:
obstile = i
break
assert obstile is not None
itile.append(obstile) # XXX unused variable?
# ID of current frame.
self.frame_id.append(self.FRAMES['frame_id'][-1])
self.spotfx.append(cv['obsspot'][0])
self.spotfy.append(cv['obsspot'][1])
self.spotcx.append(cv['predspot'][0])
self.spotcy.append(cv['predspot'][1])
self.observed_cntr_x.append(cv['obscenter'][0])
self.observed_cntr_y.append(cv['obscenter'][1])
self.refined_cntr_x.append(cv['refinedcenter'][0])
self.refined_cntr_y.append(cv['refinedcenter'][1])
self.HKL.append(cv['hkl'])
self.azimut.append(cv['azimuthal'])
self.radial.append(cv['radial'])
#print self.FRAMES['frame_id'][-1]
# Should honour the max_frames phil parameter
#if len(self.FRAMES['frame_id']) >= 1000:
if self.params.max_frames is not None and \
len(self.FRAMES['frame_id']) >= self.params.max_frames:
break
"""
For 5000 first images:
STATS FOR TILE 14
sel_delx -6.59755265524 -4.41676757746e-10 5.7773557278
sel_dely -6.30796620634 -8.3053734774e-10 6.3362200841
symmetric_offset_x -6.5975526548 -2.73229417105e-15 5.77735572824
symmetric_offset_y -6.30796620551 1.16406818748e-15 6.33622008493
symmetric rsq 0.000255199593417 2.95803352999 56.1918083904
rmsd 1.71989346472
For 10000 first images:
STATS FOR TILE 14
sel_delx -6.92345292727 6.9094552919e-10 611.497770006
sel_dely -6.39690476093 1.1869355797e-09 894.691806871
symmetric_offset_x -6.92345292796 1.28753258216e-14 611.497770005
symmetric_offset_y -6.39690476212 -2.10251420168e-15 894.69180687
symmetric rsq 1.58067791823e-05 30.3331143761 1174402.952
rmsd 5.50755066941
"""
# This is mark3.fit_translation2.nominal_tile_centers()
self.To_x = flex.double(len(self.tiles) // 4)
self.To_y = flex.double(len(self.tiles) // 4)
for x in range(len(self.tiles) // 4):
self.To_x[x] = (self.tiles[4 * x + 0] + self.tiles[4 * x + 2]) / 2
self.To_y[x] = (self.tiles[4 * x + 1] + self.tiles[4 * x + 3]) / 2
delx = self.spotcx - self.spotfx
dely = self.spotcy - self.spotfy
self.delrsq = self.delrsq_functional(calcx = self.spotcx, calcy = self.spotcy)
self.initialize_per_tile_sums()
self.tile_rmsd = [0.]*(len(self.tiles) // 4)
self.asymmetric_tile_rmsd = [0.]*(len(self.tiles) // 4)
# XXX Is (beam1x, beam1y) really observed center and (beamrx,
# beamry) refined center? Nick thinks YES!
#
#itile2 = flex.int([self.register_line(a[2],a[3],a[4],a[5],a[6],a[7],a[8],a[9]) for a in ALL])
itile2 = flex.int(
[self.register_line(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7])
for a in zip(self.observed_cntr_x, self.observed_cntr_y,
self.refined_cntr_x, self.refined_cntr_y,
self.spotfx, self.spotfy,
self.spotcx, self.spotcy)])
if params.show_consistency: consistency_controls(self,params)
T = Timer("calcs based on C++ store")
self.selections = []
self.selection_counts = []
for x in range(len(self.tiles) // 4):
if self.tilecounts[x]==0:
self.radii[x] = 0
self.mean_cv[x] = matrix.col((0, 0))
else:
self.radii[x]/=self.tilecounts[x]
self.mean_cv[x] = matrix.col(self.mean_cv[x]) / self.tilecounts[x]
selection = (self.master_tiles == x)
self.selections.append(selection)
selected_cv = self.master_cv.select(selection)
self.selection_counts.append(selected_cv.size()) # for curvatures
if len(selected_cv)>0:
self.asymmetric_tile_rmsd[x] = math.sqrt(flex.mean (self.delrsq.select(selection)))
sel_delx = delx.select(selection)
sel_dely = dely.select(selection)
symmetric_offset_x = sel_delx - self.mean_cv[x][0]
symmetric_offset_y = sel_dely - self.mean_cv[x][1]
symmetricrsq = symmetric_offset_x*symmetric_offset_x + symmetric_offset_y*symmetric_offset_y
self.tile_rmsd[x] =math.sqrt(flex.mean(symmetricrsq))
else:
self.asymmetric_tile_rmsd[x]=0.
self.tile_rmsd[x]=0.
self.overall_N = flex.sum(flex.int( [int(t) for t in self.tilecounts] ))
self.overall_cv = matrix.col(self.overall_cv)/self.overall_N
self.overall_rmsd = math.sqrt( self.sum_sq_cv / self.overall_N )
# master weights for mark3 calculation takes 0.3 seconds
self.master_weights = flex.double(len(self.master_tiles))
self.largest_sample = max(self.tilecounts)
for x in range(len(self.tiles) // 4):
self.master_weights.set_selected( self.selections[x], self.tile_weight(x))
print "AFTER read cx, cy", flex.mean(self.spotcx), flex.mean(self.spotcy)
print "AFTER read fx, fy", flex.mean(self.spotfx), flex.mean(self.spotfy)
print "AFTER read rmsd_x, rmsd_y", math.sqrt(flex.mean(flex.pow(self.spotcx - self.spotfx, 2))), \
math.sqrt(flex.mean(flex.pow(self.spotcy - self.spotfy, 2)))
return
def parameter_based_model(self, params):
PIXEL_SZ = 0.11 # mm/pixel
all_model = mark3_collect_data(self.frame_id, self.HKL)
for iframe in range(len(self.FRAMES["frame_id"])):
frame_id = self.FRAMES["frame_id"][iframe]
if frame_id not in self.bandpass_models:
reserve_orientation = self.FRAMES["orientation"][iframe]
effective_orientation = reserve_orientation
#Not necessary to apply the 3 offset rotations; they have apparently
# been applied already.\
# .rotate_thru((1,0,0),self.FRAMES["rotation100_rad"][iframe]
# ).rotate_thru((0,1,0),self.FRAMES["rotation010_rad"][iframe]
# ).rotate_thru((0,0,1),self.FRAMES["rotation001_rad"][iframe])
detector_origin = col((-self.FRAMES["beam_x"][iframe],
-self.FRAMES["beam_y"][iframe], 0.))
crystal = symmetry(unit_cell=effective_orientation.unit_cell(),
space_group="P1")
indices = all_model.frame_indices(frame_id)
parameters = parameters_bp3(
indices=indices,
orientation=effective_orientation,
incident_beam=col(correction_vectors.INCIDENT_BEAM),
packed_tophat=col((1., 1., 0.)),
detector_normal=col(correction_vectors.DETECTOR_NORMAL),
detector_fast=col((0., 1., 0.)),
detector_slow=col((1., 0., 0.)),
pixel_size=col((PIXEL_SZ, PIXEL_SZ, 0)),
pixel_offset=col((0., 0., 0.0)),
distance=self.FRAMES["distance"][iframe],
detector_origin=detector_origin)
ucbp3 = use_case_bp3(parameters=parameters)
ucbp3.set_active_areas(
self.tiles) #params.effective_tile_boundaries
integration_signal_penetration = 0.5
ucbp3.set_sensor_model(
thickness_mm=0.5,
mu_rho=8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration=integration_signal_penetration)
half_mosaicity_rad = self.FRAMES["half_mosaicity_deg"][
iframe] * pi / 180.
ucbp3.set_mosaicity(half_mosaicity_rad)
ucbp3.set_bandpass(self.FRAMES["wave_HE_ang"][iframe],
self.FRAMES["wave_LE_ang"][iframe])
ucbp3.set_orientation(effective_orientation)
ucbp3.set_domain_size(self.FRAMES["domain_size_ang"][iframe])
ucbp3.picture_fast_slow_force()
self.bandpass_models[frame_id] = ucbp3
all_model.collect(self.bandpass_models[frame_id].hi_E_limit,
self.bandpass_models[frame_id].lo_E_limit,
self.bandpass_models[frame_id].observed_flag,
frame_id)
sq_displacements = ((all_model.cx - self.spotcx) *
(all_model.cx - self.spotcx) +
(all_model.cy - self.spotcy) *
(all_model.cy - self.spotcy))
selected_sq_displacements = sq_displacements.select(
all_model.flags == True)
print("Root Mean squared displacement all spots %8.3f" %
math.sqrt(
flex.sum(selected_sq_displacements) /
len(selected_sq_displacements)))
return all_model.cx, all_model.cy, all_model.flags
def standalone_check(self,setting_id,entry,d,cutoff):
wavelength = (d['wavelength'])
beam_x = (d['xbeam'])
beam_y = (d['ybeam'])
distance = (d['distance'])
orientation = (d['current_orientation'][0])
print "testing frame....................",entry
for cv in d['correction_vectors'][0]:
from rstbx.bandpass import use_case_bp3, parameters_bp3
from scitbx.matrix import col
from math import hypot, pi
indices = flex.miller_index()
indices.append(cv['hkl'])
parameters = parameters_bp3(
indices=indices,
orientation=orientation,
incident_beam=col(self.INCIDENT_BEAM),
packed_tophat=col((1.,1.,0.)),
detector_normal=col(self.DETECTOR_NORMAL),
detector_fast=col((0.,1.,0.)),detector_slow=col((1.,0.,0.)),
pixel_size=col((0.11,0.11,0)), # XXX hardcoded, twice!
pixel_offset=col((0.,0.,0.0)),
distance=distance,
detector_origin=col((-beam_x,-beam_y,0))
)
ucbp3 = use_case_bp3(parameters=parameters)
ucbp3.set_active_areas(self.tiles)
integration_signal_penetration=0.5
ucbp3.set_sensor_model(thickness_mm=0.5,
mu_rho=8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration=integration_signal_penetration)
ucbp3.set_mosaicity(0.)
ucbp3.set_bandpass(wavelength,
wavelength)
ucbp3.set_orientation(orientation)
ucbp3.set_domain_size(5000.)
ucbp3.picture_fast_slow_force()
ucbp3_prediction = 0.5 * (ucbp3.hi_E_limit + ucbp3.lo_E_limit)
diff = hypot(ucbp3_prediction[0][0] - cv['predspot'][1],
ucbp3_prediction[0][1] - cv['predspot'][0])
if diff > cutoff:
print "Correction vector too long: %6.2f pixels; ignore image or increase diff_cutoff (current value=%5.1f)"%(diff,cutoff)
return False
# For some reason, the setting_id is recorded for each
# correction vector as well--assert that it is consistent.
#if cv['setting_id'] != setting_id:
# print "HATTNE BIG SLIPUP 2"
if not cv['setting_id'] == setting_id: return False
# For each observed spot, figure out what tile it is on, and
# store in itile. XXX This is probably not necessary here, as
# correction_vector_store::register_line() does the same thing.
obstile = None
for i in range(0, len(self.tiles), 4):
if cv['obsspot'][0] >= self.tiles[i + 0] \
and cv['obsspot'][0] <= self.tiles[i + 2] \
and cv['obsspot'][1] >= self.tiles[i + 1] \
and cv['obsspot'][1] <= self.tiles[i + 3]:
obstile = i
break
if obstile is None: return False
spotfx = (cv['obsspot'][0])
spotfy = (cv['obsspot'][1])
spotcx = (cv['predspot'][0])
spotcy = (cv['predspot'][1])
correction_vector_x = spotcx - spotfx
correction_vector_y = spotcy - spotfy
length = hypot(correction_vector_x, correction_vector_y)
if length > 8:
print "LENGTH SLIPUP",length
return False
return True
def extend_predictions(pdata,
int_pickle_path,
image_info,
dmin=1.5,
dump=False,
detector_phil=None):
"""
Given a LABELIT format integration pickle, generate a new predictor for reflections
extending to a higher resolution dmin matching the current unit cell, orientation,
mosaicity and domain size.
"""
# image_info is an instance of ImageInfo
img_path = image_info.img_path
img_size = image_info.img_size
pixel_size = image_info.pixel_size
# pdata is the integration pickle object
ori = pdata['current_orientation'][0]
ucell = ori.unit_cell()
sg = pdata['pointgroup']
cbop = pdata['current_cb_op_to_primitive'][0]
xbeam = pdata['xbeam']
ybeam = pdata['ybeam']
wavelength = pdata['wavelength']
if 'effective_tiling' in pdata.keys():
tm_int = pdata['effective_tiling']
else:
tiling = image_info.tiling_from_image(detector_phil=detector_phil)
tm_int = tiling.effective_tiling_as_flex_int(
reapply_peripheral_margin=True, encode_inactive_as_zeroes=True)
xtal = symmetry(unit_cell=ucell, space_group="P1")
indices = xtal.build_miller_set(anomalous_flag=True, d_min=dmin)
params = parameters_bp3(
indices=indices.indices(),
orientation=ori,
incident_beam=col((0., 0., -1.)),
packed_tophat=col((1., 1., 0.)),
detector_normal=col((0., 0., -1.)),
detector_fast=col((0., 1., 0.)), # if buggy, try changing sign
detector_slow=col((1., 0., 0.)), # if buggy, try changing sign
pixel_size=col((pixel_size, pixel_size, 0.)),
pixel_offset=col((0.5, 0.5, 0.0)),
distance=pdata['distance'],
detector_origin=col(
(-ybeam, -xbeam, 0.))) # if buggy, try changing signs
ucbp3 = use_case_bp3(parameters=params)
# use the tiling manager above to construct the predictor parameters
ucbp3.set_active_areas(tm_int)
signal_penetration = 0.5 # from LG36 trial 94 params_2.phil
ucbp3.set_sensor_model(thickness_mm=0.1,
mu_rho=8.36644,
signal_penetration=signal_penetration)
# presume no subpixel corrections for now
ucbp3.prescreen_indices(wavelength)
ucbp3.set_orientation(ori)
ucbp3.set_mosaicity(pdata['ML_half_mosaicity_deg'][0] * math.pi /
180) # radians
ucbp3.set_domain_size(pdata['ML_domain_size_ang'][0])
bandpass = 1.E-3
wave_hi = wavelength * (1. - (bandpass / 2.))
wave_lo = wavelength * (1. + (bandpass / 2.))
ucbp3.set_bandpass(wave_hi, wave_lo)
ucbp3.picture_fast_slow()
# the full set of predictable reflections can now be accessed
predicted = ucbp3.selected_predictions_labelit_format()
hkllist = ucbp3.selected_hkls()
# construct the experiment list and other dials backbone to be able to write predictions
frame = construct_reflection_table_and_experiment_list(
int_pickle_path, img_path, pixel_size, proceed_without_image=True)
frame.assemble_experiments()
frame.assemble_reflections()
predictor = StillsReflectionPredictor(frame.experiment, dmin=1.5)
Rcalc = flex.reflection_table.empty_standard(len(hkllist))
Rcalc['miller_index'] = hkllist
expt_xtal = frame.experiment.crystal
predictor.for_reflection_table(Rcalc, expt_xtal.get_A())
predicted = Rcalc['xyzcal.mm']
# # apply the active area filter
# from iotbx.detectors.active_area_filter import active_area_filter
# active_areas = list(tm.effective_tiling_as_flex_int())
# active_area_object = active_area_filter(active_areas)
# aa_predicted, aa_hkllist = active_area_object(predicted, hkllist, 0.11)
# extended_mapped_predictions = flex.vec2_double()
# for i in range(len(aa_predicted)):
# extended_mapped_predictions.append(aa_predicted[i][0:2])
# return predictions without re-applying an active area filter
newpreds = flex.vec2_double()
for i in range(len(predicted)):
newpreds.append((predicted[i][0] / pixel_size,
img_size - predicted[i][1] / pixel_size))
# finally, record new predictions as member data
pdata['mapped_predictions_to_edge'] = newpreds
pdata['indices_to_edge'] = hkllist
if dump:
newpath = int_pickle_path.split(".pickle")[0] + "_extended.pickle"
easy_pickle.dump(newpath, pdata)
else:
return (hkllist, newpreds)
def read_data(self,params):
from os import listdir, path
from libtbx import easy_pickle
from cctbx.crystal_orientation import crystal_orientation # XXX Necessary later?
#directory = "/net/viper/raid1/hattne/L220/merging/05fs"
#directory = "/reg/d/psdm/cxi/cxib8113/scratch/sauter/metrology/008"
#directory = "/reg/d/psdm/xpp/xpp74813/scratch/sauter/metrology/204"
#directory = "/net/viper/raid1/hattne/L220/merging/test"
#directory = "/reg/d/psdm/xpp/xppb4313/scratch/brewster/results/r0243/003/integration"
#directory = "/reg/d/psdm/cxi/cxic0614/scratch/sauter/metrology/004/integration"
#directory = "/reg/d/psdm/cxi/cxic0614/scratch/sauter/metrology/150/integration"
#directory = "/reg/d/psdm/cxi/cxic0614/scratch/sauter/metrology/152/integration"
directory = "/reg/d/psdm/cxi/cxib6714/scratch/sauter/metrology/009/integration"
dir_glob = "/reg/d/psdm/CXI/cxib6714/scratch/sauter/results/r*/009/integration"
dir_glob = "/reg/d/psdm/CXI/cxib6714/scratch/sauter/results/r*/801/integration"
dir_glob = "/reg/d/psdm/xpp/xpp74813/scratch/sauter/r*/216/integration"
dir_glob = "/reg/d/psdm/xpp/xpp74813/ftc/sauter/result/r*/104/integration"
dir_glob = "/reg/d/psdm/cxi/cxid9114/scratch/sauter/metrology/001/integration"
dir_glob = "/reg/d/psdm/CXI/cxid9114/ftc/brewster/results/r00[3-4]*/003/integration"
dir_glob = "/reg/d/psdm/CXI/cxid9114/ftc/sauter/results/r00[3-4]*/004/integration"
dir_glob = "/reg/d/psdm/CXI/cxid9114/ftc/sauter/results/r00[3-4]*/006/integration"
dir_list = ["/reg/d/psdm/CXI/cxid9114/ftc/brewster/results/r%04d/006/integration"%seq for seq in range(95,115)]
dir_list = ["/reg/d/psdm/CXI/cxid9114/ftc/sauter/results/r%04d/018/integration"%seq for seq in range(102,115)]
dir_list = params.data
T = Timer("populate C++ store with register line")
itile = flex.int()
self.spotfx = flex.double()
self.spotfy = flex.double()
self.spotcx = flex.double()
self.spotcy = flex.double()
self.observed_cntr_x = flex.double()
self.observed_cntr_y = flex.double()
self.refined_cntr_x = flex.double()
self.refined_cntr_y = flex.double()
self.HKL = flex.miller_index()
self.radial = flex.double()
self.azimut = flex.double()
self.FRAMES = dict(
frame_id=flex.int(),
wavelength=flex.double(),
beam_x=flex.double(),
beam_y=flex.double(),
distance=flex.double(),
orientation=[],
rotation100_rad=flex.double(),
rotation010_rad=flex.double(),
rotation001_rad=flex.double(),
half_mosaicity_deg=flex.double(),
wave_HE_ang=flex.double(),
wave_LE_ang=flex.double(),
domain_size_ang=flex.double(),
unique_file_name=[]
)
self.frame_id = flex.int()
import glob
#for directory in glob.glob(dir_glob):
for directory in dir_list:
if self.params.max_frames is not None and len(self.FRAMES['frame_id']) >= self.params.max_frames:
break
for entry in listdir(directory):
tttd = d = easy_pickle.load(path.join(directory, entry))
# XXX Hardcoded, should honour the phil! And should be verified
# to be consistent for each correction vector later on!
#import pdb; pdb.set_trace()
setting_id = d['correction_vectors'][0][0]['setting_id']
#if setting_id != 5:
#if setting_id != 12:
if setting_id != self.params.bravais_setting_id:
#if setting_id != 22:
#print "HATTNE BIG SLIPUP 1"
continue
# Assert that effective_tiling is consistent, and a non-zero
# multiple of eight (only whole sensors considered for now--see
# mark10.fit_translation4.print_table()). self.tiles is
# initialised to zero-length in the C++ code. XXX Should now be
# able to retire the "effective_tile_boundaries" parameter.
#
# XXX Other checks from correction_vector plot, such as consistent
# setting?
if hasattr(self, 'tiles') and len(self.tiles) > 0:
assert (self.tiles == d['effective_tiling']).count(False) == 0
else:
assert len(d['effective_tiling']) > 0 \
and len(d['effective_tiling']) % 8 == 0
self.tiles = d['effective_tiling']
if not self.standalone_check(self,setting_id,entry,d,params.diff_cutoff): continue
# Reading the frame data. The frame ID is just the index of the
# image.
self.FRAMES['frame_id'].append(len(self.FRAMES['frame_id']) + 1) # XXX try zero-based here
self.FRAMES['wavelength'].append(d['wavelength'])
self.FRAMES['beam_x'].append(d['xbeam'])
self.FRAMES['beam_y'].append(d['ybeam'])
self.FRAMES['distance'].append(d['distance'])
self.FRAMES['orientation'].append(d['current_orientation'][0])
self.FRAMES['rotation100_rad'].append(0) # XXX FICTION
self.FRAMES['rotation010_rad'].append(0) # XXX FICTION
self.FRAMES['rotation001_rad'].append(0) # XXX FICTION
self.FRAMES['half_mosaicity_deg'].append(0) # XXX FICTION
# self.FRAMES['wave_HE_ang'].append(0.995 * d['wavelength']) # XXX FICTION -- what does Nick use?
# self.FRAMES['wave_LE_ang'].append(1.005 * d['wavelength']) # XXX FICTION
self.FRAMES['wave_HE_ang'].append(d['wavelength'])
self.FRAMES['wave_LE_ang'].append(d['wavelength'])
self.FRAMES['domain_size_ang'].append(5000) # XXX FICTION
self.FRAMES['unique_file_name'].append(path.join(directory, entry))
print "added frame", self.FRAMES['frame_id'][-1],entry
for cv in d['correction_vectors'][0]:
# Try to reproduce every predicition using the model from the
# frame -- skip CV if fail. Could be because of wrong HKL:s?
#
# Copy these two images to test directory to reproduce:
# int-s01-2011-02-20T21:27Z37.392_00000.pickle
# int-s01-2011-02-20T21:27Z37.725_00000.pickle
from rstbx.bandpass import use_case_bp3, parameters_bp3
from scitbx.matrix import col
from math import hypot, pi
indices = flex.miller_index()
indices.append(cv['hkl'])
parameters = parameters_bp3(
indices=indices,
orientation=self.FRAMES['orientation'][-1],
incident_beam=col(self.INCIDENT_BEAM),
packed_tophat=col((1.,1.,0.)),
detector_normal=col(self.DETECTOR_NORMAL),
detector_fast=col((0.,1.,0.)),detector_slow=col((1.,0.,0.)),
pixel_size=col((0.11,0.11,0)), # XXX hardcoded, twice!
pixel_offset=col((0.,0.,0.0)),
distance=self.FRAMES['distance'][-1],
detector_origin=col((-self.FRAMES['beam_x'][-1],
-self.FRAMES['beam_y'][-1],
0))
)
ucbp3 = use_case_bp3(parameters=parameters)
ucbp3.set_active_areas(self.tiles)
integration_signal_penetration=0.5
ucbp3.set_sensor_model(thickness_mm=0.5,
mu_rho=8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration=integration_signal_penetration)
half_mosaicity_rad = self.FRAMES['half_mosaicity_deg'][-1] * pi/180.
ucbp3.set_mosaicity(half_mosaicity_rad)
ucbp3.set_bandpass(self.FRAMES['wave_HE_ang'][-1],
self.FRAMES['wave_LE_ang'][-1])
ucbp3.set_orientation(self.FRAMES['orientation'][-1])
ucbp3.set_domain_size(self.FRAMES['domain_size_ang'][-1])
ucbp3.picture_fast_slow_force()
ucbp3_prediction = 0.5 * (ucbp3.hi_E_limit + ucbp3.lo_E_limit)
diff = hypot(ucbp3_prediction[0][0] - cv['predspot'][1],
ucbp3_prediction[0][1] - cv['predspot'][0])
if diff > self.params.diff_cutoff:
print "HATTNE INDEXING SLIPUP"
continue
# For some reason, the setting_id is recorded for each
# correction vector as well--assert that it is consistent.
#if cv['setting_id'] != setting_id:
# print "HATTNE BIG SLIPUP 2"
assert cv['setting_id'] == setting_id
# For each observed spot, figure out what tile it is on, and
# store in itile. XXX This is probably not necessary here, as
# correction_vector_store::register_line() does the same thing.
obstile = None
for i in range(0, len(self.tiles), 4):
if cv['obsspot'][0] >= self.tiles[i + 0] \
and cv['obsspot'][0] <= self.tiles[i + 2] \
and cv['obsspot'][1] >= self.tiles[i + 1] \
and cv['obsspot'][1] <= self.tiles[i + 3]:
obstile = i
break
assert obstile is not None
itile.append(obstile) # XXX unused variable?
# ID of current frame.
self.frame_id.append(self.FRAMES['frame_id'][-1])
self.spotfx.append(cv['obsspot'][0])
self.spotfy.append(cv['obsspot'][1])
self.spotcx.append(cv['predspot'][0])
self.spotcy.append(cv['predspot'][1])
self.observed_cntr_x.append(cv['obscenter'][0])
self.observed_cntr_y.append(cv['obscenter'][1])
self.refined_cntr_x.append(cv['refinedcenter'][0])
self.refined_cntr_y.append(cv['refinedcenter'][1])
self.HKL.append(cv['hkl'])
self.azimut.append(cv['azimuthal'])
self.radial.append(cv['radial'])
#print self.FRAMES['frame_id'][-1]
# Should honour the max_frames phil parameter
#if len(self.FRAMES['frame_id']) >= 1000:
if self.params.max_frames is not None and \
len(self.FRAMES['frame_id']) >= self.params.max_frames:
break
"""
For 5000 first images:
STATS FOR TILE 14
sel_delx -6.59755265524 -4.41676757746e-10 5.7773557278
sel_dely -6.30796620634 -8.3053734774e-10 6.3362200841
symmetric_offset_x -6.5975526548 -2.73229417105e-15 5.77735572824
symmetric_offset_y -6.30796620551 1.16406818748e-15 6.33622008493
symmetric rsq 0.000255199593417 2.95803352999 56.1918083904
rmsd 1.71989346472
For 10000 first images:
STATS FOR TILE 14
sel_delx -6.92345292727 6.9094552919e-10 611.497770006
sel_dely -6.39690476093 1.1869355797e-09 894.691806871
symmetric_offset_x -6.92345292796 1.28753258216e-14 611.497770005
symmetric_offset_y -6.39690476212 -2.10251420168e-15 894.69180687
symmetric rsq 1.58067791823e-05 30.3331143761 1174402.952
rmsd 5.50755066941
"""
# This is mark3.fit_translation2.nominal_tile_centers()
self.To_x = flex.double(len(self.tiles) // 4)
self.To_y = flex.double(len(self.tiles) // 4)
for x in range(len(self.tiles) // 4):
self.To_x[x] = (self.tiles[4 * x + 0] + self.tiles[4 * x + 2]) / 2
self.To_y[x] = (self.tiles[4 * x + 1] + self.tiles[4 * x + 3]) / 2
delx = self.spotcx - self.spotfx
dely = self.spotcy - self.spotfy
self.delrsq = self.delrsq_functional(calcx = self.spotcx, calcy = self.spotcy)
self.initialize_per_tile_sums()
self.tile_rmsd = [0.]*(len(self.tiles) // 4)
self.asymmetric_tile_rmsd = [0.]*(len(self.tiles) // 4)
# XXX Is (beam1x, beam1y) really observed center and (beamrx,
# beamry) refined center? Nick thinks YES!
#
#itile2 = flex.int([self.register_line(a[2],a[3],a[4],a[5],a[6],a[7],a[8],a[9]) for a in ALL])
itile2 = flex.int(
[self.register_line(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7])
for a in zip(self.observed_cntr_x, self.observed_cntr_y,
self.refined_cntr_x, self.refined_cntr_y,
self.spotfx, self.spotfy,
self.spotcx, self.spotcy)])
if params.show_consistency: consistency_controls(self,params)
T = Timer("calcs based on C++ store")
self.selections = []
self.selection_counts = []
for x in range(len(self.tiles) // 4):
if self.tilecounts[x]==0:
self.radii[x] = 0
self.mean_cv[x] = matrix.col((0, 0))
else:
self.radii[x]/=self.tilecounts[x]
self.mean_cv[x] = matrix.col(self.mean_cv[x]) / self.tilecounts[x]
selection = (self.master_tiles == x)
self.selections.append(selection)
selected_cv = self.master_cv.select(selection)
self.selection_counts.append(selected_cv.size()) # for curvatures
if len(selected_cv)>0:
self.asymmetric_tile_rmsd[x] = math.sqrt(flex.mean (self.delrsq.select(selection)))
sel_delx = delx.select(selection)
sel_dely = dely.select(selection)
symmetric_offset_x = sel_delx - self.mean_cv[x][0]
symmetric_offset_y = sel_dely - self.mean_cv[x][1]
symmetricrsq = symmetric_offset_x*symmetric_offset_x + symmetric_offset_y*symmetric_offset_y
self.tile_rmsd[x] =math.sqrt(flex.mean(symmetricrsq))
else:
self.asymmetric_tile_rmsd[x]=0.
self.tile_rmsd[x]=0.
self.overall_N = flex.sum(flex.int( [int(t) for t in self.tilecounts] ))
self.overall_cv = matrix.col(self.overall_cv)/self.overall_N
self.overall_rmsd = math.sqrt( self.sum_sq_cv / self.overall_N )
# master weights for mark3 calculation takes 0.3 seconds
self.master_weights = flex.double(len(self.master_tiles))
self.largest_sample = max(self.tilecounts)
for x in range(len(self.tiles) // 4):
self.master_weights.set_selected( self.selections[x], self.tile_weight(x))
print "AFTER read cx, cy", flex.mean(self.spotcx), flex.mean(self.spotcy)
print "AFTER read fx, fy", flex.mean(self.spotfx), flex.mean(self.spotfy)
print "AFTER read rmsd_x, rmsd_y", math.sqrt(flex.mean(flex.pow(self.spotcx - self.spotfx, 2))), \
math.sqrt(flex.mean(flex.pow(self.spotcy - self.spotfy, 2)))
return
def standalone_check(self,setting_id,entry,d,cutoff):
wavelength = (d['wavelength'])
beam_x = (d['xbeam'])
beam_y = (d['ybeam'])
distance = (d['distance'])
orientation = (d['current_orientation'][0])
print "testing frame....................",entry
for cv in d['correction_vectors'][0]:
from rstbx.bandpass import use_case_bp3, parameters_bp3
from scitbx.matrix import col
from math import hypot, pi
indices = flex.miller_index()
indices.append(cv['hkl'])
parameters = parameters_bp3(
indices=indices,
orientation=orientation,
incident_beam=col(self.INCIDENT_BEAM),
packed_tophat=col((1.,1.,0.)),
detector_normal=col(self.DETECTOR_NORMAL),
detector_fast=col((0.,1.,0.)),detector_slow=col((1.,0.,0.)),
pixel_size=col((0.11,0.11,0)), # XXX hardcoded, twice!
pixel_offset=col((0.,0.,0.0)),
distance=distance,
detector_origin=col((-beam_x,-beam_y,0))
)
ucbp3 = use_case_bp3(parameters=parameters)
ucbp3.set_active_areas(self.tiles)
integration_signal_penetration=0.5
ucbp3.set_sensor_model(thickness_mm=0.5,
mu_rho=8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration=integration_signal_penetration)
ucbp3.set_mosaicity(0.)
ucbp3.set_bandpass(wavelength,
wavelength)
ucbp3.set_orientation(orientation)
ucbp3.set_domain_size(5000.)
ucbp3.picture_fast_slow_force()
ucbp3_prediction = 0.5 * (ucbp3.hi_E_limit + ucbp3.lo_E_limit)
diff = hypot(ucbp3_prediction[0][0] - cv['predspot'][1],
ucbp3_prediction[0][1] - cv['predspot'][0])
if diff > cutoff:
print "Correction vector too long: %6.2f pixels; ignore image or increase diff_cutoff (current value=%5.1f)"%(diff,cutoff)
return False
# For some reason, the setting_id is recorded for each
# correction vector as well--assert that it is consistent.
#if cv['setting_id'] != setting_id:
# print "HATTNE BIG SLIPUP 2"
if not cv['setting_id'] == setting_id: return False
# For each observed spot, figure out what tile it is on, and
# store in itile. XXX This is probably not necessary here, as
# correction_vector_store::register_line() does the same thing.
obstile = None
for i in range(0, len(self.tiles), 4):
if cv['obsspot'][0] >= self.tiles[i + 0] \
and cv['obsspot'][0] <= self.tiles[i + 2] \
and cv['obsspot'][1] >= self.tiles[i + 1] \
and cv['obsspot'][1] <= self.tiles[i + 3]:
obstile = i
break
if obstile is None: return False
spotfx = (cv['obsspot'][0])
spotfy = (cv['obsspot'][1])
spotcx = (cv['predspot'][0])
spotcy = (cv['predspot'][1])
correction_vector_x = spotcx - spotfx
correction_vector_y = spotcy - spotfy
length = hypot(correction_vector_x, correction_vector_y)
if length > 8:
print "LENGTH SLIPUP",length
return False
return True
def parameter_based_model(self,params):
PIXEL_SZ = 0.11 # mm/pixel
all_model = mark3_collect_data(self.frame_id, self.HKL)
for iframe in xrange(len(self.FRAMES["frame_id"])):
frame_id = self.FRAMES["frame_id"][iframe]
if not self.bandpass_models.has_key(frame_id):
reserve_orientation = self.FRAMES["orientation"][iframe]
effective_orientation = reserve_orientation
#Not necessary to apply the 3 offset rotations; they have apparently
# been applied already.\
# .rotate_thru((1,0,0),self.FRAMES["rotation100_rad"][iframe]
# ).rotate_thru((0,1,0),self.FRAMES["rotation010_rad"][iframe]
# ).rotate_thru((0,0,1),self.FRAMES["rotation001_rad"][iframe])
detector_origin = col((-self.FRAMES["beam_x"][iframe],
-self.FRAMES["beam_y"][iframe], 0.))
crystal = symmetry(unit_cell=effective_orientation.unit_cell(),space_group = "P1")
indices = all_model.frame_indices(frame_id)
parameters = parameters_bp3(
indices=indices, orientation=effective_orientation,
incident_beam=col(correction_vectors.INCIDENT_BEAM),
packed_tophat=col((1.,1.,0.)),
detector_normal=col(correction_vectors.DETECTOR_NORMAL),
detector_fast=col((0.,1.,0.)),detector_slow=col((1.,0.,0.)),
pixel_size=col((PIXEL_SZ,PIXEL_SZ,0)),
pixel_offset=col((0.,0.,0.0)),
distance=self.FRAMES["distance"][iframe],
detector_origin=detector_origin
)
ucbp3 = use_case_bp3(parameters=parameters)
ucbp3.set_active_areas( self.tiles ) #params.effective_tile_boundaries
integration_signal_penetration=0.5
ucbp3.set_sensor_model( thickness_mm = 0.5, mu_rho = 8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration = integration_signal_penetration)
half_mosaicity_rad = self.FRAMES["half_mosaicity_deg"][iframe] * pi/180.
ucbp3.set_mosaicity(half_mosaicity_rad)
ucbp3.set_bandpass(self.FRAMES["wave_HE_ang"][iframe],self.FRAMES["wave_LE_ang"][iframe])
ucbp3.set_orientation(effective_orientation)
ucbp3.set_domain_size(self.FRAMES["domain_size_ang"][iframe])
ucbp3.picture_fast_slow_force()
self.bandpass_models[frame_id]=ucbp3
all_model.collect(self.bandpass_models[frame_id].hi_E_limit,
self.bandpass_models[frame_id].lo_E_limit,
self.bandpass_models[frame_id].observed_flag,
frame_id);
sq_displacements = ((all_model.cx - self.spotcx)*(all_model.cx - self.spotcx) +
(all_model.cy - self.spotcy)*(all_model.cy - self.spotcy))
selected_sq_displacements = sq_displacements.select( all_model.flags == True )
print "Root Mean squared displacement all spots %8.3f"%math.sqrt(
flex.sum(selected_sq_displacements)/len(selected_sq_displacements))
return all_model.cx, all_model.cy, all_model.flags
