def write_pae_file(pae_matrix, file_name, range_to_keep=None):
pae_as_list_of_lists = pae_matrix.tolist()
if range_to_keep:
n = range_to_keep.end + 1 - range_to_keep.start
shape = (n, n)
pae_1d = []
for l in pae_as_list_of_lists[range_to_keep.start:range_to_keep.end +
1]:
new_l = l[range_to_keep.start:range_to_keep.end + 1]
pae_1d += new_l
else:
assert type(pae_matrix.tolist()[0][0]) == type(float(1))
shape = tuple(pae_matrix.shape)
# Flatten it out
pae_1d = pae_matrix.flatten().tolist()
# Read in to flex array
flex_array = flex.float(pae_1d)
from scitbx.array_family.flex import grid
flex_grid = grid(shape)
n, n = shape
# Reshape the flex array
flex_array.reshape(flex_grid)
# Write out array to text file as json
residues_1 = []
residues_2 = []
distances = []
for i in range(n):
ii = i + 1
for j in range(n):
jj = j + 1
residues_1.append(ii)
residues_2.append(jj)
distances.append(float("%.2f" % (flex_array[i, j])))
residue_dict = {
"residue1": residues_1,
"residue2": residues_2,
"distance": distances,
"max_predicted_aligned_error": 0
}
values = [residue_dict]
text = str(values).replace(" ", "").replace("'", '"')
f = open(file_name, 'w')
print(text, file=f)
f.close()
print("Wrote json file to %s" % (file_name))
def __init__(self,
seq_a,
seq_b,
style="global",
gap_opening_penalty=1,
gap_extension_penalty=1,
similarity_function="identity",
masking_a=None):
assert style in ["global", "local", "no_end_gaps"]
sim_fun_str = similarity_function
self.similarity_function_call = None
if (similarity_function is None or similarity_function == "identity"):
self.similarity_function_call = identity
elif (similarity_function == "dayhoff"):
self.similarity_function_call = dayhoff
elif (similarity_function == "blosum50"):
self.similarity_function_call = blosum50
elif (isinstance(similarity_function, str)):
raise RuntimeError('Unknown similarity_function: "%s"' %
similarity_function)
from six import string_types
if isinstance(seq_a, string_types):
seq_a = seq_a.upper()
seq_b = seq_b.upper()
else:
seq_a = flex.std_string(seq_a)
seq_b = flex.std_string(seq_b)
if masking_a: # Gap penalty for insertion scaled by masking_a after
# corresponding position in sequence a and scaled by
# masking_a for deletion before corresponding position in
# sequence a. Allows increasing mask everywhere
# except where you want a gap to occur
assert len(list(masking_a)) == len(seq_a)
else:
masking_a = [1] * len(
seq_a) # no masking; standard gap penalty everywhere
m = flex.float(masking_a)
super(align, self).__init__(
seq_a=seq_a,
seq_b=seq_b,
masking=m,
style=style,
gap_opening_penalty=gap_opening_penalty,
gap_extension_penalty=gap_extension_penalty,
similarity_function=sim_fun_str,
)
self.seq_a = seq_a
self.seq_b = seq_b
self.style = style
self.similarity_function = sim_fun_str
m, n = self.m, self.n = len(seq_a), len(seq_b)
def createFromText(infile):
'''Loads rotamer or Ramachandran data from a text file, returning a new object.
Can pass in either a file handle or a file name'''
if isinstance(infile, str):
infile = open(infile, 'r')
ndt = NDimTable()
ndt.ourName = re.search(r': +"(.+)"$', infile.readline()).group(1)
ndt.nDim = int(re.search(r': +(\d+)$', infile.readline()).group(1))
infile.readline() # lower_bound upper_bound number_of_bins wrapping
ndt.minVal = []
ndt.maxVal = []
ndt.nBins = []
ndt.doWrap = []
ndt.wBin = []
data_re = re.compile(r': +([^ ]+) +([^ ]+) +([^ ]+) +([^ ]+)$')
for i in range(ndt.nDim):
match = data_re.search(infile.readline())
ndt.minVal.append(float(match.group(1)))
ndt.maxVal.append(float(match.group(2)))
ndt.nBins.append(int(match.group(3)))
doWrap = match.group(4).lower().strip()
if doWrap == "true" or doWrap == "yes" or doWrap == "on" or doWrap == "1":
ndt.doWrap.append(True)
else:
ndt.doWrap.append(False)
ndt.wBin.append((ndt.maxVal[i] - ndt.minVal[i]) / ndt.nBins[i])
#print ndt.minVal, ndt.maxVal, ndt.nBins, ndt.doWrap, ndt.wBin
ndt.lookupTable = flex.float(flex.grid(ndt.nBins), 0)
if re.search(r'first', infile.readline()): valueFirst = True
else: valueFirst = False
#print valueFirst
s = infile.readline()
while s:
fields = s.split()
if (len(fields) <= ndt.nDim): continue
if valueFirst:
val = float(fields[0])
coords = [float(fld) for fld in fields[1:ndt.nDim + 1]]
else:
val = float(fields[ndt.nDim])
coords = [float(fld) for fld in fields[0:ndt.nDim]]
#print val, coords
ndt.setValueAt(ndt.whereIs(coords), val)
s = infile.readline()
return ndt
def createFromText(infile):
'''Loads rotamer or Ramachandran data from a text file, returning a new object.
Can pass in either a file handle or a file name'''
if isinstance(infile, types.StringTypes):
infile = file(infile, 'r')
ndt = NDimTable()
ndt.ourName = re.search(r': +"(.+)"$', infile.readline()).group(1)
ndt.nDim = int(re.search(r': +(\d+)$', infile.readline()).group(1))
infile.readline() # lower_bound upper_bound number_of_bins wrapping
ndt.minVal = []
ndt.maxVal = []
ndt.nBins = []
ndt.doWrap = []
ndt.wBin = []
data_re = re.compile(r': +([^ ]+) +([^ ]+) +([^ ]+) +([^ ]+)$')
for i in range(ndt.nDim):
match = data_re.search(infile.readline())
ndt.minVal.append(float(match.group(1)))
ndt.maxVal.append(float(match.group(2)))
ndt.nBins.append( int(match.group(3)))
doWrap = match.group(4).lower().strip()
if doWrap == "true" or doWrap == "yes" or doWrap == "on" or doWrap == "1": ndt.doWrap.append(True)
else: ndt.doWrap.append(False)
ndt.wBin.append((ndt.maxVal[i] - ndt.minVal[i]) / ndt.nBins[i])
#print ndt.minVal, ndt.maxVal, ndt.nBins, ndt.doWrap, ndt.wBin
ndt.lookupTable = flex.float(flex.grid(ndt.nBins), 0)
if re.search(r'first', infile.readline()): valueFirst = True
else: valueFirst = False
#print valueFirst
s = infile.readline()
while s:
s = infile.readline()
fields = s.split()
if(len(fields) <= ndt.nDim): continue
if valueFirst:
val = float(fields[0])
coords = [float(fld) for fld in fields[1:ndt.nDim+1]]
else:
val = float(fields[ndt.nDim])
coords = [float(fld) for fld in fields[0:ndt.nDim]]
#print val, coords
ndt.setValueAt( ndt.whereIs(coords), val )
return ndt
def get_raw_data(self, index=0):
from scitbx.array_family import flex
detector_2d_assembled_1 = self._run['detector_2d_assembled_1']
tag = detector_2d_assembled_1[self._images[index]]
data = tag['detector_data']
return flex.float(data[:, :]).as_double()
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
For now, log error and set bogus value to allow stuff to continue
-- must check for the bogosity later
XXX The dead time of the detector complicates checking how often
things are updated! Move this to the ring buffer?
@param evt Event data object, a configure object
@param env Environment object
"""
from pyana.event import Event
from acqiris_ext import acqiris_integrate, apd_hitfind
super(mod_ledge, self).event(evt, env)
if evt.status() != Event.Normal:
pass # XXX return -- Never skip because arrays will end up
# different length, so ignore this?
# Get the time of the event, in fractional seconds since the
# epoch. This is needed for all subsequent history-keeping, and
# is hence determined first. XXX Is history-keeping even
# justified?
time = cspad_tbx.evt_time(evt)
if time is None:
time = float("nan")
else:
time = time[0] + time[1] / 1e3
self._timestamp.append(time)
# The repetition rate is currently just used for sanity checking.
repetition_rate = cspad_tbx.evt_repetition_rate(evt)
if repetition_rate is None:
repetition_rate = float("nan")
self._repetition_rate.append(repetition_rate)
# Get the I0. No need to warn about it here, it will be done once
# the image is written out.
I0 = cspad_tbx.evt_pulse_energy(evt)
if I0 is None:
I0 = float("nan")
self._I0.append(I0)
# Get the FEE energy. Average the two readings before and after
# attenuation separately. XXX What are the units? It look like
# it could be mJ?
fee_before = 0.5 * sum(evt.getFeeGasDet()[0:2])
if fee_before is None:
fee_before = float("nan")
self._fee_before.append(fee_before)
fee_after = 0.5 * sum(evt.getFeeGasDet()[2:4])
if fee_after is None:
fee_after = float("nan")
self._fee_after.append(fee_after)
# XXX Just a check: this is what xtcexplorer does:
fee_energy = evt.get(xtc.TypeId.Type.Id_FEEGasDetEnergy)
if fee_energy is not None:
assert (
evt.getFeeGasDet()[0] == fee_energy.f_11_ENRC
and evt.getFeeGasDet()[1] == fee_energy.f_12_ENRC
and evt.getFeeGasDet()[2] == fee_energy.f_21_ENRC
and evt.getFeeGasDet()[3] == fee_energy.f_22_ENRC
)
"""
# For Bill: expect 84240 data points for r0054
#
# grep "^BILL_POINT" | cut -d' ' -f2,3,4,5,6 > t.dat
# gnuplot> m=0.1 ; k=-0.01e-8; f(x) = k * x + m
# gnuplot> fit f(x) "t.dat" using ($3):($5) via k,m
if not hasattr(self, '_gmd_seqno'):
self._gmd_seqno = 0
gmd = evt.get(key=xtc.TypeId.Type.Id_GMD)
if gmd is None:
return
acq_apd = evt.getAcqValue('SxrEndstation-0|Acqiris-1', 0, env)
if acq_apd is not None and acq_apd.waveform() is not None:
w = acq_apd.waveform()
baseline = numpy.mean(w[0:(w.shape[0] / 5)])
peak = numpy.min(w[(w.shape[0] / 5):w.shape[0]])
self._gmd_seqno += 1
print "BILL_POINT %d %s %s %s %s" % (self._gmd_seqno,
repr(gmd.fBgValuePerSample),
repr(gmd.fCorrectedSumPerPulse),
repr(gmd.fRelativeEnergyPerPulse),
repr(peak - baseline))
return
"""
"""
# XXX Record injector motion--note that they cannot be added--see
# Ray's email.
injector_micos_xyz = cspad_tbx.env_pv3_get(
env,
['SXR:EXP:MZM:%02d:ENCPOSITIONGET' % i for i in [1, 2, 3]])
if injector_micos_xyz is None:
self.logger.error("No micos injector motor positions")
injector_micos_xyz = (float('nan'), float('nan'), float('nan'))
self._injector_micos_xyz.append(injector_micos_xyz)
injector_rough_xyz = cspad_tbx.env_pv3_get(
env,
['SXR:EXP:MMS:%02d.RBV' % i for i in [1, 2, 3]])
if injector_rough_xyz is None:
self.logger.error("No rough injector motor positions")
injector_rough_xyz = (float('nan'), float('nan'), float('nan'))
self._injector_rough_xyz.append(injector_rough_xyz)
# Injector power supplies XXX There is a third PSU, no?
#
# The -5kV supply
# SXR:EXP:SHV:VHS6:CH0:VoltageMeasure
# SXR:EXP:SHV:VHS6:CH0:CurrentMeasure
#
# The plus 5kV supply
# SXR:EXP:SHV:VHS2:CH0:VoltageMeasure
# SXR:EXP:SHV:VHS2:CH0:CurrentMeasure
injector_plus_current = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS6:CH0:CurrentMeasure')
if injector_plus_current is None:
self.logger.error("No plus-motor current")
injector_plus_current = -1
self._injector_plus_current.append(injector_plus_current)
injector_plus_voltage = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS6:CH0:VoltageMeasure')
if injector_plus_voltage is None:
self.logger.error("No plus-motor voltage")
injector_plus_voltage = -1
self._injector_plus_voltage.append(injector_plus_voltage)
injector_minus_current = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS2:CH0:CurrentMeasure')
if injector_minus_current is None:
self.logger.error("No minus-motor current")
injector_minus_current = -1
self._injector_minus_current.append(injector_minus_current)
injector_minus_voltage = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS2:CH0:VoltageMeasure')
if injector_minus_voltage is None:
self.logger.error("No minus-motor voltage")
injector_minus_voltage = -1
self._injector_minus_voltage.append(injector_minus_voltage)
"""
"""
# The spectrometer motor positions are just used for sanity
# checking.
spectrometer_xyz = cspad_tbx.env_spectrometer_xyz_sxr(env)
if spectrometer_xyz is None:
self.logger.error("No spectrometer motor positions")
spectrometer_xyz = (float('nan'), float('nan'), float('nan'))
self._spectrometer_xyz.append(spectrometer_xyz)
"""
# Get the pulse energy after monochromator, and fall back on the
# pre-monochromator energy if the former is absent. Record in
# list for mean and stddev. XXX Verify that the wavelength after
# the monochromator is updated at around 1 Hz.
#
# For the publication an offset and scale were calibrated.
wavelength = cspad_tbx.env_wavelength_sxr(evt, env)
if wavelength is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
energy = float("nan")
else:
energy = 12398.4187 / wavelength
self._energy.append(energy)
self._history_energy.push(time, energy) # XXX Not necessary?!
"""
# Laser shutters XXX need to sort out laser numbering XXX Laser
# power stuff? XXX Position of polarizer/analyser
shutters = cspad_tbx.env_laser_shutters(env)
#print "Got shutters", shutters
"""
# Read out the diode traces from the via the Acqiris. XXX In any
# case, the APD and the more sensitive Opto Diode in the monitor
# tank (i.e. the transmission diode) should be anti-correlated, so
# check it! The entire trace always covers 10 us. XXX Could this
# be figured out from xtc.TypeId.Type.Id_AcqConfig?
#
# XXX This appears to be suboptimal: look at the
# skewness-transform for the APD to sort this out.
acq_apd = evt.getAcqValue("SxrEndstation-0|Acqiris-1", 0, env)
acq_apd_integral = float("nan")
if acq_apd is not None:
waveform = acq_apd.waveform()
if waveform is not None:
# With a 40k-point trace, one should integrate from 18200 to
# 18400.
waveform = waveform.flatten()
nmemb = len(waveform) // 200
if nmemb > 0:
acq_apd_integral = acqiris_integrate(flex.double(waveform), 91 * nmemb, 100 * nmemb, nmemb)
self._acq_apd_integral.append(acq_apd_integral)
if evt.expNum() == 208:
# Opto diode address for L632.
acq_opto_diode = evt.getAcqValue("SxrEndstation-0|Acqiris-1", 1, env)
elif evt.expNum() == 363:
# Opto diode address for LB68.
acq_opto_diode = evt.getAcqValue("SxrEndstation-0|Acqiris-2", 2, env)
acq_opto_diode_integral = float("nan")
if acq_opto_diode is not None:
waveform = acq_opto_diode.waveform()
if waveform is not None:
# With a 40k-point trace, one should integrate from 16000 to
# 24000. With a 20k-point trace, a suitable integration
# region is bounded by 8000 and 12000. There is no need for
# thresholding, because the integral of the Opto Diode will
# not be used for hit finding. XXX What are the "misses" we
# record on the Opto Diode? XXX The direct beam is completely
# gone after it hits the sample, because soft X-rays.
waveform = waveform.flatten()
nmemb = len(waveform) // 5
if nmemb > 0:
acq_opto_diode_integral = acqiris_integrate(flex.double(waveform), 2 * nmemb, 4 * nmemb, nmemb)
self._acq_opto_diode_integral.append(acq_opto_diode_integral)
# Sanity check: verify that the timestamps for the two Acqiris
# traces are similar enough.
if acq_apd is not None and acq_opto_diode is not None:
assert (
len(acq_apd.timestamps()) == len(acq_opto_diode.timestamps())
and numpy.any(numpy.abs(acq_apd.timestamps() - acq_opto_diode.timestamps())) < 1e-6
)
# self.logger.info("DIODE INTEGRALS: %f %f %f" % (I0, acq_apd_integral, acq_opto_diode_integral))
"""
import matplotlib.pyplot as plt
hit_array_apd = apd_hitfind(
flex.double(acq_apd.waveform()),
len(acq_apd.waveform()) // 5)
hit_array_opto_diode = apd_hitfind(
flex.double(acq_opto_diode.waveform()),
len(acq_opto_diode.waveform()) // 5)
fig = plt.figure()
ax = fig.add_subplot(111)
#ax.plot(
# range(len(acq_apd.timestamps())), acq_apd.waveform())
ax.plot(
range(len(acq_opto_diode.timestamps())), acq_opto_diode.waveform()[0, :])
plt.show()
fig = plt.figure()
ax = fig.add_subplot(111)
#ax.plot(
# acq_apd.timestamps()[0:len(hit_array_apd)], hit_array)
ax.plot(
acq_opto_diode.timestamps()[0:len(hit_array_opto_diode)], hit_array)
plt.show()
"""
# Determine whether the beam hit the sample, and register the
# outcome. If not using any diodes for hit-finding, every shot is
# assumed to be a hit. XXX Unfortunately, this crucial piece is
# very unreliable. The threshold for the APD needs to be
# verified--inspect all the histograms. XXX hitfind_flags is
# probable better as a module parameter.
# hitfind_flags = 0x3
hitfind_flags = 0
hit = False
if not hitfind_flags:
hit = True
elif hitfind_flags & 0x1 and acq_apd_integral > 0.2:
hit = True
self._hit.append(hit)
# Always proceed all the way through (even if some shots have
# invalid values of e.g. I0) because images are precious. XXX
# Must reset counters before returning! XXX What about skipping
# all of the above if display is True?
if self.cspad_img is not None:
self._nframes += 1
"""
# The spectrometer should not move!
t = (self._spectrometer_xyz -
self._spectrometer_xyz.mean()).rms_length()
print "Spectrometer displacement", t
# Fine/rough motor position deviations from the mean. See Ray's
# email.
t = (self._injector_micos_xyz -
self._injector_micos_xyz.mean()).rms_length()
print "Injector micos displacement", t
t = (self._injector_rough_xyz -
self._injector_rough_xyz.mean()).rms_length()
print "Injector rough displacement", t
# Injector motor position means and deviations
if self._injector_plus_current.size() > 1:
t = flex.mean_and_variance(self._injector_plus_current)
print "Injector plus current mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
if self._injector_plus_voltage.size() > 1:
t = flex.mean_and_variance(self._injector_plus_voltage)
print "Injector plus voltage mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
if self._injector_minus_current.size() > 1:
t = flex.mean_and_variance(self._injector_minus_current)
print "Injector minus current mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
if self._injector_minus_voltage.size() > 1:
t = flex.mean_and_variance(self._injector_minus_voltage)
print "Injector minus voltage mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
"""
# Energy statistics are collected from all shots, regardless of
# whether they are hits or not. Since this statistic mentions
# the frame number, it should be reported first. XXX The energy
# should have a really small standard deviation. Check
# self._energy.size() and self._history_energy.frequency() XXX
# verify that it works for one data point.
(energy_mean, energy_stddev, energy_nmemb, n) = self._filtered_stats(
lambda x: not math.isnan(x) and x > 0, self._energy
)
if n > 0:
self.logger.warning("%d shots have undefined energy" % n)
(I0_mean, I0_stddev, I0_nmemb, n) = self._filtered_stats(lambda x: not math.isnan(x), self._I0)
if n > 0:
self.logger.warning("%d shots have undefined I0" % n)
self.logger.info(
"Frame %d: E=%.3f+/-%.3f (N=%d) I0=%.0f+/-%.0f (N=%d)"
% (self._nframes, energy_mean, energy_stddev, energy_nmemb, I0_mean, I0_stddev, I0_nmemb)
)
# Sanity check: unless changed while integrating the frame, the
# repetition rate should have a standard deviation of zero.
dt = self._timestamp[-1] - self._timestamp[0]
rr_mean = rr_observed = rr_stddev = 0
if dt > 0:
rr_observed = (len(self._timestamp) - 1) / dt
rr = filter(lambda x: not math.isnan(x) and x > 0, self._repetition_rate)
if len(rr) > 1:
rr_stats = flex.mean_and_variance(flex.double(rr))
rr_mean = rr_stats.mean()
rr_stddev = rr_stats.unweighted_sample_standard_deviation()
self.logger.info(
"Repetition rate: %.3f Hz (observed), %.3f+/-%.3f Hz (expected)" % (rr_observed, rr_mean, rr_stddev)
)
# Compare observed and configured exposure time.
config = cspad_tbx.getConfig(self.address, env)
exposure_time = 0
if config is not None and dt > 0 and len(self._timestamp) > 0:
exposure_time = dt * (len(self._timestamp) + 1) / len(self._timestamp)
self.logger.info(
"Exposure time: %.3f s (observed), %.3f s (configured)" % (exposure_time, config.exposureTime())
)
# Compute the leading dead time, the time between starting the
# readout of the previous frame and the arrival of the shot
# immediately following it. This is an interesting statistic,
# no matter what. XXX Maybe look at its distribution?
dead_time = 0
if rr_observed > 0 and hasattr(self, "_previous_readout_time"):
dead_time = self._timestamp[0] - self._previous_readout_time - 1 / rr_observed
if math.isnan(dead_time):
dead_time = 0
self.logger.info("Dead time: %.3f s" % dead_time)
self._previous_readout_time = self._timestamp[-1]
assert time == self._timestamp[-1] # XXX ZAP once one run survives it!
# Flag blank images (i.e. images that had no hits), because
# these may interesting for background subtraction.
hits = self._hit.count(True)
self.logger.info("Hit rate: %d/%d (%.2f%%)" % (hits, self._hit.size(), 100 * hits / self._hit.size()))
if hits == 0:
self.logger.info("Frame %d is blank" % self._nframes)
# Get the normalisation factor by summing up I0 for all hits.
# Invalid and non-positive values of I0 are treated as zeroes.
# XXX Make this kind of summing a function of its own.
I0 = sum(filter(lambda x: not math.isnan(x) and x > 0, self._I0.select(self._hit)))
I0_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._I0))
fee_before_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._fee_before))
fee_after_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._fee_after))
# Register the template to the image and locate the regions of
# interest based on the registration parameters. XXX Should
# also give contrast: fit 2D-Gaussian to peak and report its
# standard deviations and fit?
if self._template is not None:
gamma = lewis(self._template, self.cspad_img)
p = flex.max_index(gamma)
peak = (
p // gamma.focus()[1] - self._template.focus()[0] + 1,
p % gamma.focus()[1] - self._template.focus()[1] + 1,
)
# """
### REFERENCE CHECK ###
from os.path import dirname, isdir, join
from scipy import io
mat_dirname = dirname(cspad_tbx.pathsubst(self._mat_path, evt, env, frame_number=self._nframes))
if not isdir(mat_dirname):
makedirs(mat_dirname)
io.savemat(
file_name=join(mat_dirname, "cross-check-%05d.mat" % self._nframes),
mdict=dict(
image=self.cspad_img.as_numpy_array(),
template=self._template.as_numpy_array(),
gamma=gamma.as_numpy_array(),
peak=numpy.array(peak),
),
appendmat=False,
do_compression=True,
oned_as="column",
)
return
### REFERENCE CHECK ###
# """
else:
# Alternative: position everything with respect to the frame
# origin.
peak = (0, 0)
# XXX Come up with a better way to handle the offsets! They
# really do depend on the template, and should therefore be
# packaged with it.
self.logger.info("Template registration anchor point (%d, %d)" % (peak[0], peak[1]))
roi = []
if evt.expNum() == 208:
# Regions of interest for L632 (experiment number 208). XXX
# Could perhaps migrate the template matching here instead?
# The left, middle, and right manganese signals. XXX Extend the
# rightmost ROI three pixels in upward direction (see runs 145
# and onwards, also note narrower slit)?
roi.append((peak[0] + 59, peak[1] - 24, 12, 5))
roi.append((peak[0] + 61, peak[1] + 28, 12, 4))
roi.append((peak[0] + 61, peak[1] + 79, 12, 5))
# Two background regions between the manganese spots, with the
# same total area as the signal.
roi.append((peak[0] + 62, peak[1] + 1, 8, 8))
roi.append((peak[0] + 63, peak[1] + 51, 8, 8))
# The left and right direct reflections from the Si substrate
# (i.e. the areas between the zone plates). These were the
# features used for template registration.
roi.append((peak[0], peak[1], 40, 10))
roi.append((peak[0], peak[1] + 50, 40, 9))
# Spot between the direct reflections. XXX What is this?
roi.append((peak[0] + 1, peak[1] + 23, 22, 13))
# The horizontal slit, where the direct reflection occurs. This
# is fixed. XXX Verify this!
roi.append((22, 0, 41, 128))
# Background stripe, below the manganese spots. This is fixed
# to the bottom of the detector.
roi.append((104, 0, 20, 128))
elif evt.expNum() == 363:
# Regions of interest for LB68 (experiment number 363).
# 0-pixel are active, 255-pixel are inactive
from scipy.misc import imread
# Dec 5, 2013 (09:00 - 21:00): initial estimates from r0010
"""
roi.append((peak[0] + 14, peak[1] + 138 + 23, 25, 50 - 25))
roi.append((peak[0] + 45, peak[1] + 138 + 23, 25, 50 - 25))
roi.append((peak[0] + 78, peak[1] + 137 + 23, 25, 50 - 25))
roi.append((peak[0] + 111, peak[1] + 137 + 23, 25, 50 - 25))
roi.append((peak[0] + 144, peak[1] + 137 + 23, 25, 50 - 25))
roi.append((peak[0] + 177, peak[1] + 136 + 23, 25, 50 - 25))
roi.append((peak[0] + 210, peak[1] + 136 + 23, 25, 50 - 25))
roi.append((peak[0] + 243, peak[1] + 136 + 23, 25, 50 - 25))
roi.append((peak[0] + 278, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 312, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 344, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 376, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 408, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 442, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 475, peak[1] + 135 + 23, 25, 50 - 25))
"""
# Dec 6, 2013 (09:00 - 21:00): rough estimates
"""
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen
roi.append((peak[0] + 0, peak[1] + 160, 512, 25)) # signal
roi.append((peak[0] + 0, peak[1] + 300, 512, 130)) # zeroth order
"""
# Dec 7, 2013 (09:00 - 21:00): overlap between oxygen and
# signal. Will loose some signal.
"""
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 50)) # oxygen
roi.append((peak[0] + 0, peak[1] + 185, 512, 40)) # signal
roi.append((peak[0] + 0, peak[1] + 270, 512, 170)) # zeroth order
"""
"""
# Dec 7 2013 (09:00 - 21:00): binary masks stored in PNG
# images.
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen
#roi_image = flex.float(
# imread('/reg/neh/home1/hattne/myrelease/LB68-r0039-max-mask.png',
# flatten=True))
#roi_image = flex.float(
# imread('/reg/neh/home1/hattne/myrelease/LB68-r0039-std-mask.png',
# flatten=True))
roi_image = flex.float(
imread('/reg/neh/home1/hattne/myrelease/LB68-r0052-avg-mask.png',
flatten=True))
roi_image = (255 - roi_image)
#roi.append((0, 0, self.cspad_img.focus()[0], self.cspad_img.focus()[1]))
roi.append(roi_image)
roi.append((peak[0] + 0, peak[1] + 270, 512, 170)) # zeroth order
"""
# Dec 9, 2013 (09:00 - 21:00)
# """
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen
# roi.append((peak[0] + 0, peak[1] + 160, 512, 25)) # signal
roi_image = flex.float(imread("/reg/neh/home1/hattne/myrelease/LB68-r0067-max-mask.png", flatten=True))
roi.append(roi_image)
roi.append((peak[0] + 0, peak[1] + 240, 512, 180)) # zeroth order
# """
else:
self.logger.error(
"No regions of interest for %s (experiment number %d)" % (env.experiment(), evt.expNum())
)
# Clip the regions of interest to the actual image. If the ROI
# does not overlap with the image at all, set its width and
# height to zero. XXX Do the integration here as well?
for i in range(len(roi)):
if not isinstance(roi[i], tuple):
continue
r = roi[i]
if (
r[0] + r[2] < 0
or r[0] >= self.cspad_img.focus()[0]
or r[1] + r[3] < 0
or r[1] >= self.cspad_img.focus()[1]
):
roi[i] = (r[0], r[1], 0, 0)
continue
r = roi[i]
if r[0] < 0:
roi[i] = (0, r[1], r[2] + r[0], r[3])
r = roi[i]
if r[1] < 0:
roi[i] = (r[0], 0, r[2], r[3] + r[1])
r = roi[i]
if r[0] + r[2] > self.cspad_img.focus()[0]:
roi[i] = (r[0], r[1], self.cspad_img.focus()[0] - r[0], r[3])
r = roi[i]
if r[1] + r[3] > self.cspad_img.focus()[1]:
roi[i] = (r[0], r[1], r[2], self.cspad_img.focus()[1] - r[1])
# Sum up intensities in all regions of interest, and keep track
# of the actual number of pixels summed. The common_mode module
# takes care of dark-subtraction. XXX Would like to estimate
# sigma for spot, like in spotfinder/LABELIT.
I = flex.double(len(roi))
I_nmemb = flex.int(len(roi))
for i in range(len(roi)):
if isinstance(roi[i], flex.float):
sel = roi[i].as_1d() < 128
I[i] = flex.sum(self.cspad_img.as_1d().select(sel))
I_nmemb[i] = sel.count(True)
continue
if roi[i][2] <= 0 or roi[i][3] <= 0:
I[i] = 0
I_nmemb[i] = 0
else:
I[i] = flex.sum(
self.cspad_img.matrix_copy_block(
i_row=roi[i][0], i_column=roi[i][1], n_rows=roi[i][2], n_columns=roi[i][3]
)
)
I_nmemb[i] = roi[i][2] * roi[i][3]
"""
# Sanity check: white out the region of interest.
self.cspad_img.matrix_paste_block_in_place(
block=flex.double(flex.grid(roi[i][2], roi[i][3])),
i_row=roi[i][0],
i_column=roi[i][1])
"""
acq_apd_sum = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_apd_integral.select(self._hit)))
acq_opto_diode_sum = sum(
filter(lambda x: not math.isnan(x) and x > 0, self._acq_opto_diode_integral.select(self._hit))
)
acq_apd_sum_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_apd_integral))
acq_opto_diode_sum_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_opto_diode_integral))
# Append the data point to the stream: shots, hits, energy, and
# I. XXX OrderedDict requires Python 2.7, could fall back on
# regular Dict at the price of non-deterministic column order.
from collections import OrderedDict
csv_dict = OrderedDict(
[
("n_frames", self._hit.size()),
("n_hits", hits),
("I0", I0),
("I0_all", I0_all),
("fee_before_all", fee_before_all),
("fee_after_all", fee_after_all),
("energy_mean", energy_mean),
("acq_apd_sum", acq_apd_sum),
("acq_apd_sum_all", acq_apd_sum_all),
("acq_opto_diode_sum", acq_opto_diode_sum),
("acq_opto_diode_sum_all", acq_opto_diode_sum_all),
]
)
for (i, item) in enumerate(zip(roi, I, I_nmemb)):
key = "roi_" + ("bkg", "oxygen", "manganese", "zeroth_order")[i]
csv_dict["%s_nmemb" % key] = item[2]
if isinstance(item[0], tuple):
csv_dict["%s_ss_start" % key] = item[0][0]
csv_dict["%s_fs_start" % key] = item[0][1]
csv_dict["%s_ss_size" % key] = item[0][2]
csv_dict["%s_fs_size" % key] = item[0][3]
else:
csv_dict["%s_ss_start" % key] = 0
csv_dict["%s_fs_start" % key] = 0
csv_dict["%s_ss_size" % key] = item[0].focus()[0]
csv_dict["%s_fs_size" % key] = item[0].focus()[1]
csv_dict["%s_I" % key] = item[1]
# XXX assert that keys match up with what's in the file already?
# Or exploit the error-reporting mechanism already implemented?
# Write the header. XXX How to control the order of the
# columns?
if not hasattr(self, "_csv"):
from csv import DictWriter
self._csv = DictWriter(self._stream_table, csv_dict.keys())
self._csv.writerow({key: key for key in csv_dict.keys()})
self._csv.writerow(csv_dict)
# Output the non-normalised image and all other relevant data to
# a binary MATLAB file. XXX What if scipy is not available?
from os import makedirs, path
from scipy import io
mat_path = cspad_tbx.pathsubst(self._mat_path, evt, env, frame_number=self._nframes)
if not path.isdir(path.dirname(mat_path)):
makedirs(path.dirname(mat_path))
io.savemat(
file_name=mat_path,
mdict=dict(
DATA=self.cspad_img.as_numpy_array(),
DIODES=numpy.array((acq_apd_sum, acq_apd_sum_all, acq_opto_diode_sum, acq_opto_diode_sum_all)),
ENERGY=energy_mean,
HITS=numpy.array((hits, self._hit.size())),
I0=numpy.array((I0, I0_all)),
INTENSITIES=numpy.array(I),
ROIS=numpy.array([r for r in roi if isinstance(r, tuple)]),
),
appendmat=False,
do_compression=True,
oned_as="column",
)
# Optionally update the image in the viewer. See mod_view.
if self._display:
from time import localtime, strftime
# Copy over regions of interest to shared multiprocessing
# array. XXX Flip to honour wxPython convention.
for i in range(len(roi)):
if not isinstance(roi[i], tuple):
continue
self._roi[4 * i + 0] = roi[i][1]
self._roi[4 * i + 1] = roi[i][0]
self._roi[4 * i + 2] = roi[i][3]
self._roi[4 * i + 3] = roi[i][2]
time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds()))
title = "r%04d@%s: frame %d on %s" % (evt.run(), time_str, self._nframes, self.address)
# XXX No distance in the Andor experiment. So don't bother
# with the fictional beam center, distance, and saturation
# value? See also mod_average.endjob()
img_obj = (
dict(
BEAM_CENTER=(0, 0),
DATA=self.cspad_img,
DETECTOR_ADDRESS=self.address,
DISTANCE=10, # XXX Evil kludge to keep dxtbx happy!
PIXEL_SIZE=13.5e-3, # XXX Hard-coded, again!
SATURATED_VALUE=10000,
TIME_TUPLE=cspad_tbx.evt_time(evt),
WAVELENGTH=12398.4187 / energy,
),
title,
)
while not self._queue.empty():
if not self._proc.is_alive():
evt.setStatus(Event.Stop)
return
while True:
try:
self._queue.put(img_obj, timeout=1)
break
except Exception: # Queue.Full:
pass
self._reset_counters()
return
def get_raw_data(self, index=0): from scitbx.array_family import flex detector_2d_assembled_1 = self._run['detector_2d_assembled_1'] tag = detector_2d_assembled_1[self._images[index]] data = tag['detector_data'] return flex.float(data[:,:]).as_double()