def __init__(self,
dims,
pars,
ubi,
splinefile=None,
np=16,
border=10,
omegarange=list(range(360)),
maxpix=None,
mask=None):
"""
Create a new mapper intance. It will transform images into
reciprocal space (has its own rsv object holding the space)
dims - image dimensions
par - ImageD11 parameter filename for experiment
ubi - Orientation matrix (ImageD11 style)
np - Number of pixels per hkl index [16]
border - amount to add around edge of images [10]
omegarange - omega values to be mapped (0->360)
maxpix - value for saturated pixels to be ignored
mask - fit2d style mask for removing bad pixels / border
"""
if len(dims) != 2: raise Exception("For 2D dims!")
self.dims = dims
print(dims)
# Experiment parameters
if not isinstance(pars, parameters.parameters):
raise Exception("Pars should be an ImageD11 parameters object")
for key in ["distance", "wavelength"]: #etc
assert key in pars.parameters
self.pars = pars
# Orientation matrix
self.ubi = ubi
# Saturation
self.maxpix = maxpix
# Mask
self.mask = mask
if self.mask is not None:
assert self.mask.shape == self.dims, "Mask dimensions mush match image"
# spatial
if splinefile is None:
self.spatial = blobcorrector.perfect()
else:
self.spatial = blobcorrector.correctorclass(splinefile)
# npixels
self.np = np
self.uspace = np * ubi
self.find_vol(border=border, omegarange=omegarange)
self.rsv.metadata['ubi'] = ubi
self.rsv.metadata['uspace'] = self.uspace
# Make and cache the k vectors
self.make_k_vecs()
def __init__(self, param, hkl, killfile=None):
self.killfile = killfile
self.param = param
self.hkl = hkl
self.grain = []
# Simple transforms of input and set constants
self.K = -2 * n.pi / self.param['wavelength']
self.S = n.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
# Detector tilt correction matrix
self.R = tools.detect_tilt(self.param['tilt_x'], self.param['tilt_y'],
self.param['tilt_z'])
# wedge NB! wedge is in degrees
# The sign is reversed for wedge as the parameter in
# tools.find_omega_general is right handed and in ImageD11
# it is left-handed (at this point wedge is defined as in ImageD11)
self.wy = -1. * self.param['wedge'] * n.pi / 180.
self.wx = 0.
# Spatial distortion
if self.param['spatial'] != None:
from ImageD11 import blobcorrector
self.spatial = blobcorrector.correctorclass(self.param['spatial'])
# %No of images
self.nframes = (self.param['omega_end'] -
self.param['omega_start']) / self.param['omega_step']
# Generate Miller indices for reflections within a certain resolution
print('Generating reflections')
print('Finished generating reflections\n')
def make_powder_mask( parfile,
ndeg = 1,
splinefile=None,
dims=(2048, 2048) ):
"""
Compute a two theta and azimuth image
"""
pars = parameters.parameters()
pars.loadparameters( parfile )
if splinefile is None:
spatial = blobcorrector.perfect()
else:
spatial = blobcorrector.correctorclass( splinefile )
xim, yim = spatial.make_pixel_lut ( dims )
peaks = [ np.ravel( xim ) , np.ravel( yim ) ]
tth , eta = transform.compute_tth_eta( peaks , **pars.get_parameters() )
tth.shape = dims
eta.shape = dims
# Assume a circle geometry for now
# tth * eta ~ length on detector
# lim = tth * eta
# need some idea how to cut it up...
# degree bins
m = (eta.astype(np.int) % 2)==0
return m
def compute_tth_eta_lut(splinefile, pars, dims):
"""
Computes look up values of tth, eta for each pixel
"""
c = blobcorrector.correctorclass(splinefile)
p = parameters.read_par_file(pars)
xp, yp = c.make_pixel_lut(dims)
t, e = transform.compute_tth_eta((xp.ravel(), yp.ravel()), **p.parameters)
t.shape = dims
e.shape = dims
return t, e
def spatial_correct(s_raw, f_raw, spline):
si = np.round(s_raw).clip(0, 2047).astype(int)
fi = np.round(f_raw).clip(0, 2047).astype(int)
try:
dx, dy = spline
fc = f_raw + dx[si, fi]
sc = s_raw + dy[si, fi]
except:
corr = blobcorrector.correctorclass(spline)
dx, dy = corr.make_pixel_lut((2048, 2048))
fc = f_raw - fi + dy[si, fi]
sc = s_raw - si + dx[si, fi]
return sc, fc
def __init__(self, pars):
"""
pars is a dictionary containing the calibration parameters
"""
self.pars = {}
for p in self.pnames:
if p in pars:
self.pars[p] = pars[p] # make a copy
if 'dxfile' in pars:
# slow/fast coordinates on image at pixel centers
self.df = fabio.open(pars['dxfile']).data
self.ds = fabio.open(pars['dyfile']).data
self.shape = s = self.ds.shape # get shape from file
self.pars['shape'] = s
slow, fast = np.mgrid[0:s[0], 0:s[1]]
self.sc = slow + self.ds
self.fc = fast + self.df
elif 'spline' in pars: # need to test this...
from ImageD11 import blobcorrector
b = blobcorrector.correctorclass(self.pars['spline'])
s = int(b.ymax - b.ymin), int(b.xmax - b.xmin)
if 'shape' in self.pars: # override. Probabl
s = self.pars['shape']
self.shape = s
self.fc, self.sc = b.make_pixel_lut(s)
slow, fast = np.mgrid[0:s[0], 0:s[1]]
self.df = self.fc - fast
self.ds = self.sc - slow
else:
s = self.shape
self.sc, self.fc = np.mgrid[0:s[0], 0:s[1]]
self.df = None
self.ds = None
self.xyz = compute_xyz_lab(self.sc.ravel(), self.fc.ravel(),
**self.pars)
self.sinthsq = compute_sinsqth_from_xyz(self.xyz)
self.tth, self.eta = compute_tth_eta_from_xyz(self.peaks_xyz, None,
**self.pars)
# scattering angles:
self.tth, self.eta = compute_tth_eta(
(self.sc.ravel(), self.fc.ravel()), **self.pars)
# scattering vectors:
self.k = compute_k_vectors(self.tth, self.eta,
self.pars.get('wavelength'))
self.sinthsq.shape = s
self.tth.shape = s
self.eta.shape = s
self.k.shape = (3, s[0], s[1])
self.xyz.shape = (3, s[0], s[1])
def __init__(self,param,hkl,killfile=None):
'''
A find_refl object is used to hold information
that otherwise would have to be recomputed for
during the algorithm execution in run().
'''
self.killfile = killfile
self.param = param
self.hkl = hkl
self.voxel = [None]*self.param['no_voxels']
# Simple transforms of input and set constants
self.K = -2*np.pi/self.param['wavelength']
self.S = np.array([[1, 0, 0],[0, 1, 0],[0, 0, 1]])
# Detector tilt correction matrix
self.R = tools.detect_tilt(self.param['tilt_x'],
self.param['tilt_y'],
self.param['tilt_z'])
# wedge NB! wedge is in degrees
# The sign is reversed for wedge as the parameter in
# tools.find_omega_general is right handed and in ImageD11
# it is left-handed (at this point wedge is defined as in ImageD11)
self.wy = -1.*self.param['wedge']*np.pi/180.
self.wx = 0.
w_mat_x = np.array([[1, 0 , 0 ],
[0, np.cos(self.wx), -np.sin(self.wx)],
[0, np.sin(self.wx), np.cos(self.wx)]])
w_mat_y = np.array([[ np.cos(self.wy), 0, np.sin(self.wy)],
[0 , 1, 0 ],
[-np.sin(self.wy), 0, np.cos(self.wy)]])
self.r_mat = np.dot(w_mat_x,w_mat_y)
# Spatial distortion
if self.param['spatial'] != None:
from ImageD11 import blobcorrector
self.spatial = blobcorrector.correctorclass(self.param['spatial'])
# %No of images
self.nframes = (self.param['omega_end']-self.param['omega_start'])/self.param['omega_step']
self.peak_merger = cms_peak_compute.PeakMerger( self.param )
def mymain():
# If we are running from a command line:
inname = sys.argv[1]
if not os.path.exists(inname) or len(sys.argv) < 4:
help()
sys.exit()
outname = sys.argv[2]
if os.path.exists(outname):
if not input("Sure you want to overwrite %s ?" %
(outname))[0] in ['y', 'Y']:
sys.exit()
splinename = sys.argv[3]
if splinename == 'perfect':
cor = blobcorrector.perfect()
else:
cor = blobcorrector.correctorclass(splinename)
fix_flt(inname, outname, cor)
def __init__(self, splinefile=None, parfile=None):
"""
splinefile = fit2d spline file, or None for images
that are already corrected
parfile = ImageD11 parameter file. Can be fitted
using old ImageD11_gui.py or newer fable.transform
plugin.
"""
self.splinefile = splinefile
if self.splinefile is None:
self.spatial = blobcorrector.perfect()
else:
self.spatial = blobcorrector.correctorclass(splinefile)
self.parfile = parfile
self.pars = parameters.parameters()
self.pars.loadparameters(parfile)
for key in self.required_pars:
if key not in self.pars.parameters:
raise Exception("Missing parameter " + str(par))
def peaksearch_driver(options, args):
"""
To be called with options from command line
"""
################## debugging still
for a in args:
print("arg: "+str(a)+","+str(type(a)))
for o in list(options.__dict__.keys()): # FIXME
if getattr(options,o) in [ "False", "FALSE", "false" ]:
setattr(options,o,False)
if getattr(options,o) in [ "True", "TRUE", "true" ]:
setattr(options,o,True)
print("option:",str(o),str(getattr(options,o)),",",\
str(type( getattr(options,o) ) ))
###################
print("This peaksearcher is from",__file__)
if options.killfile is not None and os.path.exists(options.killfile):
print("The purpose of the killfile option is to create that file")
print("only when you want peaksearcher to stop")
print("If the file already exists when you run peaksearcher it is")
print("never going to get started")
raise ValueError("Your killfile "+options.killfile+" already exists")
if options.thresholds is None:
raise ValueError("No thresholds supplied [-t 1234]")
if len(args) == 0 and options.stem is None:
raise ValueError("No files to process")
# What to do about spatial
if options.perfect=="N" and os.path.exists(options.spline):
print("Using spatial from",options.spline)
corrfunc = blobcorrector.correctorclass(options.spline)
else:
print("Avoiding spatial correction")
corrfunc = blobcorrector.perfect()
# This is always the case now
corrfunc.orientation = "edf"
scan = None
if options.format in ['bruker', 'BRUKER', 'Bruker']:
extn = ""
if options.perfect is not "N":
print("WARNING: Your spline file is ImageD11 specific")
print("... from a fabio converted to edf first")
elif options.format == 'GE':
extn = ""
# KE: This seems to be a mistake and keeps PeakSearch from working in
# some cases. Should be revisited if commenting it out causes problems.
# options.ndigits = 0
elif options.format == 'py':
import importlib
sys.path.append( '.' )
scan = importlib.import_module( options.stem )
first_image = scan.first_image
file_series_object = scan.file_series_object
else:
extn = options.format
if scan is None:
if options.interlaced:
f0 = ["%s0_%04d%s"%(options.stem,i,options.format) for i in range(
options.first,
options.last+1)]
f1 = ["%s1_%04d%s"%(options.stem,i,options.format) for i in range(
options.first,
options.last+1)]
if options.iflip:
f1 = [a for a in f1[::-1]]
def fso(f0,f1):
for a,b in zip(f0,f1):
try:
yield fabio.open(a)
yield fabio.open(b)
except:
print(a,b)
raise
file_series_object = fso(f0,f1)
first_image = openimage( f0[0] )
else:
import fabio
if options.ndigits > 0:
file_name_object = fabio.filename_object(
options.stem,
num = options.first,
extension = extn,
digits = options.ndigits)
else:
file_name_object = options.stem
first_image = openimage( file_name_object )
import fabio.file_series
# Use traceback = True for debugging
file_series_object = fabio.file_series.new_file_series(
first_image,
nimages = options.last - options.first + 1,
traceback = True )
# Output files:
if options.outfile[-4:] != ".spt":
options.outfile = options.outfile + ".spt"
print("Your output file must end with .spt, changing to ",options.outfile)
# Omega overrides
# global OMEGA, OMEGASTEP, OMEGAOVERRIDE
OMEGA = options.OMEGA
OMEGASTEP = options.OMEGASTEP
OMEGAOVERRIDE = options.OMEGAOVERRIDE
# Make a blobimage the same size as the first image to process
# List comprehension - convert remaining args to floats
# must be unique list so go via a set
thresholds_list = list( set( [float(t) for t in options.thresholds] ) )
thresholds_list.sort()
li_objs={} # label image objects, dict of
s = first_image.data.shape # data array shape
# Create label images
for t in thresholds_list:
# the last 4 chars are guaranteed to be .spt above
mergefile="%s_t%d.flt"%(options.outfile[:-4], t)
spotfile = "%s_t%d.spt"%(options.outfile[:-4], t)
li_objs[t]=labelimage(shape = s,
fileout = mergefile,
spatial = corrfunc,
sptfile=spotfile)
print("make labelimage",mergefile,spotfile)
# Not sure why that was there (I think if glob was used)
# files.sort()
if options.dark is not None:
print("Using dark (background)",options.dark)
darkimage= openimage(options.dark).data.astype(numpy.float32)
else:
darkimage=None
if options.darkoffset!=0:
print("Adding darkoffset",options.darkoffset)
if darkimage is None:
darkimage = options.darkoffset
else:
darkimage += options.darkoffset
if options.flood is not None:
floodimage=openimage(options.flood).data
cen0 = int(floodimage.shape[0]/6)
cen1 = int(floodimage.shape[0]/6)
middle = floodimage[cen0:-cen0, cen1:-cen1]
nmid = middle.shape[0]*middle.shape[1]
floodavg = numpy.mean(middle)
print("Using flood",options.flood,"average value",floodavg)
if floodavg < 0.7 or floodavg > 1.3:
print("Your flood image does not seem to be normalised!!!")
else:
floodimage=None
start = time.time()
print("File being treated in -> out, elapsed time")
# If we want to do read-ahead threading we fill up a Queue object with data
# objects
# THERE MUST BE ONLY ONE peaksearching thread for 3D merging to work
# there could be several read_and_correct threads, but they'll have to get the order right,
# for now only one
if options.oneThread:
# Wrap in a function to allow profiling (perhaps? what about globals??)
def go_for_it(file_series_object, darkimage, floodimage,
corrfunc , thresholds_list , li_objs,
OMEGA, OMEGASTEP, OMEGAOVERRIDE ):
for data_object in file_series_object:
t = timer()
if not hasattr( data_object, "data"):
# Is usually an IOError
if isinstance( data_object[1], IOError):
sys.stdout.write(data_object[1].strerror + '\n')
# data_object[1].filename
else:
import traceback
traceback.print_exception(data_object[0],data_object[1],data_object[2])
sys.exit()
continue
filein = data_object.filename
if OMEGAOVERRIDE or "Omega" not in data_object.header:
data_object.header["Omega"] = OMEGA
OMEGA += OMEGASTEP
OMEGAOVERRIDE = True # once you do it once, continue
if not OMEGAOVERRIDE and options.omegamotor != "Omega":
data_object.header["Omega"] = float( data_object.header[options.omegamotor] )
data_object = correct( data_object, darkimage, floodimage,
do_median = options.median,
monitorval = options.monitorval,
monitorcol = options.monitorcol,
)
t.tick(filein+" io/cor")
peaksearch( filein, data_object , corrfunc ,
thresholds_list , li_objs )
for t in thresholds_list:
li_objs[t].finalise()
go_for_it(file_series_object, darkimage, floodimage,
corrfunc , thresholds_list, li_objs,
OMEGA, OMEGASTEP, OMEGAOVERRIDE )
else:
print("Going to use threaded version!")
try:
# TODO move this to a module ?
class read_only(ImageD11_thread.ImageD11_thread):
def __init__(self, queue, file_series_obj , myname="read_only",
OMEGA=0, OMEGAOVERRIDE = False, OMEGASTEP = 1):
""" Reads files in file_series_obj, writes to queue """
self.queue = queue
self.file_series_obj = file_series_obj
self.OMEGA = OMEGA
self.OMEGAOVERRIDE = OMEGAOVERRIDE
self.OMEGASTEP = OMEGASTEP
ImageD11_thread.ImageD11_thread.__init__(self ,
myname=myname)
print("Reading thread initialised", end=' ')
def ImageD11_run(self):
""" Read images and copy them to self.queue """
for data_object in self.file_series_obj:
if self.ImageD11_stop_now():
print("Reader thread stopping")
break
if not hasattr( data_object, "data" ):
import pdb; pdb.set_trace()
# Is usually an IOError
if isinstance( data_object[1], IOError):
sys.stdout.write(str(data_object[1].strerror) + '\n')
else:
import traceback
traceback.print_exception(data_object[0],data_object[1],data_object[2])
sys.exit()
continue
ti = timer()
filein = data_object.filename + "[%d]"%( data_object.currentframe )
try:
if self.OMEGAOVERRIDE:
# print "Over ride due to option",self.OMEGAOVERRIDE
data_object.header["Omega"] = self.OMEGA
self.OMEGA += self.OMEGASTEP
else:
if options.omegamotor != 'Omega' and options.omegamotor in data_object.header:
data_object.header["Omega"] = float(data_object.header[options.omegamotor])
if "Omega" not in data_object.header:
# print "Computing omega as not in header"
data_object.header["Omega"] = self.OMEGA
self.OMEGA += self.OMEGASTEP
self.OMEGAOVERRIDE = True
# print "Omega = ", data_object.header["Omega"],data_object.filename
except KeyboardInterrupt:
raise
except:
continue
ti.tick(filein)
self.queue.put((filein, data_object) , block = True)
ti.tock(" enqueue ")
if self.ImageD11_stop_now():
print("Reader thread stopping")
break
# Flag the end of the series
self.queue.put( (None, None) , block = True)
class correct_one_to_many(ImageD11_thread.ImageD11_thread):
def __init__(self, queue_read, queues_out, thresholds_list,
dark = None , flood = None, myname="correct_one",
monitorcol = None, monitorval = None,
do_median = False):
""" Using a single reading queue retains a global ordering
corrects and copies images to output queues doing
correction once """
self.queue_read = queue_read
self.queues_out = queues_out
self.dark = dark
self.flood = flood
self.do_median = do_median
self.monitorcol = monitorcol
self.monitorval = monitorval
self.thresholds_list = thresholds_list
ImageD11_thread.ImageD11_thread.__init__(self ,
myname=myname)
def ImageD11_run(self):
while not self.ImageD11_stop_now():
ti = timer()
filein, data_object = self.queue_read.get(block = True)
if filein is None:
for t in self.thresholds_list:
self.queues_out[t].put( (None, None) ,
block = True)
# exit the while 1
break
data_object = correct(data_object, self.dark,
self.flood,
do_median = self.do_median,
monitorval = self.monitorval,
monitorcol = self.monitorcol,
)
ti.tick(filein+" correct ")
for t in self.thresholds_list:
# Hope that data object is read only
self.queues_out[t].put((filein, data_object) ,
block = True)
ti.tock(" enqueue ")
print("Corrector thread stopping")
class peaksearch_one(ImageD11_thread.ImageD11_thread):
def __init__(self, q, corrfunc, threshold, li_obj,
myname="peaksearch_one" ):
""" This will handle a single threshold and labelimage
object """
self.q = q
self.corrfunc = corrfunc
self.threshold = threshold
self.li_obj = li_obj
ImageD11_thread.ImageD11_thread.__init__(
self,
myname=myname+"_"+str(threshold))
def run(self):
while not self.ImageD11_stop_now():
filein, data_object = self.q.get(block = True)
if not hasattr( data_object, "data" ):
break
peaksearch( filein, data_object , self.corrfunc ,
[self.threshold] ,
{ self.threshold : self.li_obj } )
self.li_obj.finalise()
# 8 MB images - max 40 MB in this queue
read_queue = queue.Queue(5)
reader = read_only(read_queue, file_series_object,
OMEGA = OMEGA,
OMEGASTEP = OMEGASTEP,
OMEGAOVERRIDE = OMEGAOVERRIDE )
reader.start()
queues = {}
searchers = {}
for t in thresholds_list:
print("make queue and peaksearch for threshold",t)
queues[t] = queue.Queue(3)
searchers[t] = peaksearch_one(queues[t], corrfunc,
t, li_objs[t] )
corrector = correct_one_to_many( read_queue,
queues,
thresholds_list,
dark=darkimage,
flood=floodimage,
do_median = options.median,
monitorcol = options.monitorcol,
monitorval = options.monitorval)
corrector.start()
my_threads = [reader, corrector]
for t in thresholds_list[::-1]:
searchers[t].start()
my_threads.append(searchers[t])
nalive = len(my_threads)
def empty_queue(q):
while 1:
try:
q.get(block=False, timeout=1)
except:
break
q.put((None, None), block=False)
while nalive > 0:
try:
nalive = 0
for thr in my_threads:
if thr.isAlive():
nalive += 1
if options.killfile is not None and \
os.path.exists(options.killfile):
raise KeyboardInterrupt()
time.sleep(1)
except KeyboardInterrupt:
print("Got keyboard interrupt in waiting loop")
ImageD11_thread.stop_now = True
try:
time.sleep(1)
except:
pass
empty_queue(read_queue)
for t in thresholds_list:
q = queues[t]
empty_queue(q)
for thr in my_threads:
if thr.isAlive():
thr.join(timeout=1)
print("finishing from waiting loop")
except:
print("Caught exception in waiting loop")
ImageD11_thread.stop_now = True
time.sleep(1)
empty_queue(read_queue)
for t in thresholds_list:
q = queues[t]
empty_queue(q)
for thr in my_threads:
if thr.isAlive():
thr.join(timeout=1)
raise
except ImportError:
print("Probably no threading module present")
raise
def make_image(self, frame_number=None):
"""
makeimage script produces edf diffraction images using the reflection information
Henning Osholm Sorensen, June 23, 2006.
python translation Jette Oddershede, March 31, 2008
"""
peakshape = self.graindata.param['peakshape']
if peakshape[0] == 0: # spike peak, 2x2 pixels
peak_add = 1
frame_add = 1
peakwsig = 0
elif peakshape[0] == 1: # 3d Gaussian peak
peak_add = max(1, int(round(peakshape[1])))
frame_add = max(1, int(round(peakshape[1])))
peakwsig = peakshape[2]
elif peakshape[0] == 3: # 3d Gaussian peak in 2theta,eta,omega
peak_add = 1
frame_add = 1
cen_tth = int(1.5 * peakshape[1] / Delta_tth)
frame_tth = 2 * cen_tth + 1
fwhm_tth = peakshape[1] / Delta_tth
cen_eta = int(1.5 * peakshape[2] / Delta_eta)
frame_eta = 2 * cen_eta + 1
fwhm_eta = peakshape[2] / Delta_eta
raw_tth_eta = n.zeros((frame_tth, frame_eta))
raw_tth_eta[cen_tth, cen_eta] = 1
filter_tth_eta = ndimage.gaussian_filter(
raw_tth_eta, [0.5 * fwhm_tth, 0.5 * fwhm_eta])
peakwsig = 1.
framedimy = int(self.graindata.param['dety_size'] + 2 * frame_add)
framedimz = int(self.graindata.param['detz_size'] + 2 * frame_add)
totalrefl = 0
if frame_number == None:
no_frames = list(range(len(self.graindata.frameinfo)))
print('Generating diffraction images')
else:
no_frames = [frame_number]
for i in no_frames:
check_input.interrupt(self.killfile)
t1 = time.clock()
nrefl = 0
frame = n.zeros((framedimy, framedimz))
omega = self.graindata.frameinfo[i].omega
omega_step = self.graindata.param['omega_step']
# Jettes hack to add relative movement of sample and detector, modelled to be Gaussian in y and z direction with a spread of 1 micron
# movement of 1 micron along x judged to be irrelevant, at least for farfield data
y_move = n.random.normal(0, 1. / self.graindata.param['dety_size'])
z_move = n.random.normal(0, 1. / self.graindata.param['detz_size'])
# loop over grains
for j in range(self.graindata.param['no_grains']):
# loop over reflections for each grain
gr_pos = n.array(self.graindata.param['pos_grains_%s' % j])
for k in range(len(self.graindata.grain[j].refs)):
# exploit that the reflection list is sorted according to omega
if self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi > \
omega+omega_step+2*peakwsig:
break
elif self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi < \
omega-2*peakwsig:
continue
dety = self.graindata.grain[j].refs[
k, A_id['detyd']] # must be spot position after
detz = self.graindata.grain[j].refs[
k, A_id['detzd']] # applying spatial distortion
#apply hack
# dety = self.graindata.grain[j].refs[k,A_id['dety']] + y_move
# detz = self.graindata.grain[j].refs[k,A_id['detz']] + z_move
ndety = int(round(dety))
ndetz = int(round(detz))
yrange = list(
range(ndety + frame_add - peak_add,
ndety + frame_add + peak_add + 1))
zrange = list(
range(ndetz + frame_add - peak_add,
ndetz + frame_add + peak_add + 1))
intensity = int(
round(self.graindata.grain[j].refs[k, A_id['Int']]))
nrefl = nrefl + 1
totalrefl = totalrefl + 1
# Gaussian along omega
if peakshape[0] == 1 or peakshape[0] == 3:
fraction = norm.cdf((omega-self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi+omega_step)/(0.5*peakwsig))\
-norm.cdf((omega-self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi)/(0.5*peakwsig))
else:
fraction = 1.
if peakshape[0] == 3:
# Gaussian peaks along 2theta,eta
tth = self.graindata.grain[j].refs[k, A_id['tth']]
eta = self.graindata.grain[j].refs[k, A_id['eta']]
Om = tools.form_omega_mat_general(
self.graindata.grain[j].refs[k, A_id['omega']], 0,
-1. * self.graindata.param['wedge'] * n.pi / 180.)
[tx, ty, tz] = n.dot(Om, gr_pos)
for t in range(frame_tth):
tth_present = tth + (
t - cen_tth) * Delta_tth * n.pi / 180.
for e in range(frame_eta):
eta_present = eta + (
e - cen_eta) * Delta_eta * n.pi / 180.
[dety_present,
detz_present] = detector.det_coor2(
tth_present,
eta_present,
self.graindata.param['distance'],
self.graindata.param['y_size'],
self.graindata.param['z_size'],
self.graindata.param['dety_center'],
self.graindata.param['detz_center'],
self.graindata.R,
tx,
ty,
tz,
)
if self.graindata.param['spatial'] != None:
from ImageD11 import blobcorrector
self.spatial = blobcorrector.correctorclass(
self.graindata.param['spatial'])
# To match the coordinate system of the spline file
# SPLINE(i,j): i = detz; j = (dety_size-1)-dety
# Well at least if the spline file is for frelon2k
(x, y) = detector.detyz_to_xy(
[dety_present, detz_present],
self.graindata.param['o11'],
self.graindata.param['o12'],
self.graindata.param['o21'],
self.graindata.param['o22'],
self.graindata.param['dety_size'],
self.graindata.param['detz_size'])
# Do the spatial distortion
(xd, yd) = self.spatial.distort(x, y)
# transform coordinates back to dety,detz
(dety_present,
detz_present) = detector.xy_to_detyz(
[xd, yd], self.graindata.param['o11'],
self.graindata.param['o12'],
self.graindata.param['o21'],
self.graindata.param['o22'],
self.graindata.param['dety_size'],
self.graindata.param['detz_size'])
y = int(round(dety_present))
z = int(round(detz_present))
try:
frame[y + frame_add, z + frame_add] = frame[
y + frame_add, z +
frame_add] + fraction * intensity * filter_tth_eta[
t, e]
except:
# FIXME print("Unhandled exception in make_image.py")
pass
else:
# Generate spikes, 2x2 pixels
for y in yrange:
for z in zrange:
if y > 0 and y < framedimy and z > 0 and z < framedimz and abs(
dety + frame_add - y) < 1 and abs(
detz + frame_add - z) < 1:
# frame[y-1,z] = frame[y-1,z] + fraction*intensity*(1-abs(dety+frame_add-y))*(1-abs(detz+frame_add-z))
y = int(round(y))
z = int(round(z))
frame[y, z] = frame[
y, z] + fraction * intensity * (
1 - abs(dety + frame_add - y)) * (
1 - abs(detz + frame_add - z))
# 2D Gaussian on detector
if peakshape[0] == 1:
frame = ndimage.gaussian_filter(frame, peakshape[1] * 0.5)
# add background
if self.graindata.param['bg'] > 0:
frame = frame + self.graindata.param['bg'] * n.ones(
(framedimy, framedimz))
# add noise
if self.graindata.param['noise'] != 0:
frame = n.random.poisson(frame)
# apply psf
if self.graindata.param['psf'] != 0:
frame = ndimage.gaussian_filter(
frame, self.graindata.param['psf'] * 0.5)
# resize, convert to integers and flip to same orientation as experimental frames
frame = frame[frame_add:framedimy - frame_add,
frame_add:framedimz - frame_add]
# limit values above 16 bit to be 16bit
frame = n.clip(frame, 0, 2**16 - 1)
# convert to integers
frame = n.uint16(frame)
#flip detector orientation according to input: o11, o12, o21, o22
frame = detector.trans_orientation(frame,
self.graindata.param['o11'],
self.graindata.param['o12'],
self.graindata.param['o21'],
self.graindata.param['o22'],
'inverse')
# Output frames
if '.edf' in self.graindata.param['output']:
self.write_edf(i, frame)
if '.edf.gz' in self.graindata.param['output']:
self.write_edf(i, frame, usegzip=True)
if '.tif' in self.graindata.param['output']:
self.write_tif(i, frame)
if '.tif16bit' in self.graindata.param['output']:
self.write_tif16bit(i, frame)
print('\rDone frame %i took %8f s' % (i + 1, time.clock() - t1),
end=' ')
sys.stdout.flush()
def peaksearch_driver(options, args):
"""
To be called with options from command line
"""
################## debugging still
for a in args:
print("arg: " + str(a) + "," + str(type(a)))
for o in list(options.__dict__.keys()): # FIXME
if getattr(options, o) in ["False", "FALSE", "false"]:
setattr(options, o, False)
if getattr(options, o) in ["True", "TRUE", "true"]:
setattr(options, o, True)
print("option:", str(o), str(getattr(options, o)), ",", \
str(type(getattr(options, o))))
###################
print("This peaksearcher is from", __file__)
if options.killfile is not None and os.path.exists(options.killfile):
print("The purpose of the killfile option is to create that file")
print("only when you want peaksearcher to stop")
print("If the file already exists when you run peaksearcher it is")
print("never going to get started")
raise ValueError("Your killfile " + options.killfile +
" already exists")
if options.thresholds is None:
raise ValueError("No thresholds supplied [-t 1234]")
if len(args) == 0 and options.nexusfile is None:
raise ValueError("No files to process")
# What to do about spatial
if options.perfect == "N" and os.path.exists(options.spline):
print("Using spatial from", options.spline)
corrfunc = blobcorrector.correctorclass(options.spline)
else:
print("Avoiding spatial correction")
corrfunc = blobcorrector.perfect()
# Get list of filenames to process
# if len(args) > 0 :
# # We no longer assume unlabelled arguments are filenames
# file_series_object = file_series.file_series(args)
# This is always the case now
corrfunc.orientation = "edf"
import h5py
# Read list of files and list of motor positions from Nexus file:
nexus_path = options.nexusfile
nexus_file = h5py.File(nexus_path, "r")
group = nexus_file[options.group_path]
omega_dset = group.get(options.omega_dset)
image_dset = group.get(options.image_dset)
omega_list = [x for x in omega_dset[..., :]]
image_list = [x.decode("utf-8") for x in image_dset[..., :]]
import fabio
# Output files:
import fabio.file_series
# Use traceback = True for debugging
first_image = openimage(image_list[0])
file_series_object = fabio.file_series.new_file_series(
first_image, nimages=len(image_list), traceback=True)
if options.outfile[-4:] != ".spt":
options.outfile = options.outfile + ".spt"
print("Your output file must end with .spt, changing to ",
options.outfile)
# Make a blobimage the same size as the first image to process
# List comprehension - convert remaining args to floats
# must be unique list so go via a set
thresholds_list = list(set([float(t) for t in options.thresholds]))
thresholds_list.sort()
li_objs = {} # label image objects, dict of
s = first_image.data.shape # data array shape
# Create label images
for t in thresholds_list:
# the last 4 chars are guaranteed to be .spt above
mergefile = "%s_t%d.flt" % (options.outfile[:-4], t)
spotfile = "%s_t%d.spt" % (options.outfile[:-4], t)
li_objs[t] = labelimage(shape=s,
fileout=mergefile,
spatial=corrfunc,
sptfile=spotfile)
print("make labelimage", mergefile, spotfile)
if options.dark is not None:
print("Using dark (background)", options.dark)
darkimage = openimage(options.dark).data.astype(numpy.float32)
else:
darkimage = None
if options.darkoffset != 0:
print("Adding darkoffset", options.darkoffset)
if darkimage is None:
darkimage = options.darkoffset
else:
darkimage += options.darkoffset
if options.flood is not None:
floodimage = openimage(options.flood).data
cen0 = int(floodimage.shape[0] / 6)
cen1 = int(floodimage.shape[0] / 6)
middle = floodimage[cen0:-cen0, cen1:-cen1]
nmid = middle.shape[0] * middle.shape[1]
floodavg = numpy.mean(middle)
print("Using flood", options.flood, "average value", floodavg)
if floodavg < 0.7 or floodavg > 1.3:
print("Your flood image does not seem to be normalised!!!")
else:
floodimage = None
start = time.time()
print("File being treated in -> out, elapsed time")
# If we want to do read-ahead threading we fill up a Queue object with data
# objects
# THERE MUST BE ONLY ONE peaksearching thread for 3D merging to work
# there could be several read_and_correct threads, but they'll have to get the order right,
# for now only one
if options.oneThread:
# Wrap in a function to allow profiling (perhaps? what about globals??)
def go_for_it(file_series_object, darkimage, floodimage, corrfunc,
thresholds_list, li_objs):
for inc, data_object in enumerate(file_series_object):
t = timer()
if not isinstance(data_object, fabio.fabioimage.fabioimage):
# Is usually an IOError
if isinstance(data_object[1], IOError):
sys.stdout.write(data_object[1].strerror + '\n')
# data_object[1].filename
else:
import traceback
traceback.print_exception(data_object[0],
data_object[1],
data_object[2])
sys.exit()
continue
filein = data_object.filename
data_object.header["Omega"] = float(omega_list[inc])
data_object = correct(
data_object,
darkimage,
floodimage,
do_median=options.median,
monitorval=options.monitorval,
monitorcol=options.monitorcol,
)
t.tick(filein + " io/cor")
peaksearch(filein, data_object, corrfunc, thresholds_list,
li_objs)
for t in thresholds_list:
li_objs[t].finalise()
go_for_it(file_series_object, darkimage, floodimage, corrfunc,
thresholds_list, li_objs)
else:
print("Going to use threaded version!")
try:
# TODO move this to a module ?
class read_only(ImageD11_thread.ImageD11_thread):
def __init__(self, queue, file_series_obj, myname="read_only"):
""" Reads files in file_series_obj, writes to queue """
self.queue = queue
self.file_series_obj = file_series_obj
ImageD11_thread.ImageD11_thread.__init__(self,
myname=myname)
print("Reading thread initialised", end=' ')
def ImageD11_run(self):
""" Read images and copy them to self.queue """
for inc, data_object in enumerate(self.file_series_obj):
if self.ImageD11_stop_now():
print("Reader thread stopping")
break
if not isinstance(data_object,
fabio.fabioimage.fabioimage):
# Is usually an IOError
if isinstance(data_object[1], IOError):
# print data_object
# print data_object[1]
sys.stdout.write(
str(data_object[1].strerror) + '\n')
# ': ' + data_object[1].filename + '\n')
else:
import traceback
traceback.print_exception(
data_object[0], data_object[1],
data_object[2])
sys.exit()
continue
ti = timer()
filein = data_object.filename + "[%d]" % (
data_object.currentframe)
try:
data_object.header["Omega"] = float(
omega_list[inc])
except KeyboardInterrupt:
raise
except:
continue
ti.tick(filein)
self.queue.put((filein, data_object), block=True)
ti.tock(" enqueue ")
if self.ImageD11_stop_now():
print("Reader thread stopping")
break
# Flag the end of the series
self.queue.put((None, None), block=True)
class correct_one_to_many(ImageD11_thread.ImageD11_thread):
def __init__(self,
queue_read,
queues_out,
thresholds_list,
dark=None,
flood=None,
myname="correct_one",
monitorcol=None,
monitorval=None,
do_median=False):
""" Using a single reading queue retains a global ordering
corrects and copies images to output queues doing
correction once """
self.queue_read = queue_read
self.queues_out = queues_out
self.dark = dark
self.flood = flood
self.do_median = do_median
self.monitorcol = monitorcol
self.monitorval = monitorval
self.thresholds_list = thresholds_list
ImageD11_thread.ImageD11_thread.__init__(self,
myname=myname)
def ImageD11_run(self):
while not self.ImageD11_stop_now():
ti = timer()
filein, data_object = self.queue_read.get(block=True)
if filein is None:
for t in self.thresholds_list:
self.queues_out[t].put((None, None),
block=True)
# exit the while 1
break
data_object = correct(
data_object,
self.dark,
self.flood,
do_median=self.do_median,
monitorval=self.monitorval,
monitorcol=self.monitorcol,
)
ti.tick(filein + " correct ")
for t in self.thresholds_list:
# Hope that data object is read only
self.queues_out[t].put((filein, data_object),
block=True)
ti.tock(" enqueue ")
print("Corrector thread stopping")
class peaksearch_one(ImageD11_thread.ImageD11_thread):
def __init__(self,
q,
corrfunc,
threshold,
li_obj,
myname="peaksearch_one"):
""" This will handle a single threshold and labelimage
object """
self.q = q
self.corrfunc = corrfunc
self.threshold = threshold
self.li_obj = li_obj
ImageD11_thread.ImageD11_thread.__init__(
self, myname=myname + "_" + str(threshold))
def run(self):
while not self.ImageD11_stop_now():
filein, data_object = self.q.get(block=True)
if not isinstance(data_object,
fabio.fabioimage.fabioimage):
break
peaksearch(filein, data_object, self.corrfunc,
[self.threshold],
{self.threshold: self.li_obj})
self.li_obj.finalise()
# 8 MB images - max 40 MB in this queue
read_queue = queue.Queue(5)
reader = read_only(read_queue, file_series_object)
reader.start()
queues = {}
searchers = {}
for t in thresholds_list:
print("make queue and peaksearch for threshold", t)
queues[t] = queue.Queue(3)
searchers[t] = peaksearch_one(queues[t], corrfunc, t,
li_objs[t])
corrector = correct_one_to_many(read_queue,
queues,
thresholds_list,
dark=darkimage,
flood=floodimage,
do_median=options.median,
monitorcol=options.monitorcol,
monitorval=options.monitorval)
corrector.start()
my_threads = [reader, corrector]
for t in thresholds_list[::-1]:
searchers[t].start()
my_threads.append(searchers[t])
nalive = len(my_threads)
def empty_queue(q):
while 1:
try:
q.get(block=False, timeout=1)
except:
break
q.put((None, None), block=False)
while nalive > 0:
try:
nalive = 0
for thr in my_threads:
if thr.isAlive():
nalive += 1
if options.killfile is not None and \
os.path.exists(options.killfile):
raise KeyboardInterrupt()
time.sleep(1)
except KeyboardInterrupt:
print("Got keyboard interrupt in waiting loop")
ImageD11_thread.stop_now = True
try:
time.sleep(1)
except:
pass
empty_queue(read_queue)
for t in thresholds_list:
q = queues[t]
empty_queue(q)
for thr in my_threads:
if thr.isAlive():
thr.join(timeout=1)
print("finishing from waiting loop")
except:
print("Caught exception in waiting loop")
ImageD11_thread.stop_now = True
time.sleep(1)
empty_queue(read_queue)
for t in thresholds_list:
q = queues[t]
empty_queue(q)
for thr in my_threads:
if thr.isAlive():
thr.join(timeout=1)
raise
except ImportError:
print("Probably no threading module present")
raise
def readspline(self,spline):
from ImageD11 import blobcorrector
self.corrector = blobcorrector.correctorclass(spline)
import sys
"""
Script for repairing use of incorrect spline file
"""
import h5py, tqdm, numpy as np
from ImageD11 import columnfile, blobcorrector
spline = sys.argv[1]
cor = blobcorrector.correctorclass(spline)
h5in = sys.argv[2]
pks = sys.argv[3]
outname = sys.argv[4]
with h5py.File(h5in, 'r') as hin:
print(list(hin), pks)
g = hin[pks]
cd = {name: g[name][:] for name in list(g)}
cd['sc'] = np.zeros_like(cd['s_raw'])
cd['fc'] = np.zeros_like(cd['f_raw'])
inc = columnfile.colfile_from_dict(cd)
for i in tqdm.tqdm(range(inc.nrows)):
inc.sc[i], inc.fc[i] = cor.correct(inc.s_raw[i], inc.f_raw[i])
columnfile.colfile_to_hdf(inc, outname)
line = fmt % (gid, allhkls[i][0], allhkls[i][1], allhkls[i][2],
alltth[i], alleta[i], allomega[i], allsc[i],
allfc[i], sraw[i], fraw[i])
strs.append((allomega[i], line)) # to be omega sortable
return strs
if __name__ == "__main__":
grains = grain.read_grain_file(sys.argv[1])
pars = parameters.read_par_file(sys.argv[2])
detector_size = (2048, 2048)
spline = sys.argv[3]
if spline == "perfect":
spatial = blobcorrector.perfect()
else:
spatial = blobcorrector.correctorclass(spline)
outfile = sys.argv[4]
tthmax, dsmax = tth_ds_max(pars, detector_size)
print(
"# id h k l tth eta omega sc fc s_raw f_raw"
)
peaks = []
for gid, gr in enumerate(grains):
newpeaks = forwards_project(gr, pars, detector_size, spatial, dsmax,
gid)
peaks += newpeaks
print("# Grain", gid, "npks", len(newpeaks))
peaks.sort()
out = open(outfile, "w")
out.write(
"# id h k l tth eta omega sc fc s_raw f_raw\n"
