def __init__(self, dims, pars, ubi, splinefile=None, np=16, border=10, omegarange=list(range(360)), maxpix=None, mask=None): """ Create a new mapper intance. It will transform images into reciprocal space (has its own rsv object holding the space) dims - image dimensions par - ImageD11 parameter filename for experiment ubi - Orientation matrix (ImageD11 style) np - Number of pixels per hkl index [16] border - amount to add around edge of images [10] omegarange - omega values to be mapped (0->360) maxpix - value for saturated pixels to be ignored mask - fit2d style mask for removing bad pixels / border """ if len(dims) != 2: raise Exception("For 2D dims!") self.dims = dims print(dims) # Experiment parameters if not isinstance(pars, parameters.parameters): raise Exception("Pars should be an ImageD11 parameters object") for key in ["distance", "wavelength"]: #etc assert key in pars.parameters self.pars = pars # Orientation matrix self.ubi = ubi # Saturation self.maxpix = maxpix # Mask self.mask = mask if self.mask is not None: assert self.mask.shape == self.dims, "Mask dimensions mush match image" # spatial if splinefile is None: self.spatial = blobcorrector.perfect() else: self.spatial = blobcorrector.correctorclass(splinefile) # npixels self.np = np self.uspace = np * ubi self.find_vol(border=border, omegarange=omegarange) self.rsv.metadata['ubi'] = ubi self.rsv.metadata['uspace'] = self.uspace # Make and cache the k vectors self.make_k_vecs()
def __init__(self, param, hkl, killfile=None): self.killfile = killfile self.param = param self.hkl = hkl self.grain = [] # Simple transforms of input and set constants self.K = -2 * n.pi / self.param['wavelength'] self.S = n.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) # Detector tilt correction matrix self.R = tools.detect_tilt(self.param['tilt_x'], self.param['tilt_y'], self.param['tilt_z']) # wedge NB! wedge is in degrees # The sign is reversed for wedge as the parameter in # tools.find_omega_general is right handed and in ImageD11 # it is left-handed (at this point wedge is defined as in ImageD11) self.wy = -1. * self.param['wedge'] * n.pi / 180. self.wx = 0. # Spatial distortion if self.param['spatial'] != None: from ImageD11 import blobcorrector self.spatial = blobcorrector.correctorclass(self.param['spatial']) # %No of images self.nframes = (self.param['omega_end'] - self.param['omega_start']) / self.param['omega_step'] # Generate Miller indices for reflections within a certain resolution print('Generating reflections') print('Finished generating reflections\n')
def make_powder_mask( parfile, ndeg = 1, splinefile=None, dims=(2048, 2048) ): """ Compute a two theta and azimuth image """ pars = parameters.parameters() pars.loadparameters( parfile ) if splinefile is None: spatial = blobcorrector.perfect() else: spatial = blobcorrector.correctorclass( splinefile ) xim, yim = spatial.make_pixel_lut ( dims ) peaks = [ np.ravel( xim ) , np.ravel( yim ) ] tth , eta = transform.compute_tth_eta( peaks , **pars.get_parameters() ) tth.shape = dims eta.shape = dims # Assume a circle geometry for now # tth * eta ~ length on detector # lim = tth * eta # need some idea how to cut it up... # degree bins m = (eta.astype(np.int) % 2)==0 return m
def compute_tth_eta_lut(splinefile, pars, dims): """ Computes look up values of tth, eta for each pixel """ c = blobcorrector.correctorclass(splinefile) p = parameters.read_par_file(pars) xp, yp = c.make_pixel_lut(dims) t, e = transform.compute_tth_eta((xp.ravel(), yp.ravel()), **p.parameters) t.shape = dims e.shape = dims return t, e
def spatial_correct(s_raw, f_raw, spline): si = np.round(s_raw).clip(0, 2047).astype(int) fi = np.round(f_raw).clip(0, 2047).astype(int) try: dx, dy = spline fc = f_raw + dx[si, fi] sc = s_raw + dy[si, fi] except: corr = blobcorrector.correctorclass(spline) dx, dy = corr.make_pixel_lut((2048, 2048)) fc = f_raw - fi + dy[si, fi] sc = s_raw - si + dx[si, fi] return sc, fc
def __init__(self, pars): """ pars is a dictionary containing the calibration parameters """ self.pars = {} for p in self.pnames: if p in pars: self.pars[p] = pars[p] # make a copy if 'dxfile' in pars: # slow/fast coordinates on image at pixel centers self.df = fabio.open(pars['dxfile']).data self.ds = fabio.open(pars['dyfile']).data self.shape = s = self.ds.shape # get shape from file self.pars['shape'] = s slow, fast = np.mgrid[0:s[0], 0:s[1]] self.sc = slow + self.ds self.fc = fast + self.df elif 'spline' in pars: # need to test this... from ImageD11 import blobcorrector b = blobcorrector.correctorclass(self.pars['spline']) s = int(b.ymax - b.ymin), int(b.xmax - b.xmin) if 'shape' in self.pars: # override. Probabl s = self.pars['shape'] self.shape = s self.fc, self.sc = b.make_pixel_lut(s) slow, fast = np.mgrid[0:s[0], 0:s[1]] self.df = self.fc - fast self.ds = self.sc - slow else: s = self.shape self.sc, self.fc = np.mgrid[0:s[0], 0:s[1]] self.df = None self.ds = None self.xyz = compute_xyz_lab(self.sc.ravel(), self.fc.ravel(), **self.pars) self.sinthsq = compute_sinsqth_from_xyz(self.xyz) self.tth, self.eta = compute_tth_eta_from_xyz(self.peaks_xyz, None, **self.pars) # scattering angles: self.tth, self.eta = compute_tth_eta( (self.sc.ravel(), self.fc.ravel()), **self.pars) # scattering vectors: self.k = compute_k_vectors(self.tth, self.eta, self.pars.get('wavelength')) self.sinthsq.shape = s self.tth.shape = s self.eta.shape = s self.k.shape = (3, s[0], s[1]) self.xyz.shape = (3, s[0], s[1])
def __init__(self,param,hkl,killfile=None): ''' A find_refl object is used to hold information that otherwise would have to be recomputed for during the algorithm execution in run(). ''' self.killfile = killfile self.param = param self.hkl = hkl self.voxel = [None]*self.param['no_voxels'] # Simple transforms of input and set constants self.K = -2*np.pi/self.param['wavelength'] self.S = np.array([[1, 0, 0],[0, 1, 0],[0, 0, 1]]) # Detector tilt correction matrix self.R = tools.detect_tilt(self.param['tilt_x'], self.param['tilt_y'], self.param['tilt_z']) # wedge NB! wedge is in degrees # The sign is reversed for wedge as the parameter in # tools.find_omega_general is right handed and in ImageD11 # it is left-handed (at this point wedge is defined as in ImageD11) self.wy = -1.*self.param['wedge']*np.pi/180. self.wx = 0. w_mat_x = np.array([[1, 0 , 0 ], [0, np.cos(self.wx), -np.sin(self.wx)], [0, np.sin(self.wx), np.cos(self.wx)]]) w_mat_y = np.array([[ np.cos(self.wy), 0, np.sin(self.wy)], [0 , 1, 0 ], [-np.sin(self.wy), 0, np.cos(self.wy)]]) self.r_mat = np.dot(w_mat_x,w_mat_y) # Spatial distortion if self.param['spatial'] != None: from ImageD11 import blobcorrector self.spatial = blobcorrector.correctorclass(self.param['spatial']) # %No of images self.nframes = (self.param['omega_end']-self.param['omega_start'])/self.param['omega_step'] self.peak_merger = cms_peak_compute.PeakMerger( self.param )
def mymain(): # If we are running from a command line: inname = sys.argv[1] if not os.path.exists(inname) or len(sys.argv) < 4: help() sys.exit() outname = sys.argv[2] if os.path.exists(outname): if not input("Sure you want to overwrite %s ?" % (outname))[0] in ['y', 'Y']: sys.exit() splinename = sys.argv[3] if splinename == 'perfect': cor = blobcorrector.perfect() else: cor = blobcorrector.correctorclass(splinename) fix_flt(inname, outname, cor)
def __init__(self, splinefile=None, parfile=None): """ splinefile = fit2d spline file, or None for images that are already corrected parfile = ImageD11 parameter file. Can be fitted using old ImageD11_gui.py or newer fable.transform plugin. """ self.splinefile = splinefile if self.splinefile is None: self.spatial = blobcorrector.perfect() else: self.spatial = blobcorrector.correctorclass(splinefile) self.parfile = parfile self.pars = parameters.parameters() self.pars.loadparameters(parfile) for key in self.required_pars: if key not in self.pars.parameters: raise Exception("Missing parameter " + str(par))
def peaksearch_driver(options, args): """ To be called with options from command line """ ################## debugging still for a in args: print("arg: "+str(a)+","+str(type(a))) for o in list(options.__dict__.keys()): # FIXME if getattr(options,o) in [ "False", "FALSE", "false" ]: setattr(options,o,False) if getattr(options,o) in [ "True", "TRUE", "true" ]: setattr(options,o,True) print("option:",str(o),str(getattr(options,o)),",",\ str(type( getattr(options,o) ) )) ################### print("This peaksearcher is from",__file__) if options.killfile is not None and os.path.exists(options.killfile): print("The purpose of the killfile option is to create that file") print("only when you want peaksearcher to stop") print("If the file already exists when you run peaksearcher it is") print("never going to get started") raise ValueError("Your killfile "+options.killfile+" already exists") if options.thresholds is None: raise ValueError("No thresholds supplied [-t 1234]") if len(args) == 0 and options.stem is None: raise ValueError("No files to process") # What to do about spatial if options.perfect=="N" and os.path.exists(options.spline): print("Using spatial from",options.spline) corrfunc = blobcorrector.correctorclass(options.spline) else: print("Avoiding spatial correction") corrfunc = blobcorrector.perfect() # This is always the case now corrfunc.orientation = "edf" scan = None if options.format in ['bruker', 'BRUKER', 'Bruker']: extn = "" if options.perfect is not "N": print("WARNING: Your spline file is ImageD11 specific") print("... from a fabio converted to edf first") elif options.format == 'GE': extn = "" # KE: This seems to be a mistake and keeps PeakSearch from working in # some cases. Should be revisited if commenting it out causes problems. # options.ndigits = 0 elif options.format == 'py': import importlib sys.path.append( '.' ) scan = importlib.import_module( options.stem ) first_image = scan.first_image file_series_object = scan.file_series_object else: extn = options.format if scan is None: if options.interlaced: f0 = ["%s0_%04d%s"%(options.stem,i,options.format) for i in range( options.first, options.last+1)] f1 = ["%s1_%04d%s"%(options.stem,i,options.format) for i in range( options.first, options.last+1)] if options.iflip: f1 = [a for a in f1[::-1]] def fso(f0,f1): for a,b in zip(f0,f1): try: yield fabio.open(a) yield fabio.open(b) except: print(a,b) raise file_series_object = fso(f0,f1) first_image = openimage( f0[0] ) else: import fabio if options.ndigits > 0: file_name_object = fabio.filename_object( options.stem, num = options.first, extension = extn, digits = options.ndigits) else: file_name_object = options.stem first_image = openimage( file_name_object ) import fabio.file_series # Use traceback = True for debugging file_series_object = fabio.file_series.new_file_series( first_image, nimages = options.last - options.first + 1, traceback = True ) # Output files: if options.outfile[-4:] != ".spt": options.outfile = options.outfile + ".spt" print("Your output file must end with .spt, changing to ",options.outfile) # Omega overrides # global OMEGA, OMEGASTEP, OMEGAOVERRIDE OMEGA = options.OMEGA OMEGASTEP = options.OMEGASTEP OMEGAOVERRIDE = options.OMEGAOVERRIDE # Make a blobimage the same size as the first image to process # List comprehension - convert remaining args to floats # must be unique list so go via a set thresholds_list = list( set( [float(t) for t in options.thresholds] ) ) thresholds_list.sort() li_objs={} # label image objects, dict of s = first_image.data.shape # data array shape # Create label images for t in thresholds_list: # the last 4 chars are guaranteed to be .spt above mergefile="%s_t%d.flt"%(options.outfile[:-4], t) spotfile = "%s_t%d.spt"%(options.outfile[:-4], t) li_objs[t]=labelimage(shape = s, fileout = mergefile, spatial = corrfunc, sptfile=spotfile) print("make labelimage",mergefile,spotfile) # Not sure why that was there (I think if glob was used) # files.sort() if options.dark is not None: print("Using dark (background)",options.dark) darkimage= openimage(options.dark).data.astype(numpy.float32) else: darkimage=None if options.darkoffset!=0: print("Adding darkoffset",options.darkoffset) if darkimage is None: darkimage = options.darkoffset else: darkimage += options.darkoffset if options.flood is not None: floodimage=openimage(options.flood).data cen0 = int(floodimage.shape[0]/6) cen1 = int(floodimage.shape[0]/6) middle = floodimage[cen0:-cen0, cen1:-cen1] nmid = middle.shape[0]*middle.shape[1] floodavg = numpy.mean(middle) print("Using flood",options.flood,"average value",floodavg) if floodavg < 0.7 or floodavg > 1.3: print("Your flood image does not seem to be normalised!!!") else: floodimage=None start = time.time() print("File being treated in -> out, elapsed time") # If we want to do read-ahead threading we fill up a Queue object with data # objects # THERE MUST BE ONLY ONE peaksearching thread for 3D merging to work # there could be several read_and_correct threads, but they'll have to get the order right, # for now only one if options.oneThread: # Wrap in a function to allow profiling (perhaps? what about globals??) def go_for_it(file_series_object, darkimage, floodimage, corrfunc , thresholds_list , li_objs, OMEGA, OMEGASTEP, OMEGAOVERRIDE ): for data_object in file_series_object: t = timer() if not hasattr( data_object, "data"): # Is usually an IOError if isinstance( data_object[1], IOError): sys.stdout.write(data_object[1].strerror + '\n') # data_object[1].filename else: import traceback traceback.print_exception(data_object[0],data_object[1],data_object[2]) sys.exit() continue filein = data_object.filename if OMEGAOVERRIDE or "Omega" not in data_object.header: data_object.header["Omega"] = OMEGA OMEGA += OMEGASTEP OMEGAOVERRIDE = True # once you do it once, continue if not OMEGAOVERRIDE and options.omegamotor != "Omega": data_object.header["Omega"] = float( data_object.header[options.omegamotor] ) data_object = correct( data_object, darkimage, floodimage, do_median = options.median, monitorval = options.monitorval, monitorcol = options.monitorcol, ) t.tick(filein+" io/cor") peaksearch( filein, data_object , corrfunc , thresholds_list , li_objs ) for t in thresholds_list: li_objs[t].finalise() go_for_it(file_series_object, darkimage, floodimage, corrfunc , thresholds_list, li_objs, OMEGA, OMEGASTEP, OMEGAOVERRIDE ) else: print("Going to use threaded version!") try: # TODO move this to a module ? class read_only(ImageD11_thread.ImageD11_thread): def __init__(self, queue, file_series_obj , myname="read_only", OMEGA=0, OMEGAOVERRIDE = False, OMEGASTEP = 1): """ Reads files in file_series_obj, writes to queue """ self.queue = queue self.file_series_obj = file_series_obj self.OMEGA = OMEGA self.OMEGAOVERRIDE = OMEGAOVERRIDE self.OMEGASTEP = OMEGASTEP ImageD11_thread.ImageD11_thread.__init__(self , myname=myname) print("Reading thread initialised", end=' ') def ImageD11_run(self): """ Read images and copy them to self.queue """ for data_object in self.file_series_obj: if self.ImageD11_stop_now(): print("Reader thread stopping") break if not hasattr( data_object, "data" ): import pdb; pdb.set_trace() # Is usually an IOError if isinstance( data_object[1], IOError): sys.stdout.write(str(data_object[1].strerror) + '\n') else: import traceback traceback.print_exception(data_object[0],data_object[1],data_object[2]) sys.exit() continue ti = timer() filein = data_object.filename + "[%d]"%( data_object.currentframe ) try: if self.OMEGAOVERRIDE: # print "Over ride due to option",self.OMEGAOVERRIDE data_object.header["Omega"] = self.OMEGA self.OMEGA += self.OMEGASTEP else: if options.omegamotor != 'Omega' and options.omegamotor in data_object.header: data_object.header["Omega"] = float(data_object.header[options.omegamotor]) if "Omega" not in data_object.header: # print "Computing omega as not in header" data_object.header["Omega"] = self.OMEGA self.OMEGA += self.OMEGASTEP self.OMEGAOVERRIDE = True # print "Omega = ", data_object.header["Omega"],data_object.filename except KeyboardInterrupt: raise except: continue ti.tick(filein) self.queue.put((filein, data_object) , block = True) ti.tock(" enqueue ") if self.ImageD11_stop_now(): print("Reader thread stopping") break # Flag the end of the series self.queue.put( (None, None) , block = True) class correct_one_to_many(ImageD11_thread.ImageD11_thread): def __init__(self, queue_read, queues_out, thresholds_list, dark = None , flood = None, myname="correct_one", monitorcol = None, monitorval = None, do_median = False): """ Using a single reading queue retains a global ordering corrects and copies images to output queues doing correction once """ self.queue_read = queue_read self.queues_out = queues_out self.dark = dark self.flood = flood self.do_median = do_median self.monitorcol = monitorcol self.monitorval = monitorval self.thresholds_list = thresholds_list ImageD11_thread.ImageD11_thread.__init__(self , myname=myname) def ImageD11_run(self): while not self.ImageD11_stop_now(): ti = timer() filein, data_object = self.queue_read.get(block = True) if filein is None: for t in self.thresholds_list: self.queues_out[t].put( (None, None) , block = True) # exit the while 1 break data_object = correct(data_object, self.dark, self.flood, do_median = self.do_median, monitorval = self.monitorval, monitorcol = self.monitorcol, ) ti.tick(filein+" correct ") for t in self.thresholds_list: # Hope that data object is read only self.queues_out[t].put((filein, data_object) , block = True) ti.tock(" enqueue ") print("Corrector thread stopping") class peaksearch_one(ImageD11_thread.ImageD11_thread): def __init__(self, q, corrfunc, threshold, li_obj, myname="peaksearch_one" ): """ This will handle a single threshold and labelimage object """ self.q = q self.corrfunc = corrfunc self.threshold = threshold self.li_obj = li_obj ImageD11_thread.ImageD11_thread.__init__( self, myname=myname+"_"+str(threshold)) def run(self): while not self.ImageD11_stop_now(): filein, data_object = self.q.get(block = True) if not hasattr( data_object, "data" ): break peaksearch( filein, data_object , self.corrfunc , [self.threshold] , { self.threshold : self.li_obj } ) self.li_obj.finalise() # 8 MB images - max 40 MB in this queue read_queue = queue.Queue(5) reader = read_only(read_queue, file_series_object, OMEGA = OMEGA, OMEGASTEP = OMEGASTEP, OMEGAOVERRIDE = OMEGAOVERRIDE ) reader.start() queues = {} searchers = {} for t in thresholds_list: print("make queue and peaksearch for threshold",t) queues[t] = queue.Queue(3) searchers[t] = peaksearch_one(queues[t], corrfunc, t, li_objs[t] ) corrector = correct_one_to_many( read_queue, queues, thresholds_list, dark=darkimage, flood=floodimage, do_median = options.median, monitorcol = options.monitorcol, monitorval = options.monitorval) corrector.start() my_threads = [reader, corrector] for t in thresholds_list[::-1]: searchers[t].start() my_threads.append(searchers[t]) nalive = len(my_threads) def empty_queue(q): while 1: try: q.get(block=False, timeout=1) except: break q.put((None, None), block=False) while nalive > 0: try: nalive = 0 for thr in my_threads: if thr.isAlive(): nalive += 1 if options.killfile is not None and \ os.path.exists(options.killfile): raise KeyboardInterrupt() time.sleep(1) except KeyboardInterrupt: print("Got keyboard interrupt in waiting loop") ImageD11_thread.stop_now = True try: time.sleep(1) except: pass empty_queue(read_queue) for t in thresholds_list: q = queues[t] empty_queue(q) for thr in my_threads: if thr.isAlive(): thr.join(timeout=1) print("finishing from waiting loop") except: print("Caught exception in waiting loop") ImageD11_thread.stop_now = True time.sleep(1) empty_queue(read_queue) for t in thresholds_list: q = queues[t] empty_queue(q) for thr in my_threads: if thr.isAlive(): thr.join(timeout=1) raise except ImportError: print("Probably no threading module present") raise
def make_image(self, frame_number=None): """ makeimage script produces edf diffraction images using the reflection information Henning Osholm Sorensen, June 23, 2006. python translation Jette Oddershede, March 31, 2008 """ peakshape = self.graindata.param['peakshape'] if peakshape[0] == 0: # spike peak, 2x2 pixels peak_add = 1 frame_add = 1 peakwsig = 0 elif peakshape[0] == 1: # 3d Gaussian peak peak_add = max(1, int(round(peakshape[1]))) frame_add = max(1, int(round(peakshape[1]))) peakwsig = peakshape[2] elif peakshape[0] == 3: # 3d Gaussian peak in 2theta,eta,omega peak_add = 1 frame_add = 1 cen_tth = int(1.5 * peakshape[1] / Delta_tth) frame_tth = 2 * cen_tth + 1 fwhm_tth = peakshape[1] / Delta_tth cen_eta = int(1.5 * peakshape[2] / Delta_eta) frame_eta = 2 * cen_eta + 1 fwhm_eta = peakshape[2] / Delta_eta raw_tth_eta = n.zeros((frame_tth, frame_eta)) raw_tth_eta[cen_tth, cen_eta] = 1 filter_tth_eta = ndimage.gaussian_filter( raw_tth_eta, [0.5 * fwhm_tth, 0.5 * fwhm_eta]) peakwsig = 1. framedimy = int(self.graindata.param['dety_size'] + 2 * frame_add) framedimz = int(self.graindata.param['detz_size'] + 2 * frame_add) totalrefl = 0 if frame_number == None: no_frames = list(range(len(self.graindata.frameinfo))) print('Generating diffraction images') else: no_frames = [frame_number] for i in no_frames: check_input.interrupt(self.killfile) t1 = time.clock() nrefl = 0 frame = n.zeros((framedimy, framedimz)) omega = self.graindata.frameinfo[i].omega omega_step = self.graindata.param['omega_step'] # Jettes hack to add relative movement of sample and detector, modelled to be Gaussian in y and z direction with a spread of 1 micron # movement of 1 micron along x judged to be irrelevant, at least for farfield data y_move = n.random.normal(0, 1. / self.graindata.param['dety_size']) z_move = n.random.normal(0, 1. / self.graindata.param['detz_size']) # loop over grains for j in range(self.graindata.param['no_grains']): # loop over reflections for each grain gr_pos = n.array(self.graindata.param['pos_grains_%s' % j]) for k in range(len(self.graindata.grain[j].refs)): # exploit that the reflection list is sorted according to omega if self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi > \ omega+omega_step+2*peakwsig: break elif self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi < \ omega-2*peakwsig: continue dety = self.graindata.grain[j].refs[ k, A_id['detyd']] # must be spot position after detz = self.graindata.grain[j].refs[ k, A_id['detzd']] # applying spatial distortion #apply hack # dety = self.graindata.grain[j].refs[k,A_id['dety']] + y_move # detz = self.graindata.grain[j].refs[k,A_id['detz']] + z_move ndety = int(round(dety)) ndetz = int(round(detz)) yrange = list( range(ndety + frame_add - peak_add, ndety + frame_add + peak_add + 1)) zrange = list( range(ndetz + frame_add - peak_add, ndetz + frame_add + peak_add + 1)) intensity = int( round(self.graindata.grain[j].refs[k, A_id['Int']])) nrefl = nrefl + 1 totalrefl = totalrefl + 1 # Gaussian along omega if peakshape[0] == 1 or peakshape[0] == 3: fraction = norm.cdf((omega-self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi+omega_step)/(0.5*peakwsig))\ -norm.cdf((omega-self.graindata.grain[j].refs[k,A_id['omega']]*180/n.pi)/(0.5*peakwsig)) else: fraction = 1. if peakshape[0] == 3: # Gaussian peaks along 2theta,eta tth = self.graindata.grain[j].refs[k, A_id['tth']] eta = self.graindata.grain[j].refs[k, A_id['eta']] Om = tools.form_omega_mat_general( self.graindata.grain[j].refs[k, A_id['omega']], 0, -1. * self.graindata.param['wedge'] * n.pi / 180.) [tx, ty, tz] = n.dot(Om, gr_pos) for t in range(frame_tth): tth_present = tth + ( t - cen_tth) * Delta_tth * n.pi / 180. for e in range(frame_eta): eta_present = eta + ( e - cen_eta) * Delta_eta * n.pi / 180. [dety_present, detz_present] = detector.det_coor2( tth_present, eta_present, self.graindata.param['distance'], self.graindata.param['y_size'], self.graindata.param['z_size'], self.graindata.param['dety_center'], self.graindata.param['detz_center'], self.graindata.R, tx, ty, tz, ) if self.graindata.param['spatial'] != None: from ImageD11 import blobcorrector self.spatial = blobcorrector.correctorclass( self.graindata.param['spatial']) # To match the coordinate system of the spline file # SPLINE(i,j): i = detz; j = (dety_size-1)-dety # Well at least if the spline file is for frelon2k (x, y) = detector.detyz_to_xy( [dety_present, detz_present], self.graindata.param['o11'], self.graindata.param['o12'], self.graindata.param['o21'], self.graindata.param['o22'], self.graindata.param['dety_size'], self.graindata.param['detz_size']) # Do the spatial distortion (xd, yd) = self.spatial.distort(x, y) # transform coordinates back to dety,detz (dety_present, detz_present) = detector.xy_to_detyz( [xd, yd], self.graindata.param['o11'], self.graindata.param['o12'], self.graindata.param['o21'], self.graindata.param['o22'], self.graindata.param['dety_size'], self.graindata.param['detz_size']) y = int(round(dety_present)) z = int(round(detz_present)) try: frame[y + frame_add, z + frame_add] = frame[ y + frame_add, z + frame_add] + fraction * intensity * filter_tth_eta[ t, e] except: # FIXME print("Unhandled exception in make_image.py") pass else: # Generate spikes, 2x2 pixels for y in yrange: for z in zrange: if y > 0 and y < framedimy and z > 0 and z < framedimz and abs( dety + frame_add - y) < 1 and abs( detz + frame_add - z) < 1: # frame[y-1,z] = frame[y-1,z] + fraction*intensity*(1-abs(dety+frame_add-y))*(1-abs(detz+frame_add-z)) y = int(round(y)) z = int(round(z)) frame[y, z] = frame[ y, z] + fraction * intensity * ( 1 - abs(dety + frame_add - y)) * ( 1 - abs(detz + frame_add - z)) # 2D Gaussian on detector if peakshape[0] == 1: frame = ndimage.gaussian_filter(frame, peakshape[1] * 0.5) # add background if self.graindata.param['bg'] > 0: frame = frame + self.graindata.param['bg'] * n.ones( (framedimy, framedimz)) # add noise if self.graindata.param['noise'] != 0: frame = n.random.poisson(frame) # apply psf if self.graindata.param['psf'] != 0: frame = ndimage.gaussian_filter( frame, self.graindata.param['psf'] * 0.5) # resize, convert to integers and flip to same orientation as experimental frames frame = frame[frame_add:framedimy - frame_add, frame_add:framedimz - frame_add] # limit values above 16 bit to be 16bit frame = n.clip(frame, 0, 2**16 - 1) # convert to integers frame = n.uint16(frame) #flip detector orientation according to input: o11, o12, o21, o22 frame = detector.trans_orientation(frame, self.graindata.param['o11'], self.graindata.param['o12'], self.graindata.param['o21'], self.graindata.param['o22'], 'inverse') # Output frames if '.edf' in self.graindata.param['output']: self.write_edf(i, frame) if '.edf.gz' in self.graindata.param['output']: self.write_edf(i, frame, usegzip=True) if '.tif' in self.graindata.param['output']: self.write_tif(i, frame) if '.tif16bit' in self.graindata.param['output']: self.write_tif16bit(i, frame) print('\rDone frame %i took %8f s' % (i + 1, time.clock() - t1), end=' ') sys.stdout.flush()
def peaksearch_driver(options, args): """ To be called with options from command line """ ################## debugging still for a in args: print("arg: " + str(a) + "," + str(type(a))) for o in list(options.__dict__.keys()): # FIXME if getattr(options, o) in ["False", "FALSE", "false"]: setattr(options, o, False) if getattr(options, o) in ["True", "TRUE", "true"]: setattr(options, o, True) print("option:", str(o), str(getattr(options, o)), ",", \ str(type(getattr(options, o)))) ################### print("This peaksearcher is from", __file__) if options.killfile is not None and os.path.exists(options.killfile): print("The purpose of the killfile option is to create that file") print("only when you want peaksearcher to stop") print("If the file already exists when you run peaksearcher it is") print("never going to get started") raise ValueError("Your killfile " + options.killfile + " already exists") if options.thresholds is None: raise ValueError("No thresholds supplied [-t 1234]") if len(args) == 0 and options.nexusfile is None: raise ValueError("No files to process") # What to do about spatial if options.perfect == "N" and os.path.exists(options.spline): print("Using spatial from", options.spline) corrfunc = blobcorrector.correctorclass(options.spline) else: print("Avoiding spatial correction") corrfunc = blobcorrector.perfect() # Get list of filenames to process # if len(args) > 0 : # # We no longer assume unlabelled arguments are filenames # file_series_object = file_series.file_series(args) # This is always the case now corrfunc.orientation = "edf" import h5py # Read list of files and list of motor positions from Nexus file: nexus_path = options.nexusfile nexus_file = h5py.File(nexus_path, "r") group = nexus_file[options.group_path] omega_dset = group.get(options.omega_dset) image_dset = group.get(options.image_dset) omega_list = [x for x in omega_dset[..., :]] image_list = [x.decode("utf-8") for x in image_dset[..., :]] import fabio # Output files: import fabio.file_series # Use traceback = True for debugging first_image = openimage(image_list[0]) file_series_object = fabio.file_series.new_file_series( first_image, nimages=len(image_list), traceback=True) if options.outfile[-4:] != ".spt": options.outfile = options.outfile + ".spt" print("Your output file must end with .spt, changing to ", options.outfile) # Make a blobimage the same size as the first image to process # List comprehension - convert remaining args to floats # must be unique list so go via a set thresholds_list = list(set([float(t) for t in options.thresholds])) thresholds_list.sort() li_objs = {} # label image objects, dict of s = first_image.data.shape # data array shape # Create label images for t in thresholds_list: # the last 4 chars are guaranteed to be .spt above mergefile = "%s_t%d.flt" % (options.outfile[:-4], t) spotfile = "%s_t%d.spt" % (options.outfile[:-4], t) li_objs[t] = labelimage(shape=s, fileout=mergefile, spatial=corrfunc, sptfile=spotfile) print("make labelimage", mergefile, spotfile) if options.dark is not None: print("Using dark (background)", options.dark) darkimage = openimage(options.dark).data.astype(numpy.float32) else: darkimage = None if options.darkoffset != 0: print("Adding darkoffset", options.darkoffset) if darkimage is None: darkimage = options.darkoffset else: darkimage += options.darkoffset if options.flood is not None: floodimage = openimage(options.flood).data cen0 = int(floodimage.shape[0] / 6) cen1 = int(floodimage.shape[0] / 6) middle = floodimage[cen0:-cen0, cen1:-cen1] nmid = middle.shape[0] * middle.shape[1] floodavg = numpy.mean(middle) print("Using flood", options.flood, "average value", floodavg) if floodavg < 0.7 or floodavg > 1.3: print("Your flood image does not seem to be normalised!!!") else: floodimage = None start = time.time() print("File being treated in -> out, elapsed time") # If we want to do read-ahead threading we fill up a Queue object with data # objects # THERE MUST BE ONLY ONE peaksearching thread for 3D merging to work # there could be several read_and_correct threads, but they'll have to get the order right, # for now only one if options.oneThread: # Wrap in a function to allow profiling (perhaps? what about globals??) def go_for_it(file_series_object, darkimage, floodimage, corrfunc, thresholds_list, li_objs): for inc, data_object in enumerate(file_series_object): t = timer() if not isinstance(data_object, fabio.fabioimage.fabioimage): # Is usually an IOError if isinstance(data_object[1], IOError): sys.stdout.write(data_object[1].strerror + '\n') # data_object[1].filename else: import traceback traceback.print_exception(data_object[0], data_object[1], data_object[2]) sys.exit() continue filein = data_object.filename data_object.header["Omega"] = float(omega_list[inc]) data_object = correct( data_object, darkimage, floodimage, do_median=options.median, monitorval=options.monitorval, monitorcol=options.monitorcol, ) t.tick(filein + " io/cor") peaksearch(filein, data_object, corrfunc, thresholds_list, li_objs) for t in thresholds_list: li_objs[t].finalise() go_for_it(file_series_object, darkimage, floodimage, corrfunc, thresholds_list, li_objs) else: print("Going to use threaded version!") try: # TODO move this to a module ? class read_only(ImageD11_thread.ImageD11_thread): def __init__(self, queue, file_series_obj, myname="read_only"): """ Reads files in file_series_obj, writes to queue """ self.queue = queue self.file_series_obj = file_series_obj ImageD11_thread.ImageD11_thread.__init__(self, myname=myname) print("Reading thread initialised", end=' ') def ImageD11_run(self): """ Read images and copy them to self.queue """ for inc, data_object in enumerate(self.file_series_obj): if self.ImageD11_stop_now(): print("Reader thread stopping") break if not isinstance(data_object, fabio.fabioimage.fabioimage): # Is usually an IOError if isinstance(data_object[1], IOError): # print data_object # print data_object[1] sys.stdout.write( str(data_object[1].strerror) + '\n') # ': ' + data_object[1].filename + '\n') else: import traceback traceback.print_exception( data_object[0], data_object[1], data_object[2]) sys.exit() continue ti = timer() filein = data_object.filename + "[%d]" % ( data_object.currentframe) try: data_object.header["Omega"] = float( omega_list[inc]) except KeyboardInterrupt: raise except: continue ti.tick(filein) self.queue.put((filein, data_object), block=True) ti.tock(" enqueue ") if self.ImageD11_stop_now(): print("Reader thread stopping") break # Flag the end of the series self.queue.put((None, None), block=True) class correct_one_to_many(ImageD11_thread.ImageD11_thread): def __init__(self, queue_read, queues_out, thresholds_list, dark=None, flood=None, myname="correct_one", monitorcol=None, monitorval=None, do_median=False): """ Using a single reading queue retains a global ordering corrects and copies images to output queues doing correction once """ self.queue_read = queue_read self.queues_out = queues_out self.dark = dark self.flood = flood self.do_median = do_median self.monitorcol = monitorcol self.monitorval = monitorval self.thresholds_list = thresholds_list ImageD11_thread.ImageD11_thread.__init__(self, myname=myname) def ImageD11_run(self): while not self.ImageD11_stop_now(): ti = timer() filein, data_object = self.queue_read.get(block=True) if filein is None: for t in self.thresholds_list: self.queues_out[t].put((None, None), block=True) # exit the while 1 break data_object = correct( data_object, self.dark, self.flood, do_median=self.do_median, monitorval=self.monitorval, monitorcol=self.monitorcol, ) ti.tick(filein + " correct ") for t in self.thresholds_list: # Hope that data object is read only self.queues_out[t].put((filein, data_object), block=True) ti.tock(" enqueue ") print("Corrector thread stopping") class peaksearch_one(ImageD11_thread.ImageD11_thread): def __init__(self, q, corrfunc, threshold, li_obj, myname="peaksearch_one"): """ This will handle a single threshold and labelimage object """ self.q = q self.corrfunc = corrfunc self.threshold = threshold self.li_obj = li_obj ImageD11_thread.ImageD11_thread.__init__( self, myname=myname + "_" + str(threshold)) def run(self): while not self.ImageD11_stop_now(): filein, data_object = self.q.get(block=True) if not isinstance(data_object, fabio.fabioimage.fabioimage): break peaksearch(filein, data_object, self.corrfunc, [self.threshold], {self.threshold: self.li_obj}) self.li_obj.finalise() # 8 MB images - max 40 MB in this queue read_queue = queue.Queue(5) reader = read_only(read_queue, file_series_object) reader.start() queues = {} searchers = {} for t in thresholds_list: print("make queue and peaksearch for threshold", t) queues[t] = queue.Queue(3) searchers[t] = peaksearch_one(queues[t], corrfunc, t, li_objs[t]) corrector = correct_one_to_many(read_queue, queues, thresholds_list, dark=darkimage, flood=floodimage, do_median=options.median, monitorcol=options.monitorcol, monitorval=options.monitorval) corrector.start() my_threads = [reader, corrector] for t in thresholds_list[::-1]: searchers[t].start() my_threads.append(searchers[t]) nalive = len(my_threads) def empty_queue(q): while 1: try: q.get(block=False, timeout=1) except: break q.put((None, None), block=False) while nalive > 0: try: nalive = 0 for thr in my_threads: if thr.isAlive(): nalive += 1 if options.killfile is not None and \ os.path.exists(options.killfile): raise KeyboardInterrupt() time.sleep(1) except KeyboardInterrupt: print("Got keyboard interrupt in waiting loop") ImageD11_thread.stop_now = True try: time.sleep(1) except: pass empty_queue(read_queue) for t in thresholds_list: q = queues[t] empty_queue(q) for thr in my_threads: if thr.isAlive(): thr.join(timeout=1) print("finishing from waiting loop") except: print("Caught exception in waiting loop") ImageD11_thread.stop_now = True time.sleep(1) empty_queue(read_queue) for t in thresholds_list: q = queues[t] empty_queue(q) for thr in my_threads: if thr.isAlive(): thr.join(timeout=1) raise except ImportError: print("Probably no threading module present") raise
def readspline(self,spline): from ImageD11 import blobcorrector self.corrector = blobcorrector.correctorclass(spline)
import sys """ Script for repairing use of incorrect spline file """ import h5py, tqdm, numpy as np from ImageD11 import columnfile, blobcorrector spline = sys.argv[1] cor = blobcorrector.correctorclass(spline) h5in = sys.argv[2] pks = sys.argv[3] outname = sys.argv[4] with h5py.File(h5in, 'r') as hin: print(list(hin), pks) g = hin[pks] cd = {name: g[name][:] for name in list(g)} cd['sc'] = np.zeros_like(cd['s_raw']) cd['fc'] = np.zeros_like(cd['f_raw']) inc = columnfile.colfile_from_dict(cd) for i in tqdm.tqdm(range(inc.nrows)): inc.sc[i], inc.fc[i] = cor.correct(inc.s_raw[i], inc.f_raw[i]) columnfile.colfile_to_hdf(inc, outname)
line = fmt % (gid, allhkls[i][0], allhkls[i][1], allhkls[i][2], alltth[i], alleta[i], allomega[i], allsc[i], allfc[i], sraw[i], fraw[i]) strs.append((allomega[i], line)) # to be omega sortable return strs if __name__ == "__main__": grains = grain.read_grain_file(sys.argv[1]) pars = parameters.read_par_file(sys.argv[2]) detector_size = (2048, 2048) spline = sys.argv[3] if spline == "perfect": spatial = blobcorrector.perfect() else: spatial = blobcorrector.correctorclass(spline) outfile = sys.argv[4] tthmax, dsmax = tth_ds_max(pars, detector_size) print( "# id h k l tth eta omega sc fc s_raw f_raw" ) peaks = [] for gid, gr in enumerate(grains): newpeaks = forwards_project(gr, pars, detector_size, spatial, dsmax, gid) peaks += newpeaks print("# Grain", gid, "npks", len(newpeaks)) peaks.sort() out = open(outfile, "w") out.write( "# id h k l tth eta omega sc fc s_raw f_raw\n"