def testmissingfile(self): try: e = FileNotFoundError except: e = IOError with self.assertRaises((IOError, e)): columnfile.columnfile("a_filename_that_does_not_exist.flt")
def test_to_hdf(self): c = columnfile.columnfile("test.col") columnfile.colfile_to_hdf(c, "testcolfile.hdf") h = columnfile.columnfile("testcolfile.hdf") for t in c.titles: assert ((c.getcolumn(t) == h.getcolumn(t)).all()) assert t in h.titles
def two2one(in1, in2, out): """ Example: merge.two2one('peaks_fa.flt','peaks_fb.flt','merged.flt') This merges files peaks_fa.flt and peaks_fb.flt into merged.flt Jette Oddershede, October 2008 """ f1 = ic.columnfile(in1) f2 = ic.columnfile(in2) print(in1, f1.nrows) print(in2, f2.nrows) assert f1.titles == f2.titles, 'Attempting to merge columnfiles with different column titles' f = open(in1, 'r') flt1 = f.read() f.close() f = open(in2, 'r') flt2lines = f.readlines() f.close() flt2 = '' for i in range(len(flt2lines)): if '#' not in flt2lines[i]: flt2 = flt2 + flt2lines[i] flt = open(out, 'w') flt.write(flt1 + '\n' + flt2) flt.close print(out, f1.nrows + f2.nrows)
def testmissingfile(self): e = False try: columnfile.columnfile("a_filename_that_does_not_exist.flt") except: # got an exception e = True assert (e)
def recon_slice(self, flt_file, grain_file, label): print("\n \nStarting reconstruction of slice: " + label) print(flt_file) flt = columnfile.columnfile(flt_file) #--------------------------------------------------------------------------- # Fix the dty column of the flt such that it scanns across 0 anddty is in microns. flt.dty[:] = ((flt.dty - self.ymin_flt) * self.unit / (10**(-6))) - self.ystep * (self.number_y_scans // 2) print( '\nFixed the dty column, existing dtys are now (in units of microns):' ) print(np.unique(flt.dty)) print('') #--------------------------------------------------------------------------- grains = grain.read_grain_file(grain_file) grain_topology_masks = self.recon_topology(flt, grains) g, s = self.select_grains(grains, grain_topology_masks) recons = self.reconstructor.reconstruct( flt, g, self.number_y_scans,\ self.ymin, self.ystep, s ) print("Recon shape: ", recons['E11'].shape) print("\n Finished reconstruction of slice: " + label) print("Saving results..") self.slices[label] = recons self.slice_masks[label] = sum(grain_topology_masks) self.save_slice_as_npy(recons, label) return recons
def main(): colf = columnfile.columnfile(sys.argv[1]) print("Read", colf.nrows, "peaks") p = parameters.read_par_file(sys.argv[2]) # input a sample radius to use for computing ranges radius = int(sys.argv[3]) omegastep = guess_omega_step(colf) print("Guess omega step as", omegastep) colf.updateGeometry(p) for i, t in enumerate(colf.titles): print("\n", t, end=" ") for j in (0, 1, 2, colf.nrows // 2, colf.nrows // 2 + 1, colf.nrows // 2 + 2, colf.nrows - 3, colf.nrows - 2, colf.nrows - 1): print(colf.bigarray[i, j], end=" ") print() modXL = np.sqrt(colf.xl * colf.xl + colf.yl * colf.yl + colf.zl * colf.zl) modXLmean = modXL.mean() dangle = np.degrees(np.arctan2(radius, modXLmean)) print("Angle precision is about ", dangle, "for average distance", modXLmean, "and sample size ", radius) findpairs_2d(colf, dangle, omegastep)
def test3(self): c = columnfile.columnfile("test.col") self.assertEqual(list(c.a), [1, 6, 4, 5]) c.removerows("a", [1]) c.removerows("a", [4]) self.assertEqual(c.nrows, 2) self.assertEqual(list(c.b), [1, 6])
def merge_flts(p, flts, motor): ars = [] titles = None for f in flts: try: c = columnfile(f) except: print(f, "Empty file") continue # num = int(f.split("_")[5]) dty = get_dty(f, motor=motor) print(f, dty, c.nrows) try: c.addcolumn(ones(c.nrows) * dty, "dty") except: print(dty, c.nrows, type(dty), type(c.nrows)) print(ones(c.nrows) * dty) raise c.updateGeometry(p) if titles is None: titles = [t for t in c.titles] for told, tnew in zip(titles, c.titles): assert told == tnew, "titles do not match %s %s %s" % (f, told, tnew) ars.append(c.bigarray) c.bigarray = concatenate(ars, axis=1) c.nrows = c.bigarray.shape[1] c.set_attributes() return c
def read_peaks(self, peaksfile): """ Read in the peaks from a peaksearch """ start = time.time() colf = columnfile.columnfile(peaksfile) logging.info("reading file %f/s"%(time.time()-start)) # hmm - is this the right way around? self.rlgrid = 1.0*self.cell_size/self.npx self.px = colf.omega self.px = np.where(self.px > self.npx/2 , self.px - self.npx , self.px)*self.rlgrid self.py = colf.sc self.py = np.where(self.py > self.npx/2 , self.py - self.npx , self.py)*self.rlgrid self.pz = colf.fc self.pz = np.where(self.pz > self.npx/2 , self.pz - self.npx , self.pz)*self.rlgrid self.UBIALL = np.array( [self.px, self.py, self.pz] ).T logging.info("Number of peaks found %d %f/s, now fit some"%( self.px.shape[0],time.time()-start)) for i in range(colf.nrows): print(".",end="") self.UBIALL[i] = refine_vector( self.UBIALL[i], self.gv ) logging.info("Fitting vectors %f /s"%(time.time()-start)) self.colfile = colf
def testaddcol1(self): c = columnfile.columnfile("test.col") c.addcolumn([5, 4, 1, 2, 0], 'alice') self.assertEqual(c.titles[-1], 'alice') self.assertEqual(list(c.alice), [5, 4, 1, 2, 0]) self.assertEqual(c.ncols, 5) self.assertEqual(c.nrows, 5)
def setup(): try: c = columnfile( sys.argv[1] ) g = read_grain_file( sys.argv[2] ) parfile = sys.argv[3] cmds = [] except: print( "Usage: %s colfile.flt.new grains.map parameters.par --omega_slop=1 etc"%(sys.argv[0])) sys.exit() if platform.system() != "Windows": fmt = "%s %s" else: fmt = '%s "%s"' cmd0 = fmt%( sys.executable, os.path.join( os.path.split(__file__)[0], "fitgrain.py" ) ) for i in range(len(g)): #g[i].translation[2] = 0.0 write_grain_file("%d.ubi"%(i),[g[i]]) d = c.copy() d.filter( d.labels == i ) d.writefile("%d.flt"%(i)) cmd = cmd0 + " -p %s -u %d.ubi -U %d.ubi -P %d.par -f %d.flt -x t_z"%( parfile,i,i,i,i) for extra_arg in sys.argv[4:]: cmd += " "+extra_arg cmds.append( cmd ) return cmds
def testaddcol2(self): c = columnfile.columnfile("test.col") c.addcolumn([5, 4, 1, 2, 9], 'a') self.assertEqual(c.titles.index("a"), 1) self.assertEqual(list(c.a), [5, 4, 1, 2, 9]) self.assertEqual(c.ncols, 4) self.assertEqual(c.nrows, 5)
def test_xyz_from_yaml(self): for p in parfiles: parfile = os.path.join(TEST, p) fltfile = os.path.join(TEST, "test.flt") ymlfile = os.path.join( os.path.split(general_geometry.__file__)[0], "data", "fable.yml") pars = parameters.read_par_file(parfile).parameters colf = columnfile.columnfile(fltfile) sc = colf.sc[:4] fc = colf.fc[:4] geometry = general_geometry.from_yml( pars, ymlfile, path=["Positioners", "Fable_detector"], noisy=False) # test old xyz1 = transform.compute_xyz_lab((sc, fc), **pars) # test new v = np.zeros((3, len(sc))) v[1] = fc v[2] = sc xyz2 = geometry((np.zeros(len(fc)), fc, sc)) if not np.allclose(xyz1, xyz2): print("Geometry", geometry) print("Parfile:", p) for i in range(len(fc)): print(xyz1[:, i]) print(xyz2[:, i]) print() assert np.allclose(xyz1, xyz2)
def main(): import sys, time c = columnfile(sys.argv[1]) p = read_par_file(sys.argv[2]) u = unitcell_from_parameters(p) gl = read_grain_file(sys.argv[3]) if gl[0].translation is None: gl[0].translation = np.array((0., 0., 0.)) start = time.time() # Setup and assign hkls w = p.get("wavelength") peaks_Cxyz, beam_Cxyz = getCxyz(c, p) t = gl[0].translation.copy() ub = gl[0].ub.copy() ubi = gl[0].ubi.copy() gve = compute_Cgve(t, peaks_Cxyz, beam_Cxyz, w) hi = np.round(np.dot(ubi, gve)) lastgof = 1e9 ubn, tn = fit_ub_t(ub, t, hi, peaks_Cxyz, beam_Cxyz, w) print("Before\nt=", gl[0].translation) print("UB=", gl[0].ub) gl[0].set_ubi(np.linalg.inv(ubn)) gl[0].translation = tn dt = time.time() - start print("time calculating", dt, "gps", 1 / dt) print("After\nt=", gl[0].translation) print("UB=", gl[0].ub) write_grain_file(sys.argv[4], gl)
def process_layer(args, layer): ## Checks for existance of directory for saving files pdir = '%s/%02d' % (args.output, layer) if os.path.exists(pdir): if args.force: shutil.rmtree(pdir) else: raise IOError('%s already exists' % pdir) ## Creats directory for saving files os.mkdir(pdir) ## Defines first image and last image number in layer first_im = args.nstart + args.nimages * layer ### last_im = args.nimages + first_im - 1 ## Creats median file using fable 'median.py' ndigits = None for i in range(7): fmtstr = '%s%0' + str(i) + 'd%s' first_im_name = fmtstr % (args.stem, args.nstart, args.image_format) if os.path.exists(first_im_name): ndigits = i break if ndigits is None: raise RuntimeError('format string problem, revisit') ## defines directory path for raw data/images ## first image, last image, step size between images if args.image_format == '.ge2': medianskip = int(args.nimages / 240) args.medianstep = 1 else: medianskip = int(args.nimages / args.medianstep) command = ('median.py -i %s -f %i -l %i -s %i -o %s/median' % (first_im_name, first_im, medianskip, args.medianstep, pdir)) if args.verbose: print(command) os.system(command) ## Runs peaksearch.py on images using specified thresholds command = ('peaksearch.py -n %s -F %s -f %i -l %i -o %s/peaks ' '-d %s/median.edf -p Y --ndigits=%i --OmegaOverRide ' '-S %f -T %f' % (args.stem, args.image_format, first_im, last_im, pdir, pdir, ndigits, args.step_ome, args.start_ome)) ## Adds threshold values onto command for t in args.thresholds: command += ' -t %s' % t if args.verbose: print(command) os.system(command) ## removes any peak entries arising from spots containing less than N pixels for t in args.thresholds: num_lines = sum(1 for line in open('%s/peaks_t%i.flt' % (pdir, t))) if os.path.exists('%s/peaks_t%i.flt' % (pdir, t)) and num_lines > 1: d = ic.columnfile('%s/peaks_t%i.flt' % (pdir, t)) d.removerows('Number_of_pixels', list(range(args.pixels))) d.writefile('%s/peaks_min%d_t%i.flt' % (pdir, args.pixels, t))
def test3(self): c = columnfile.columnfile("test.col") assert list(c.a) == [1, 6, 4, 5, 7] assert list(c.b) == [2, 1, 2, 6, 9] c.removerows("a", [1]) c.removerows("a", [4]) assert c.nrows == 3 assert list(c.b) == [1, 6, 9]
def __init__(self, fname, parfile): self.colfile = columnfile(fname) self.pars = parameters.read_par_file(parfile) self.ds_tol = 0.005 self.bins = np.arange(0, 0.5, 0.005) print("Setting up") self.compute_XLYLZL() self.computegv()
def spot3d_id(filename): """ Example: merge.spot3d_id('peaks.flt') This makes unique spot3d_id in file peaks.flt Jette Oddershede, October 2008 """ flt = ic.columnfile(filename) flt.setcolumn(n.arange(flt.nrows), 'spot3d_id') flt.writefile(filename)
def testfilter(self): c = columnfile.columnfile("testgeom.flt") d = c.copy() d.filter(abs(d.sc - 22) > .1) self.assertTrue(d.nrows == 4) d.filter(abs(d.sc - 22) > .1) self.assertTrue(d.nrows == 4) self.assertTrue(c.nrows == 5) d = c.copy() self.assertTrue(d.nrows == 5)
def testpeaksearch_singlethread(self): os.system(self.CMD + " -n tiftest " + \ " -t 7 -F .tif -S 0.3 " + \ " -T 11 -p Y --singleThread " +\ " -f %d -l %d"%(self.FIRST, self.LAST)) results = columnfile.columnfile( "peaks_t7.flt" ) self.assertEqual( results.nrows, self.NPK) if os.path.exists( 'peaks_t7.flt' ): self.assertEqual( open( 'peaks_t7.flt').read().rstrip(), open( 'peaks_t6.flt').read().rstrip() )
def test_1(self): self.data1[20:41, 120:141] = 1. self.data2[20:41, 120:141] = 1. self.data3[20:41, 120:141] = 1. self.data4[20:41, 120:141] = 1. self.data5[20:41, 120:141] = 1. self.data3[30, 130] = 2. # in the middle self.data1[55, 65] = 1. self.data2[55, 65] = 2. self.data3[55, 65] = 3. self.data4[55, 65] = 2. self.data5[55, 65] = 1. lio = labelimage.labelimage(self.dims, self.outfile) lio.peaksearch(self.data1, 0.1, 1.) lio.mergelast() lio.peaksearch(self.data2, 0.1, 2.) lio.mergelast() lio.peaksearch(self.data3, 0.1, 3.) lio.mergelast() lio.peaksearch(self.data4, 0.1, 4.) lio.mergelast() lio.peaksearch(self.data5, 0.1, 5.) lio.mergelast() lio.finalise() lio.outfile.close() co = columnfile.columnfile(self.outfile) self.assertEqual(co.nrows, 2) # first peaks 20->41, 120->141 self.assertAlmostEqual(co.fc[0], 130., 6) self.assertAlmostEqual(co.sc[0], 30., 6) self.assertAlmostEqual(co.omega[0], 3.0, 6) self.assertAlmostEqual(co.Number_of_pixels[0], 21 * 21 * 5, 6) self.assertAlmostEqual(co.avg_intensity[0], 1, 1) self.assertAlmostEqual(co.IMax_int[0], 2, 6) self.assertAlmostEqual(co.IMax_f[0], 130., 6) self.assertAlmostEqual(co.IMax_s[0], 30.0, 6) self.assertAlmostEqual(co.IMax_o[0], 3.0, 6) self.assertAlmostEqual(co.fc[1], 65., 6) self.assertAlmostEqual(co.sc[1], 55., 6) self.assertAlmostEqual(co.omega[1], 3.0, 6) self.assertAlmostEqual(co.Number_of_pixels[1], 5, 6) self.assertAlmostEqual(co.avg_intensity[1], 1.8, 1) self.assertAlmostEqual(co.IMax_int[1], 3, 6) self.assertAlmostEqual(co.IMax_f[1], 65., 6) self.assertAlmostEqual(co.IMax_s[1], 55., 6) self.assertAlmostEqual(co.IMax_o[1], 3.0, 6) mask = co.Number_of_pixels > 10 co.filter(mask) self.assertEqual(co.nrows, 1) co.writefile("l1.out")
def loadfiltered(self, filename): """ Read in 3D peaks from peaksearch """ self.colfile = columnfile.columnfile(filename) if (self.colfile.titles[0:3] == ["sc", "fc", "omega"]): self.setxyomcols("sc", "fc", "omega") if (self.colfile.titles[0:3] == ["xc", "yc", "omega"]): self.setxyomcols("xc", "yc", "omega") if "spot3d_id" not in self.colfile.titles: self.colfile.addcolumn(range(self.colfile.nrows), "spot3d_id")
def __init__(self, fname, parfile): self.colfile = columnfile( fname ) self.pars = parameters.read_par_file( parfile ) self.ds_tol = 0.005 self.bins=np.arange(0,0.5,0.005) self.compute_XLYLZL() self.computegv() self.makecell() self.ringassign()
def runtest(start, step): """ check tif range """ pksh = sys.executable + ' ' + os.path.join("..","..","scripts","peaksearch.py") os.system(pksh + " -n test -F .tif -f 0 -l 9 -t 1.0 -D 0 " + "-T %f -S %f > testpksearch.log"%(start,step)) c = columnfile("peaks_t1.flt") print "start",start,"step", step,"finds:", print "Min_o",c.Min_o,"Max_o",c.Max_o,"omega",c.omega if len(c.Min_o) == 3: print "I think it might be OK" return len(c.Min_o)==3
def get( self, fname ): if fname not in self.cache: # integer dty positions for indexing arrays colfile = columnfile( fname ) iy = np.round( (colfile.dty - self.ymin)/self.ystep ).astype(int) colfile.addcolumn( iy, "iy") colfile.NY = self.NY # ensure tth, eta, gx, gy, gz are up-to-date if self.parfile is not None: colfile.parameters.loadparameters( self.parfile ) colfile.updateGeometry() self.cache[fname] = colfile return self.cache[fname]
def readallpeaks_flt(self, peaksfilename): """ read in flt peaks found with peaksearch (ImageD11) """ import os, fabio self.lines = open(peaksfilename, "r").readlines() try: from ImageD11 import columnfile except: return False cf = columnfile.columnfile(peaksfilename) self.images = cf return
def loadcolfile(cfile): """ fixme: move to ImageD11.colfile if not there already """ if cfile.find("::")>-1: h,s = cfile.split("::") try: colfile = colfile_from_hdf( h, name = s ) except: print "*",cfile,"*" print "Trying to open",h, os.path.exists(h) print s raise else: colfile = columnfile(cfile) return colfile
def test(start, step): import sys if sys.platform == "win32": pksh = "%s ..\\..\\scripts\\peaksearch.py " % (sys.executable) else: pksh = "peaksearch.py " os.system(pksh + "-n test -F .tif -f 0 -l 9 -t 1.0 -D 0 " + "-T %f -S %f >> testpksearch.log" % (start, step)) c = columnfile("peaks_t1.flt") print "start", start, "step", step, "finds:", print "Min_o", c.Min_o, "Max_o", c.Max_o, "omega", c.omega if len(c.Min_o) == 3: print "I think it might be OK"
def main2(): import sys c = columnfile(sys.argv[1]) p = read_par_file(sys.argv[2]) gl = read_grain_file(sys.argv[3]) for i, g in enumerate(gl): mask = c.labels == i g.sc = np.compress(mask, c.sc) g.fc = np.compress(mask, c.fc) g.omega = np.compress(mask, c.omega) ubnew, tnew = fitagrain(g, p) g.set_ubi(np.linalg.inv(ubnew)) g.translation[:] = tnew print(i, len(g.sc), tnew) write_grain_file(sys.argv[4], gl)
def loadColfile(self, filename=None): """ Read in a new file """ if filename is None or filename is False: filename = silx.gui.qt.QFileDialog(self.win,"Columnfile").getOpenFileName()[0] try: c = columnfile( filename ) self.colfile = c except: print("problem opening file",filename) return print("loaded file",filename) self.fnamelabel.setText( "Columnfile: " + self.colfile.filename ) self.setTitles() self.update() self.scatter_widget.resetZoom() self.scatter_widget.activateWindow() self.win.activateWindow()
def main(): flt = columnfile.columnfile( sys.argv[1] ) grains = grain.read_grain_file( sys.argv[2] ) pars = parameters.read_par_file( sys.argv[3] ) newgrainfile = sys.argv[4] hkltol = 0.05 # for first peak assignments nmedian = 5 # for removing peak fit outliers omegastep = 1.0 # for omega images ymin = 13.5 # dty start (so -15 -> +15 in 0.25 steps) ystep = 0.02 # step in dty from scan rcut = 0.2 # cutoff for segmentation of reconstruction flt.filter( flt.dty >= ymin ) flt.idty = np.round((flt.dty - ymin)/ystep).astype(np.int32) - 35 flt.NY = 71 # flt.idty.max()+1 OMSLOP = omegastep / 2.0 tth, eta, gve = update_cols( flt, pars, OMSLOP ) assign_peaks( grains, gve, flt, pars, nmedian, hkltol ) # pl.ioff() print("\n\n") out = open( newgrainfile, "w" ) out.write("# grain ix iy npks ubi00 ubi01 ubi02 ubi10 ubi11 ubi12 ubi20 ubi21 ubi22\n") for i,g in enumerate(grains): print("# Grain:",i) fit_one_grain( g, flt, pars ) y0,x,y = map_out_cell( g, flt ) sinoangles, sino, recon = map_grain( g, flt, ymin, ystep, omegastep ) if 0: pl.subplot(211) pl.imshow( sino ) pl.subplot(212) pl.imshow( recon ) pl.show() active = recon > recon.max() * rcut ii, jj = np.mgrid[ 0:recon.shape[0], 0:recon.shape[0] ] - recon.shape[0]//2 for ix, iy in zip(ii[active], jj[active]): gf = fit_one_point( g, flt, pars, ix, iy, ystep ) r = ("%-4d "*4)%(i,ix,iy,gf.mask.astype(int).sum()) print(r) u = ("%.7f "*9)%tuple(gf.ubi.ravel()) out.write(r) out.write(u+"\n") g.translation = (x,y,0)