def doindex(gve, x, y, z): global NUL global tmp global NPKS ss = sys.stdout # turns of printing #sys.stdout = NUL myindexer = indexing.indexer() myindexer.readgvfile(gve) for ring1 in [1, 0, 2]: for ring2 in [1, 0]: myindexer.parameterobj.set_parameters({ 'ds_tol': 0.004, 'minpks': NPKS, 'max_grains': 1000, 'hkl_tol': 0.02, 'ring_1': ring1, 'ring_2': ring2 }) myindexer.loadpars() myindexer.assigntorings() myindexer.find() myindexer.scorethem() grains = [grain.grain(ubi, [x, y, z]) for ubi in myindexer.ubis] grain.write_grain_file("%s.ubi" % (tmp), grains) sys.stdout = ss return len(grains)
def grid_index_parallel(fltfile, parfile, tmp, gridpars, translations): """ fltfile containing peaks parfile containing instrument geometry and unit cell tmp - base name for scratch files and results gridpars : dictionary of control parameters (rings to use, etc) translations : list of translation positions to try Runs a grid index algorithm using pythons multiprocessing module splits workload over processes (blocks of translations to each process) This thread should catch results via a queue """ gridpars = initgrid(fltfile, parfile, tmp, gridpars) print("Done init") if 'NPROC' not in gridpars or gridpars['NPROC'] is None: NPR = multiprocessing.cpu_count() - 1 cImageD11.cimaged11_omp_set_num_threads( 2) # assume hyperthreading is useful? else: NPR = int(gridpars['NPROC']) if 'NTHREAD' in gridpars: cImageD11.cimaged11_omp_set_num_threads(int(gridpars['NTHREAD'])) elif NPR > 1: cImageD11.cimaged11_omp_set_num_threads(1) tsplit = [translations[i::NPR] for i in range(NPR)] args = [("%s.flt" % (tmp), parfile, t, gridpars) for i, t in enumerate(tsplit)] q = PQueue() p = Pool(processes=NPR, initializer=wrap_test_many_points_init, initargs=[q]) print("Using a pool of", NPR, "processes") pa = p.map_async(wrap_test_many_points, args) ul = uniq_grain_list(gridpars['SYMMETRY'], gridpars['toldist'], gridpars['tolangle']) lastsave = 0 while True: # If we go more than 30 seconds without something, die try: grs = q.get(True, 10) gb4 = len(ul.uniqgrains) ul.add(grs) gnow = len(ul.uniqgrains) print("Got % 5d new %d from %d" % (gnow, gnow - gb4, len(grs))) if len(ul.uniqgrains) > lastsave: lastsave = len(ul.uniqgrains) grain.write_grain_file("all" + tmp + ".map", ul.uniqgrains) if pa._number_left == 0: break except Queue.Empty: sys.stderr.write(" Caught queue empty exception\n") if pa._number_left == 0: break except KeyboardInterrupt: break # write here to be on the safe side .... grain.write_grain_file("all" + tmp + ".map", ul.uniqgrains) p.close() p.join()
def doindex(gve, x, y, z, w): global NUL global tmp global NPKS global UC global TOLSEQ ss = sys.stdout # turns of printing sys.stdout = NUL myindexer = indexing.indexer(wavelength=w, unitcell=UC, gv=gve.T) myindexer.ds = np.sqrt((gve * gve).sum(axis=0)) myindexer.ga = np.zeros(len(myindexer.ds), int) - 1 # Grain assignments # myindexer.readgvfile( gve ) for ring1 in RING1: for ring2 in RING2: myindexer.parameterobj.set_parameters({ 'ds_tol': DSTOL, 'minpks': NPKS, 'max_grains': 1000, 'hkl_tol': TOLSEQ[0], 'ring_1': ring1, 'ring_2': ring2 }) myindexer.loadpars() myindexer.assigntorings() myindexer.find() myindexer.scorethem() grains = [grain.grain(ubi, [x, y, z]) for ubi in myindexer.ubis] grain.write_grain_file("%s.ubi" % (tmp), grains) sys.stdout = ss return len(grains)
def setup(): try: c = columnfile( sys.argv[1] ) g = read_grain_file( sys.argv[2] ) parfile = sys.argv[3] cmds = [] except: print( "Usage: %s colfile.flt.new grains.map parameters.par --omega_slop=1 etc"%(sys.argv[0])) sys.exit() if platform.system() != "Windows": fmt = "%s %s" else: fmt = '%s "%s"' cmd0 = fmt%( sys.executable, os.path.join( os.path.split(__file__)[0], "fitgrain.py" ) ) for i in range(len(g)): #g[i].translation[2] = 0.0 write_grain_file("%d.ubi"%(i),[g[i]]) d = c.copy() d.filter( d.labels == i ) d.writefile("%d.flt"%(i)) cmd = cmd0 + " -p %s -u %d.ubi -U %d.ubi -P %d.par -f %d.flt -x t_z"%( parfile,i,i,i,i) for extra_arg in sys.argv[4:]: cmd += " "+extra_arg cmds.append( cmd ) return cmds
def main(): import sys, time c = columnfile(sys.argv[1]) p = read_par_file(sys.argv[2]) u = unitcell_from_parameters(p) gl = read_grain_file(sys.argv[3]) if gl[0].translation is None: gl[0].translation = np.array((0., 0., 0.)) start = time.time() # Setup and assign hkls w = p.get("wavelength") peaks_Cxyz, beam_Cxyz = getCxyz(c, p) t = gl[0].translation.copy() ub = gl[0].ub.copy() ubi = gl[0].ubi.copy() gve = compute_Cgve(t, peaks_Cxyz, beam_Cxyz, w) hi = np.round(np.dot(ubi, gve)) lastgof = 1e9 ubn, tn = fit_ub_t(ub, t, hi, peaks_Cxyz, beam_Cxyz, w) print("Before\nt=", gl[0].translation) print("UB=", gl[0].ub) gl[0].set_ubi(np.linalg.inv(ubn)) gl[0].translation = tn dt = time.time() - start print("time calculating", dt, "gps", 1 / dt) print("After\nt=", gl[0].translation) print("UB=", gl[0].ub) write_grain_file(sys.argv[4], gl)
def main2(): import sys c = columnfile(sys.argv[1]) p = read_par_file(sys.argv[2]) gl = read_grain_file(sys.argv[3]) for i, g in enumerate(gl): mask = c.labels == i g.sc = np.compress(mask, c.sc) g.fc = np.compress(mask, c.fc) g.omega = np.compress(mask, c.omega) ubnew, tnew = fitagrain(g, p) g.set_ubi(np.linalg.inv(ubnew)) g.translation[:] = tnew print(i, len(g.sc), tnew) write_grain_file(sys.argv[4], gl)
def domap(OmFloat, OmSlop, pars, colfile, grainsfile, tolseq=[0.03, 0.02, 0.01], symmetry="triclinic"): """mapping function - does what makemap.py does""" global NPKS ss = sys.stdout # turns off printing def fit(tol): o = refinegrains.refinegrains( OmFloat=OmFloat, OmSlop=OmSlop, tolerance=tol, intensity_tth_range=(0, 180), ) o.parameterobj = pars # o.loadfiltered ... o.scannames = ["internal"] o.scantitles = colfile.titles o.scandata["internal"] = colfile o.tolerance = tol o.readubis(grainsfile) if symmetry != "triclinic": o.makeuniq(symmetry) o.generate_grains() o.refinepositions() return o for tol in tolseq: sys.stdout = NUL o = fit(tol) gl = filter(lambda x: x.npks > NPKS, o.grains.values()) sys.stdout = ss if len(gl) == 0: print("I killed all your grains!") break else: print("Keeping", len(gl), "from", len(o.grains.values()), "grains with at least", NPKS, "peaks", tol) grain.write_grain_file(grainsfile, gl) # re-assign after last filter fit(tol) return len(gl)
def main(): flt = columnfile.columnfile(sys.argv[1]) grains = grain.read_grain_file(sys.argv[2]) pars = parameters.read_par_file(sys.argv[3]) newgrainfile = sys.argv[4] hkltol = 0.05 # for first peak assignments nmedian = 5 # for removing peak fit outliers omegastep = 1.0 # for omega images ymin = -18 # dty start (so -15 -> +15 in 0.25 steps) ystep = 0.25 # step in dty from scan rcut = 0.2 # cutoff for segmentation of reconstruction OMSLOP = omegastep / 2.0 tth, eta, gve = update_cols(flt, pars, OMSLOP) assign_peaks(grains, gve, flt, pars, nmedian, hkltol) pl.ioff() print("\n\n") for i, g in enumerate(grains): print("# Grain:", i) fit_one_grain(g, flt, pars) y0, x, y = map_out_cell(g, flt) sinoangles, sino, recon = map_grain(g, flt, ymin, ystep, omegastep) if 0: pl.subplot(211) pl.imshow(sino) pl.subplot(212) pl.imshow(recon) pl.show() active = recon > recon.max() * rcut ii, jj = np.mgrid[0:recon.shape[0], 0:recon.shape[0]] - recon.shape[0] // 2 for ix, iy in zip(ii[active], jj[active]): gf = fit_one_point(g, flt, pars, ix, iy, ystep) print(("%-4d " * 4) % (i, ix, iy, gf.mask.astype(int).sum()), end=" ") print(("%.6f " * 6) % (indexing.ubitocellpars(gf.ubi)), end=" ") print(("%.6f " * 3) % tuple(gf.Rod)) g.translation = (x, y, 0) grain.write_grain_file(newgrainfile, grains)
def savegrains(self, filename, sort_npks=True): """ Save the refined grains """ ks = list(self.grains.keys()) # sort by number of peaks indexed to write out if sort_npks: # npks in x array order = numpy.argsort([self.grains[k].npks for k in ks]) ks = [ks[i] for i in order[::-1]] else: ks.sort() gl = [(self.grains[k], k) for k in ks] # Update the datafile and grain names reflect indices in grain list for g, k in gl: name, fltname = g.name.split(":") assert fltname in self.scandata, "Sorry - logical flaw" assert len(list(self.scandata.keys()) ) == 1, "Sorry - need to fix for multi data" self.set_translation(k[0], fltname) self.compute_gv(g, update_columns=True) numpy.put(self.scandata[fltname].gx, g.ind, self.gv[:, 0]) numpy.put(self.scandata[fltname].gy, g.ind, self.gv[:, 1]) numpy.put(self.scandata[fltname].gz, g.ind, self.gv[:, 2]) hkl_real = numpy.dot(g.ubi, self.gv.T) numpy.put(self.scandata[fltname].hr, g.ind, hkl_real[0, :]) numpy.put(self.scandata[fltname].kr, g.ind, hkl_real[1, :]) numpy.put(self.scandata[fltname].lr, g.ind, hkl_real[2, :]) hkl = numpy.floor(hkl_real + 0.5) numpy.put(self.scandata[fltname].h, g.ind, hkl[0, :]) numpy.put(self.scandata[fltname].k, g.ind, hkl[1, :]) numpy.put(self.scandata[fltname].l, g.ind, hkl[2, :]) # Count "uniq" reflections... sign_eta = numpy.sign(self.scandata[fltname].eta_per_grain[g.ind]) uniq_list = [(int(h), int(k), int(l), int(s)) for (h, k, l), s in zip(hkl.T, sign_eta)] g.nuniq = len(set(uniq_list)) grain.write_grain_file(filename, [g[0] for g in gl])
def savegrains(self, filename, sort_npks=True): """ Save the refined grains """ ks = self.grains.keys() # sort by number of peaks indexed to write out if sort_npks: # npks in x array gl = [(self.grains[k].npks, self.grains[k], k) for k in ks] gl.sort() gl = [(g[1], g[2]) for g in gl[::-1]] else: ks.sort() gl = [(self.grains[k], k) for k in ks] grain.write_grain_file(filename, [g[0] for g in gl]) # Update the datafile and grain names reflect indices in grain list for g, k in gl: name, fltname = g.name.split(":") assert self.scandata.has_key(fltname), "Sorry - logical flaw" assert len(self.scandata.keys() ) == 1, "Sorry - need to fix for multi data" self.set_translation(k[0], fltname) self.compute_gv(g, update_columns=True) numpy.put(self.scandata[fltname].gx, g.ind, self.gv[:, 0]) numpy.put(self.scandata[fltname].gy, g.ind, self.gv[:, 1]) numpy.put(self.scandata[fltname].gz, g.ind, self.gv[:, 2]) hkl_real = numpy.dot(g.ubi, self.gv.T) numpy.put(self.scandata[fltname].hr, g.ind, hkl_real[0, :]) numpy.put(self.scandata[fltname].kr, g.ind, hkl_real[1, :]) numpy.put(self.scandata[fltname].lr, g.ind, hkl_real[2, :]) hkl = numpy.floor(hkl_real + 0.5) numpy.put(self.scandata[fltname].h, g.ind, hkl[0, :]) numpy.put(self.scandata[fltname].k, g.ind, hkl[1, :]) numpy.put(self.scandata[fltname].l, g.ind, hkl[2, :])
from ImageD11.grain import read_grain_file, write_grain_file import sys, numpy as np gl = read_grain_file(sys.argv[1]) s = 10. for g in gl: g.translation += (np.random.random(3) - 0.5) * s g.ubi += (np.random.random((3, 3)) - 0.5) * 0.002 write_grain_file(sys.argv[2], gl)
] * 3 else: tols = [0.05, 0.01, 0.0075] for gref in gl: print("BEFORE: ", 3 * "% 7.3f " % tuple(gref.translation), 6 * "%.6f " % (ubitocellpars(gref.ubi))) for ii, tol in enumerate(tols): #print gref.translation gref = o.refine(gref, tol=tol) #print i,gref.npks # gref.pks = None # re-assign after convergence # gref = o.refine( gref, tol=0.0075) gfl.append(gref) ng += 1 print(ng, ng * 100.0 / len(gl), "%.2f " % (time.time() - start), gref.npks, end=' ') print((3 * "%.4f ") % tuple(gref.translation), end=' ') print((6 * "%.6f ") % ubitocellpars(gref.ubi)) print(time.time() - start) grain.write_grain_file(sys.argv[4], gfl) # 1/0 # for i in range(2): # o.find_triplets( i*17 ) # pylab.show() # ~/ImageD11/test/simul_1000_grains % python ../../sandbox/fittrans.py Al1000/Al1000.flt Al1000/Al1000.par allgrid.map allgridfittrans.map
#!/usr/bin/env python from __future__ import print_function """ Removes grains from a grain file if they have less than npks peaks Usage : cutgrains.py ubi_in ubi_out npks """ from ImageD11.grain import read_grain_file, write_grain_file import sys try: GRAINS = read_grain_file(sys.argv[1]) NPKS = int(sys.argv[3]) KEEP = [] for g in GRAINS: if int(g.npks) >= NPKS: KEEP.append(g) write_grain_file( sys.argv[2], KEEP) except: print(__doc__) raise
def check(gl1, gl2): dTs = 0 dVs = 0 mxT = 0 mxTi = 0 mxV = 0 mxVi = 0 for i, (g1, g2) in enumerate(zip( gl1, gl2)): dT = g1.translation - g2.translation dvol = np.linalg.det(g1.ubi) - np.linalg.det(g2.ubi) t = abs(dT).max() dTs += t dVs += abs(dvol) if t > mxT: mxTi, mxT = i, t if abs(dvol) > mxV: mxVi, mxV = i, abs(dvol) print dTs/i, dVs/i, mxT, mxTi, mxV, mxVi print "fname avg_pos_err avg_vol_err max_pos_err grain_max_pos_err max_vol_err grain_max_vol_err" for f in fnames: print f, check(read_grain_file(f), ideal) gbad = read_grain_file("shaken.map")[159] write_grain_file("gbad.ubi",[gbad,])
#!/usr/bin/python from ImageD11.columnfile import columnfile from ImageD11.grain import read_grain_file, write_grain_file import sys, os c = columnfile( sys.argv[1] ) g = read_grain_file( sys.argv[2] ) for i in range(len(g)): #g[i].translation[2] = 0.0 write_grain_file("%d.ubi"%(i),[g[i]]) d = c.copy() d.filter( d.labels == i ) d.writefile("%d.flt"%(i)) os.system( "fitgrain.py -p %s -u %d.ubi -U %d.ubi -P %d.par -f %d.flt -x t_z"%(sys.argv[3],i,i,i,i)) # for j in range(3): # os.system( "fitgrain.py -u %d.ubi -U %d.ubi -p %d.par -P %d.par -f %d.flt"%(i,i,i,i,i))
def fitallgrains( gfile, pfile, cfile, ngrains = None): colfile = loadcolfile( cfile ) grains = read_grain_file( gfile ) pars = read_par_file( pfile ) variables = [ 't_x','t_y', 't_z', 'y_center', 'tilt_y', 'tilt_z', 'tilt_x', 'distance', 'wedge'] pfitted = [] grs = [] cfs = [] if ngrains is None: ng = len(grains) else: ng = ngrains for i in range(ng): print "***",i, gr = grains[i] cf = assignpeaks( gr, pars, colfile, tol = 0.02 ) cfs.append(cf) grs.append(gr) pi = parameters( **pars.parameters ) refpars = fitmanygrains( cfs, grs, pi, variables ) for i in range(3): refpars = fitmanygrains( cfs, grs, refpars, variables ) if 0: pi = parameters( **pars.parameters ) pi.set('t_x', gr.translation[0]) pi.set('t_y', gr.translation[1]) pi.set('t_z', gr.translation[2]) diff, Ddiff = fitgrainfunc( cf, gr, pi, variables ) print "%.5g"%((diff*diff).ravel().sum()), gr, pfit = fitgrain( cf, gr, pi, variables, quiet=True ) grains[i] = gr pfitted.append( pfit ) diff, Ddiff = fitgrainfunc( cf, gr, pfit, variables ) print "%.5g"%((diff*diff).ravel().sum()) if 0: v = Ddiff.keys() for v in ['y_center', 'distance']: pylab.figure(1) pylab.title("Versus omega") pylab.plot( cf.omega, project_diff_on_variable( diff, v, Ddiff) , ",", label=v) pylab.figure(2) pylab.title("Versus fc") pylab.plot( cf.fc, project_diff_on_variable( diff, v, Ddiff) , ",", label=v) pylab.figure(3) pylab.title("Versus sc") pylab.plot( cf.sc, project_diff_on_variable( diff, v, Ddiff) , ",", label=v) pylab.figure(4) pylab.title("Versus sigo") pylab.plot( cf.sigo, project_diff_on_variable( diff, v, Ddiff) , ",", label=v) pylab.legend() pylab.show() raw_input() write_grain_file( gfile+".fit", grains)
break if TESTING: print("translation", t) print("Cell", indexing.ubitocellpars(np.linalg.inv(UB))) print("Translation %.5f %.5f %.5f" % tuple(t)) print("Cell %.7f %.7f %.7f %.8f %.8f %.8f" % (indexing.ubitocellpars(np.linalg.inv(UB)))) g.translation = t g.set_ubi(np.linalg.inv(UB)) return grains # TODO: # # DONE : Least squares as SVD problem # - Allow a constraint matrix to be used # - Compute weights from experimental data (sig/cov etc) # DONE (but not weighted or printed) : Compute error estimates # - Allow fitting of other geometrical parameters # - Allow refinement in terms of other parameterisation of UB (e.g. U, B) # - Documentation if __name__ == "__main__": colfile, parfile, grainsfile, newgrainsfile = sys.argv[1:5] c = columnfile.columnfile(colfile) p = parameters.read_par_file(parfile) g = grain.read_grain_file(grainsfile) grain.write_grain_file(newgrainsfile, refit_makemap(c, p, g)) # python teo.py ../test/simul_1000_grains/g187.flt ../test/simul_1000_grains/Al1000/Al1000.par ../test/simul_1000_grains/g187.map teo187fitted.map
matching = diffs < tol ubi = ubis[matching].mean(axis=0) remaining = ubis[~matching] return ubi, remaining, tol ubiall = ubis.copy() tol = None uniqs = [] while len(ubis > 0): uniq, ubis, tol = removeone(ubis, ubis[0], tol) print(uniq, len(ubis)) uniqs.append(uniq) # now check if any uniqs have a collision due to symmetry: uniqs = np.array(uniqs) for i in range(len(uniqs)): for operator in grp.group[1:]: # skip the identity symubi = np.dot(operator, uniqs[i].reshape(3, 3)).ravel() scors = abs(uniqs - symubi).sum(axis=1) found = scors < tol if found.sum() > 0: print("Symmetry collision!") print(i, operator.ravel(), scors[found], np.arange(len(uniqs))[found]) grain.write_grain_file( uniqfile, [grain.grain(ubi.reshape(3, 3), (0, 0, 0)) for ubi in uniqs])
def filtergrain(options): """ Filter a peaks file according to which peaks are indexed """ o = refinegrains.refinegrains(tolerance=0.9, OmFloat=options.omega_float, OmSlop=options.omega_slop) o.loadparameters(options.parfile) o.readubis(options.ubifile) if options.grain is None: if len(o.grainnames) == 1: gn = o.grainnames[0] else: for i, gn in zip(list(range(len(o.grainnames))), o.grainnames): logging.info("Choose %d for grain %s" % (i, gn)) gn = o.grainnames[int(input("select which grain "))] else: gn = o.grainnames[int(options.grain)] o.grainnames = [ gn, ] o.loadfiltered(options.fltfile) o.generate_grains() o.assignlabels() o.set_translation(gn, options.fltfile) o.compute_gv(o.grains[(gn, options.fltfile)]) if options.tol is None: for tol in [0.01, 0.025, 0.05, 0.1, 0.15, 0.25, 0.5]: o.tolerance = tol logging.info("tol %f" % (o.tolerance)) o.refineubis(quiet=False, scoreonly=True) o.tolerance = float(input("Enter tolerance ")) options.tol = o.tolerance else: o.tolerance = options.tol matrix = o.grains[(gn, options.fltfile)].ubi o.assignlabels() drlv2 = indexing.calc_drlv2(matrix, o.gv) logging.info("Total peaks before filtering %d" % o.scandata[options.fltfile].nrows) gotpks = o.scandata[options.fltfile].copy() gotpks.filter(gotpks.labels == gn) gotpks.writefile(options.newfltfile) logging.info("Peaks which were indexed %d written to %s" % (gotpks.nrows, options.newfltfile)) # don't bother to copy here as we can overwrite if options.notindexed is not None: notpks = o.scandata[options.fltfile].copy() notpks.addcolumn(o.tth, "tth") notpks.addcolumn(o.eta, "eta") notpks.addcolumn(o.gv[:, 0], "gx") notpks.addcolumn(o.gv[:, 1], "gy") notpks.addcolumn(o.gv[:, 2], "gz") notpks.filter(drlv2 > o.tolerance * o.tolerance) notpks.writefile(options.notindexed) logging.info("Peaks which were not indexed %d written to %s" % (notpks.nrows, options.notindexed)) if options.newubifile is not None: o.scandata[options.fltfile] = gotpks # matrix = o.refine(matrix,quiet=True) grain.write_grain_file(options.newubifile, [o.grains[(gn, options.fltfile)]]) logging.info("Refined ubi in %s " % (options.newubifile))