def check(runpath='./',outpath='./',itmin=0,itmax=None,kTthresh=0.17): """ Name: clean Author: Kari A. Frank Date: November 23, 2015 Purpose: Make and display some diagnostic plots to check ongoing xmc runs. Plots chi2, norm image, spectrum, and histograms. Usage: import xmcinter.diagnostics as xd xd.check(runpath='./',itmin=0,itmax=None) Input: runpath: string of path to the deconvolution files outpath: string of path to store output files itmin/itmax: minimum iteration to use in check kTthresh: float to optionally specify a minimum kT. if set, then all blobs with kT<kTthresh will not be included in the map (but will be included in all other plots) Output: - Displays plots. - Returns DataFrame of blob parameters Usage Notes: Example: """ # -- import modules -- import os from file_utilities import ls_to_list import xmcmap as xm if kTthresh is None: kTthresh = 0.0 # -- plot chi2 -- print "\nPlotting chi2 ...\n" sf = xplt.chi2(runpath,itmax=itmax, outfile=outpath+'/chi2_vs_iteration.html') # -- calculate median chi2 -- if itmax is None: itmax = np.max(sf.iteration) sf = xw.filterblobs(sf,'iteration',minvals=itmin,maxvals=itmax) medchi2 = xw.weighted_median(sf['redchi2']) # -- read deconvolution files -- print "\nReading deconvolution files ...\n" dfall=merge_output(runpath,save=False) # -- add derivative columns -- dfall = clean(dfall,itmin=itmin,itmax=itmax) print '\nIterations '+str(itmin)+' - '+str(itmax)+' : ' print "Total Number of Blobs = ",len(dfall.index) print 'Median chi2/dof = '+str(medchi2)+'\n' # -- plot model and data spectra -- print "\nPlotting spectrum ...\n" smin = itmin/100 if itmax is None: smax = None else: smax = itmax/100 sfig = xplt.spectrum(runpath=runpath,smin=smin,smax=smax,bins=0.03, ylog=True,xlog=True, outfile=outpath+'/spectrum.html', lines=True,nlines=100,energy_range=(0.5,10.0)) # -- make median traceplots -- print "\nPlotting traces ...\n" efig = xplt.trace(dfall,weights=None, outfile=outpath+'/trace_plots.html') # -- make histograms -- print "\nPlotting posteriors ...\n" nbins = 75 w = 500 h = 200 hfigs = xplt.histogram_grid([dfall,dfall],weights=[None,'blob_em'], bins=nbins,ncols=2,norm=True, outfile=outpath+'/histogram_grid.html', legends=['Unweighted','EM weighted'], width=w,height=h,iterations='iteration') print "\nPlotting posteriors with kT threshold ...\n" hfigs = xplt.histogram_grid([xw.filterblobs(dfall,'blob_kT', minvals=kTthresh), xw.filterblobs(dfall,'blob_kT', minvals=kTthresh)], weights=[None,'blob_em'], bins=nbins,ncols=2,norm=True, outfile=outpath+'/histogram_grid_kTthresh.html', legends=['Unweighted','EM weighted'], width=w,height=h,iterations='iteration') # -- scatter plots-- print "\nPlotting scatter plots ...\n" blobcols = [c for c in dfall.columns if 'blob' in c] sfigs2 = xplt.scatter_grid(dfall[blobcols],agg=None,sampling=2000) # -- make norm map from most recent iteration -- print "\nMaking blob em map ...\n" pixelsize = (dfall.phi.max()-dfall.phi.min())/50.0 # img1file = (outpath+'/bin'+str(int(pixelsize))+ # '_iter'+str(itmin)+'-'+str(itmax)) img1file = (outpath+'/bin'+str(int(pixelsize))+ '_iter'+str(itmax)) img = xm.make_map(xw.filterblobs(dfall,['blob_kT'], minvals=[kTthresh,itmax], maxvals=[None,itmax]), paramname='blob_em', paramweights=None,iteration_type='total', binsize=pixelsize,nlayers=1, withsignificance=True,nproc=2, outfile=img1file,clobber=True) return (dfall,sf)
def iteration_image(data,params,weights,nbins_x,nbins_y,binsize,xmin,ymin, iteration_type,shape,blobx,bloby,blobsize,use_ctypes, fast=True, n_int_steps=1000): """Function to combine blobs from single iteration into 1 image.""" from wrangle import weighted_median,gaussian_volume #--initialize stack of 2D images, one for each parameter-- iterimages = np.zeros((nbins_x,nbins_y,len(params))) #----Calculate blob volumes in correct units (usually arcsec^3)---- if shape == 'gauss': volumes = gaussian_volume(data[blobsize]) elif shape == 'sphere': volumes = (4.0/3.0)*np.pi*data[blobsize]**3.0 else: # points volumes = (0.1*binsize)**3.0 # set to much smaller than pixel #--loop over image-- for x in xrange(nbins_x): #get x integral lowerx = int(xmin + x*binsize) upperx = int(xmin + x*binsize + binsize) if shape == 'gauss' or shape == 'points': if fast is False: # only use fast=False if scipy.integrate is # not available x_blob_integrals = gaussian_integral(lowerx,upperx, n_int_steps, data[blobx], data[blobsize]) else: x_blob_integrals = data.apply(lambda d: \ gaussian_integral_quad(lowerx,\ upperx,d[blobx],d[blobsize],\ use_ctypes=use_ctypes),\ axis=1) elif shape == 'sphere': print "ERROR: spherical_integral() not yet implemented" x_blob_integrals = spherical_integral(lowerx,upperx,\ n_int_steps,\ data[blobx], data[blobsize]) for y in xrange(nbins_y): #get y integral lowery = int(ymin + y*binsize) uppery = int(ymin + y*binsize + binsize) if shape == 'gauss' or shape == 'points': if fast is False: y_blob_integrals = gaussian_integral(lowery,uppery,\ n_int_steps,\ data[bloby], data[blobsize]) else: y_blob_integrals = data.apply(lambda d: \ gaussian_integral_quad(lowery,\ uppery,d[bloby],d[blobsize],\ use_ctypes=use_ctypes),\ axis=1) elif shape == 'sphere': y_blob_integrals = spherical_integral(lowery,uppery,\ n_int_steps,\ data[bloby], data[blobsize]) #calculate fraction of blob volume in this pixel if shape != 'points': # !! for now this assumes gaussian volume !! fractions = (x_blob_integrals*y_blob_integrals* (2.0*np.pi*data[blobsize]**2.0)**.5 / volumes) #times dz integral to get total volume in pixel, #then divided by total volume else: # for now, points is implemented by setting the volumes # to be much smaller than a pixel size fractions = (x_blob_integrals*y_blob_integrals* (2.0*np.pi*data[blobsize]**2.0)**.5 / volumes) # print "points is not yet implemented" # if assuming points, then fraction=1 or 0 # fractions = point_integral(lowerx,upperx,lowery,uppery, # data['x'],data['y']) #-combine blobs in this pixel (loop over parameters)- for p in xrange(len(params)): if weights[p] is None: # default is equal weights w = pd.Series(np.ones_like(data[params[p]]), index=data[params[p]].index) # elif weights[p] == 'densityspecial': # for plotting density from EM - assumes the column passed # was sqrt(EM*Volume/1.21), so weights=1/Vblobpix # w = 1.0/(fractions*data['blob_volume']) else: w = data[weights[p]] if iteration_type[p] == 'median': iterimages[x,y,p]=weighted_median(data[params[p]], weights=w*fractions) elif iteration_type[p] == 'average': iterimages[x,y,p]=np.average(data[params[p]], weights=w*fractions) elif iteration_type[p] == 'total': iterimages[x,y,p]=np.sum(data[params[p]]*w*fractions) elif iteration_type[p] == 'max': iterimages[x,y,p]=np.max(data[params[p]]*w*fractions) else: print "ERROR: unrecognized iteration_type" return iterimages