def check(runpath='./',outpath='./',itmin=0,itmax=None,kTthresh=0.17):
"""
Name: clean
Author: Kari A. Frank
Date: November 23, 2015
Purpose: Make and display some diagnostic plots to check
ongoing xmc runs. Plots chi2, norm image, spectrum, and
histograms.
Usage:
import xmcinter.diagnostics as xd
xd.check(runpath='./',itmin=0,itmax=None)
Input:
runpath: string of path to the deconvolution files
outpath: string of path to store output files
itmin/itmax: minimum iteration to use in check
kTthresh: float to optionally specify a minimum kT. if set,
then all blobs with kT<kTthresh will not be included
in the map (but will be included in all other plots)
Output:
- Displays plots.
- Returns DataFrame of blob parameters
Usage Notes:
Example:
"""
# -- import modules --
import os
from file_utilities import ls_to_list
import xmcmap as xm
if kTthresh is None:
kTthresh = 0.0
# -- plot chi2 --
print "\nPlotting chi2 ...\n"
sf = xplt.chi2(runpath,itmax=itmax,
outfile=outpath+'/chi2_vs_iteration.html')
# -- calculate median chi2 --
if itmax is None:
itmax = np.max(sf.iteration)
sf = xw.filterblobs(sf,'iteration',minvals=itmin,maxvals=itmax)
medchi2 = xw.weighted_median(sf['redchi2'])
# -- read deconvolution files --
print "\nReading deconvolution files ...\n"
dfall=merge_output(runpath,save=False)
# -- add derivative columns --
dfall = clean(dfall,itmin=itmin,itmax=itmax)
print '\nIterations '+str(itmin)+' - '+str(itmax)+' : '
print "Total Number of Blobs = ",len(dfall.index)
print 'Median chi2/dof = '+str(medchi2)+'\n'
# -- plot model and data spectra --
print "\nPlotting spectrum ...\n"
smin = itmin/100
if itmax is None:
smax = None
else:
smax = itmax/100
sfig = xplt.spectrum(runpath=runpath,smin=smin,smax=smax,bins=0.03,
ylog=True,xlog=True,
outfile=outpath+'/spectrum.html',
lines=True,nlines=100,energy_range=(0.5,10.0))
# -- make median traceplots --
print "\nPlotting traces ...\n"
efig = xplt.trace(dfall,weights=None,
outfile=outpath+'/trace_plots.html')
# -- make histograms --
print "\nPlotting posteriors ...\n"
nbins = 75
w = 500
h = 200
hfigs = xplt.histogram_grid([dfall,dfall],weights=[None,'blob_em'],
bins=nbins,ncols=2,norm=True,
outfile=outpath+'/histogram_grid.html',
legends=['Unweighted','EM weighted'],
width=w,height=h,iterations='iteration')
print "\nPlotting posteriors with kT threshold ...\n"
hfigs = xplt.histogram_grid([xw.filterblobs(dfall,'blob_kT',
minvals=kTthresh),
xw.filterblobs(dfall,'blob_kT',
minvals=kTthresh)],
weights=[None,'blob_em'],
bins=nbins,ncols=2,norm=True,
outfile=outpath+'/histogram_grid_kTthresh.html',
legends=['Unweighted','EM weighted'],
width=w,height=h,iterations='iteration')
# -- scatter plots--
print "\nPlotting scatter plots ...\n"
blobcols = [c for c in dfall.columns if 'blob' in c]
sfigs2 = xplt.scatter_grid(dfall[blobcols],agg=None,sampling=2000)
# -- make norm map from most recent iteration --
print "\nMaking blob em map ...\n"
pixelsize = (dfall.phi.max()-dfall.phi.min())/50.0
# img1file = (outpath+'/bin'+str(int(pixelsize))+
# '_iter'+str(itmin)+'-'+str(itmax))
img1file = (outpath+'/bin'+str(int(pixelsize))+
'_iter'+str(itmax))
img = xm.make_map(xw.filterblobs(dfall,['blob_kT'],
minvals=[kTthresh,itmax],
maxvals=[None,itmax]),
paramname='blob_em',
paramweights=None,iteration_type='total',
binsize=pixelsize,nlayers=1,
withsignificance=True,nproc=2,
outfile=img1file,clobber=True)
return (dfall,sf)
def iteration_image(data,params,weights,nbins_x,nbins_y,binsize,xmin,ymin,
iteration_type,shape,blobx,bloby,blobsize,use_ctypes,
fast=True,
n_int_steps=1000):
"""Function to combine blobs from single iteration into 1 image."""
from wrangle import weighted_median,gaussian_volume
#--initialize stack of 2D images, one for each parameter--
iterimages = np.zeros((nbins_x,nbins_y,len(params)))
#----Calculate blob volumes in correct units (usually arcsec^3)----
if shape == 'gauss':
volumes = gaussian_volume(data[blobsize])
elif shape == 'sphere':
volumes = (4.0/3.0)*np.pi*data[blobsize]**3.0
else: # points
volumes = (0.1*binsize)**3.0 # set to much smaller than pixel
#--loop over image--
for x in xrange(nbins_x):
#get x integral
lowerx = int(xmin + x*binsize)
upperx = int(xmin + x*binsize + binsize)
if shape == 'gauss' or shape == 'points':
if fast is False:
# only use fast=False if scipy.integrate is
# not available
x_blob_integrals = gaussian_integral(lowerx,upperx,
n_int_steps,
data[blobx],
data[blobsize])
else:
x_blob_integrals = data.apply(lambda d: \
gaussian_integral_quad(lowerx,\
upperx,d[blobx],d[blobsize],\
use_ctypes=use_ctypes),\
axis=1)
elif shape == 'sphere':
print "ERROR: spherical_integral() not yet implemented"
x_blob_integrals = spherical_integral(lowerx,upperx,\
n_int_steps,\
data[blobx],
data[blobsize])
for y in xrange(nbins_y):
#get y integral
lowery = int(ymin + y*binsize)
uppery = int(ymin + y*binsize + binsize)
if shape == 'gauss' or shape == 'points':
if fast is False:
y_blob_integrals = gaussian_integral(lowery,uppery,\
n_int_steps,\
data[bloby],
data[blobsize])
else:
y_blob_integrals = data.apply(lambda d: \
gaussian_integral_quad(lowery,\
uppery,d[bloby],d[blobsize],\
use_ctypes=use_ctypes),\
axis=1)
elif shape == 'sphere':
y_blob_integrals = spherical_integral(lowery,uppery,\
n_int_steps,\
data[bloby],
data[blobsize])
#calculate fraction of blob volume in this pixel
if shape != 'points':
# !! for now this assumes gaussian volume !!
fractions = (x_blob_integrals*y_blob_integrals*
(2.0*np.pi*data[blobsize]**2.0)**.5 / volumes)
#times dz integral to get total volume in pixel,
#then divided by total volume
else:
# for now, points is implemented by setting the volumes
# to be much smaller than a pixel size
fractions = (x_blob_integrals*y_blob_integrals*
(2.0*np.pi*data[blobsize]**2.0)**.5 /
volumes)
# print "points is not yet implemented"
# if assuming points, then fraction=1 or 0
# fractions = point_integral(lowerx,upperx,lowery,uppery,
# data['x'],data['y'])
#-combine blobs in this pixel (loop over parameters)-
for p in xrange(len(params)):
if weights[p] is None: # default is equal weights
w = pd.Series(np.ones_like(data[params[p]]),
index=data[params[p]].index)
# elif weights[p] == 'densityspecial':
# for plotting density from EM - assumes the column passed
# was sqrt(EM*Volume/1.21), so weights=1/Vblobpix
# w = 1.0/(fractions*data['blob_volume'])
else:
w = data[weights[p]]
if iteration_type[p] == 'median':
iterimages[x,y,p]=weighted_median(data[params[p]],
weights=w*fractions)
elif iteration_type[p] == 'average':
iterimages[x,y,p]=np.average(data[params[p]],
weights=w*fractions)
elif iteration_type[p] == 'total':
iterimages[x,y,p]=np.sum(data[params[p]]*w*fractions)
elif iteration_type[p] == 'max':
iterimages[x,y,p]=np.max(data[params[p]]*w*fractions)
else:
print "ERROR: unrecognized iteration_type"
return iterimages
