def poor_loc(self):
""" Poorly localized sources
%(poorly_localized_table_check)s
"""
if len(self.poorloc) > 0:
self.poorly_localized_table_check = html_table(self.poorloc,
name=self.plotfolder+'/poorly_localized',
heading='<h4>Table of %d poorly localized (a>%.2f deg, or qual>%.1f with TS>%d) sources</h4>'\
% ( len(self.poorloc),self.acut,self.qualcut, self.tscut),
float_format=FloatFormat(2))
#poorly_localized_tablepath = os.path.join(self.plotfolder,'poorly_localized_table.html')
#open('poorly_localized_table.html','w').write(tohtml)
#print 'Wrote poorly_localized_table.html'
#open(os.path.join(poorly_localized_tablepath),'w').write(
# '<head>\n' + _html.style + '</head>\n<body>\n<h3>Poorly Localized Source Table</h3>'\
# + tohtml+'\n</body>')
#print 'saved html doc to %s' % os.path.join(poorly_localized_tablepath)
#self.poorly_localized_table_check =\
# '<p><a href="%s?skipDecoration"> Table of %d poorly localized (a>%.2f deg, or qual>%.1f with TS>%d) sources</a>'\
# % ( 'poorly_localized_table.html',len(self.poorloc),self.acut,self.qualcut, self.tscut)
try:
version = os.path.split(os.getcwd())[-1]
pv = makepivot.MakeCollection('poor localizations %s' %
version,
'tsmap_fail',
'poorly_localized.csv',
refresh=True)
self.poorly_localized_table_check +=\
'<br>A <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">pivot collection </a>of TS maps for these sources can be examined.'%pv.cId
except Exception, msg:
self.poorly_localized_table_check += '<br>(No pivot table: %s)' % msg
print '**** Failed to create pivot: %s' % msg
def pulsar_candidates(self, mints=16):
"""Pulsar candidate selection
Only choose those that are NOT in the gtlike list
%(pulsar_pivot_info)s
"""
df = self.dfx
cut = cut = (df.no_gtlike) & (np.abs(df.glat) > 5) & (df.ts > mints)
sedrec = self.dfx['sedrec']
tflow = []
tfmed = []
for sr in sedrec:
ts = sum(sr.ts)
low, middle, high = [
sum(sr.ts[i:i + 4] / ts).round(2) for i in (0, 4, 8)
]
tflow.append(low)
tfmed.append(middle)
self.dfx['tflow'] = tflow
self.dfx['tfmed'] = tfmed
pcand = df[cut & (df.tfmed > 0.6) & (df.locqual < 5)][
'ra dec glat glon ts tflow tfmed a locqual'.split()]
print len(pcand)
pcand.index.name = 'name'
pcand.to_csv('weak_pulsar_candidate.csv')
try:
pc = makepivot.MakeCollection('weak pulsar candidates',
'sedfig',
'weak_pulsar_candidate.csv',
refresh=True)
self.pulsar_pivot_info = """<p> These can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight. """ % pc.cId
except Exception, msg:
self.pulsar_pivot_info = '<p>No pivot output; job failed %s' % msg
def flag_proc(self):
""" Flagged source summary:
%(flagged_link)s
"""
# Generate summary table for flagged sources (except
t = self.df[(self.df.flags & 7 > 0) & (
self.df.ts > 10
)]['ra dec ts fitqual pull0 eflux pindex beta cutoff index2 flags roiname'
.split()]
t.to_csv('flagged_sources.csv')
print 'wrote %d sources to flagged_sources.csv' % len(t)
num = [sum(self.df.flags & 2**b > 0) for b in range(4)]
flagtable = pd.DataFrame(
dict(number=num,
description=('tails', 'poor fits', 'low energy bad',
'poor localization')))
flagtable.index.name = 'bit'
self.flagged_link = """\
<p>A number of these sources have been flagged to indicate potential issues with the spectral fit.
The flag bits and number flagged as such are:
%s<br> """ % html_table(flagtable, href=False)
try:
pc = makepivot.MakeCollection(
'flagged sources %s' % os.path.split(os.getcwd())[-1],
'sedfig', 'flagged_sources.csv')
self.flagged_link += """\
<p>These can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % pc.cId
except Exception, msg:
print "**** Failed to make pivot table, perhaps need to run sedinfo first: %s" % msg
def collection(self, name='pulsar candidates v2', refresh=False):
"""Pivot collection.
%(pivot_link)s
"""
from uw.utilities import makepivot
cid = 263
cid =makepivot.MakeCollection(name, 'sedfig', self.cvsname, refresh=refresh).cId
self.pivot_link = """\
<p>The data set, with SED images can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % cid
def delta_ts(self, dmax=10, dmin=-1, pivotit=True):
""" Delta TS
Plots of the TS for the gtlike fit spectra determined with the pointlike analysis,
compared with the pointlike value.<br>
Mismatches: %(over_ts)d with gtlike worse by %(dmax)d; %(under_ts)d with pointlike worse by %(dmin)d.<br>
<br><b>Top left</b>": Scatter plot of $\Delta$ TS with the pointlike TS.
<br><b>Top middle</b>: Histogram of $\Delta$ TS.
<br><b>Top right</b>: Distribution with respect to galactic latitude, with the
subset with discrepancies shown.
<br><b>Bottom left</b>
<br><b>Bottom middle</b> For strong sources, this checks the possibilty that the origin of
discrepancies for the gtlike fits applied to the pointlike data is a consequence of a exposure
bias. If so, there would be a correlation of the ratio of fluxes at the gtlike pivot
energy with $\Delta$ TS.
<br>%(mismatch_table)s
<br>%(pivot_info)s
"""
df = self.dfx
delta = df.ts_delta
mismatch = (delta > dmax) + (delta < dmin)
self.dmax = dmax
self.dmin = dmin
df['logflux'] = np.log10(np.asarray(df.flux, float))
fixme = df[mismatch][
'name ts ts_gtlike glat plane fitqual ts_delta ts_gt ts_pt logflux flux_ratio freebits beta roiname'
.split()].sort_index(by='roiname')
fixme.index = fixme.name
fixme.index.name = 'name'
self.mismatch_table = html_table(
fixme,
columns={},
name=self.plotfolder + '/mismatch',
heading='Table of poor matches with delta_ts<%d or >%d' %
(dmin, dmax),
href=True,
float_format=FloatFormat(2))
fixme.to_csv('gtlike_mismatch.csv')
print 'wrote %d entries to gtlike_mismatch.csv' % len(fixme)
version = os.path.split(os.getcwd())[-1]
if pivotit:
try:
pc = makepivot.MakeCollection('gtlike mismatch %s/%s' %
(version, self.catname),
'gtlike/sed',
'gtlike_mismatch.csv',
refresh=True)
self.pivot_info = """<p> These can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight. """ % pc.cId
except Exception, msg:
self.pivot_info = '<p>No pivot output; job failed %s' % msg
def make_collection(self):
self.collection_html = ''
try:
t = makepivot.MakeCollection('flagged localizations %s' %
self.skymodel,
'tsmap_fail',
'flagged_localizations.csv',
refresh=True)
makepivot.set_format(t.cId)
self.collection_html = """
<p>The images and associated values can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % t.cId
except Exception, msg:
print "**** Failed to make pivot table: %s" % msg
def chisq_plots(self,
use10=True,
unweight=False,
hsize=(1.0, 0.7, 1.5, 0.7),
vmin=0,
vmax=50,
bcut=10,
grid_flag=True,
makecollection=False):
""" chi squared plots
chi squared distribution
<p>Only for bins below 10 GeV.
<p>%(bad_roi_link)s
<p>%(bad_roi_html)s
"""
fig, axs = self.subplot_array(hsize, figsize=(11, 5))
if unweight:
chisq = self.rois.uchisq
else:
chisq = self.rois.chisq if not use10 else self.rois.chisq10
chisqtxt = r'$\chi^2$'
# make a table of the bad ones, sorted by chisq
bad_rois = self.rois[chisq > vmax]['glat glon chisq'.split()]
bad_rois['chisq'] = chisq
bad_rois.sort_values(by='chisq', ascending=False)
bad_rois.to_csv('bad_rois.csv')
if not self.skymodel.startswith('month') and makecollection:
try:
pc = makepivot.MakeCollection(
'bad rois %s' % os.path.split(os.getcwd())[-1], 'countfig',
'bad_rois.csv')
self.bad_roi_link = """\
<p>A list of %d bad ROIs, with chisq>%.0f, can examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % (len(bad_rois), vmax,
pc.cId)
except Exception, msg:
self.bad_roi_link = 'Failed to create Pivot: {}'.format(msg)
print self.bad_roi_link
def load_moment_analysis(self, make_collection=True):
""" check results of moment analysis
"""
m =self.df.moment
has_moment = [x is not None for x in m]
print 'Found %d sources with moment analysls' % sum(has_moment)
mdf = pd.DataFrame(m[has_moment])
u = np.array([list(x) for x in mdf.moment])
self.dfm=md= pd.DataFrame(u, index=mdf.index, columns='rax decx ax bx angx size peak_fract'.split())
md['locqual'] = self.df.locqual
md['ts'] = self.df.ts
md['delta_ts'] = self.df.delta_ts
md['roiname'] = self.df.roiname
md['a'] = self.df.a
# generate the angular difference of fit vs. current positions
delta=[]
for n,t in self.df.iterrows():
ellipse = t['ellipse']
delta.append(np.degrees(SkyDir(*ellipse[:2]).difference(t['skydir'])) if ellipse is not None else None )
self.df['delta'] = delta
md['delta']= self.df.delta
filename = 'moment_localizations.csv'
md.to_csv(filename)
print 'Write file %s' % filename
if not make_collection: return md
# now make a collection with images and data
self.moment_collection_html=''
try:
t = makepivot.MakeCollection('moment analysis localizations %s'%self.skymodel, 'tsmap_fail', 'moment_localizations.csv',
refresh=True)
makepivot.set_format(t.cId)
self.moment_collection_html="""
<p>The images and associated values can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % t.cId
except Exception, msg:
print "**** Failed to make moment pivot table: %s" % msg
def load_moment_analysis(self):
""" check results of moment analysis
"""
m = self.df.moment
has_moment = [x is not None for x in m]
print 'Found %d sources with moment analysls' % sum(has_moment)
mdf = pd.DataFrame(m[has_moment])
u = np.array([list(x) for x in mdf.moment])
self.dfm = md = pd.DataFrame(
u,
index=mdf.index,
columns='rax decx ax bx angx size peak_fract'.split())
md['locqual'] = self.df.locqual
md['ts'] = self.df.ts
md['delta_ts'] = self.df.delta_ts
md['roiname'] = self.df.roiname
md['a'] = self.df.a
filename = 'moment_localizations.csv'
md.to_csv(filename)
print 'Write file %s' % filename
# no make a collection with images and data
self.moment_collection_html = ''
try:
t = makepivot.MakeCollection('moment analysis localizations %s' %
self.skymodel,
'tsmap_fail',
'moment_localizations.csv',
refresh=True)
makepivot.set_format(t.cId)
self.moment_collection_html = """
<p>The images and associated values can be examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % t.cId
except Exception, msg:
print "**** Failed to make moment pivot table: %s" % msg
def make_collection(self):
"""Pivot table with all sources
Inputs are:
<ul><li> a table of sources, a <a href="../../%(csv_file)s?download=True">csv file</a>
<li><a href="../../%(sedfig_zip)s?download=true">zip file</a> containing all SED plots.
</ul>
<br>
%(full_pivot_link)s
"""
self.make_facets()
assert os.path.exists(
self.csv_file), 'Did not find the file %s' % self.csv_file
self.sedfig_zip = 'sedfig.zip'
try:
pc = makepivot.MakeCollection('all sources %s' % self.skymodel,
'sedfig', self.csv_file)
self.full_pivot_link = """\
<p>These can be examined with a
<a href="http://%s/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % (makepivot.http_host,
pc.cId)
except Exception, msg:
print "**** Failed to make pivot table, perhaps need to run sedinfo first: %s" % msg
def chisq_plots(self,
use10=False,
unweight=True,
hsize=(1.0, 0.7, 1.5, 0.7),
vmin=0,
vmax=50,
bcut=10,
grid_flag=True):
""" chi squared plots
chi squared distribution
<p>Note that this chi squared is modified by the unweighting factors.
%(bad_roi_link)s
"""
fig, axs = self.subplot_array(hsize, figsize=(11, 5))
if unweight:
chisq = self.rois.uchisq
else:
chisq = self.rois.chisq if not use10 else self.rois.chisq10
chisqtxt = r'$\chi^2$'
# make a table of the bad ones
bad_rois = self.rois[chisq > vmax]['glat glon chisq'.split()]
bad_rois['chisq'] = chisq
bad_rois.to_csv('bad_rois.csv')
pc = makepivot.MakeCollection(
'bad rois %s' % os.path.split(os.getcwd())[-1], 'countfig',
'bad_rois.csv')
self.bad_roi_link = """\
<p>A list of %d bad ROIs, with chisq>%.0f, can examined with a
<a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>,
which requires Silverlight.""" % (len(bad_rois), vmax, pc.cId)
def chisky(ax):
self.basic_skyplot(ax,
self.rois.glon,
self.rois.singlat,
chisq,
s=55,
marker='D',
vmin=vmin,
vmax=vmax,
edgecolor='none',
colorbar=True,
cbtext=chisqtxt)
def chihist(ax, htype='stepfilled'):
bins = np.linspace(0, vmax, 26)
lolat = np.abs(self.rois.glat) < bcut
ax.hist(chisq.clip(0, vmax),
bins,
label='all: mean=%.0f' % chisq.mean(),
histtype=htype)
ax.hist(chisq.clip(0, vmax)[lolat],
bins,
color='orange',
label='|b|<%d (%.0f)' % (bcut, chisq[lolat].mean()),
histtype=htype)
ax.legend(loc='upper right', prop=dict(size=10))
plt.setp(ax, xlabel=chisqtxt, xlim=(0, vmax))
ax.grid(grid_flag)
for f, ax in zip((chihist, chisky), axs.flatten()):
f(ax)
return fig
