def poor_loc(self): """ Poorly localized sources %(poorly_localized_table_check)s """ if len(self.poorloc) > 0: self.poorly_localized_table_check = html_table(self.poorloc, name=self.plotfolder+'/poorly_localized', heading='<h4>Table of %d poorly localized (a>%.2f deg, or qual>%.1f with TS>%d) sources</h4>'\ % ( len(self.poorloc),self.acut,self.qualcut, self.tscut), float_format=FloatFormat(2)) #poorly_localized_tablepath = os.path.join(self.plotfolder,'poorly_localized_table.html') #open('poorly_localized_table.html','w').write(tohtml) #print 'Wrote poorly_localized_table.html' #open(os.path.join(poorly_localized_tablepath),'w').write( # '<head>\n' + _html.style + '</head>\n<body>\n<h3>Poorly Localized Source Table</h3>'\ # + tohtml+'\n</body>') #print 'saved html doc to %s' % os.path.join(poorly_localized_tablepath) #self.poorly_localized_table_check =\ # '<p><a href="%s?skipDecoration"> Table of %d poorly localized (a>%.2f deg, or qual>%.1f with TS>%d) sources</a>'\ # % ( 'poorly_localized_table.html',len(self.poorloc),self.acut,self.qualcut, self.tscut) try: version = os.path.split(os.getcwd())[-1] pv = makepivot.MakeCollection('poor localizations %s' % version, 'tsmap_fail', 'poorly_localized.csv', refresh=True) self.poorly_localized_table_check +=\ '<br>A <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">pivot collection </a>of TS maps for these sources can be examined.'%pv.cId except Exception, msg: self.poorly_localized_table_check += '<br>(No pivot table: %s)' % msg print '**** Failed to create pivot: %s' % msg
def pulsar_candidates(self, mints=16): """Pulsar candidate selection Only choose those that are NOT in the gtlike list %(pulsar_pivot_info)s """ df = self.dfx cut = cut = (df.no_gtlike) & (np.abs(df.glat) > 5) & (df.ts > mints) sedrec = self.dfx['sedrec'] tflow = [] tfmed = [] for sr in sedrec: ts = sum(sr.ts) low, middle, high = [ sum(sr.ts[i:i + 4] / ts).round(2) for i in (0, 4, 8) ] tflow.append(low) tfmed.append(middle) self.dfx['tflow'] = tflow self.dfx['tfmed'] = tfmed pcand = df[cut & (df.tfmed > 0.6) & (df.locqual < 5)][ 'ra dec glat glon ts tflow tfmed a locqual'.split()] print len(pcand) pcand.index.name = 'name' pcand.to_csv('weak_pulsar_candidate.csv') try: pc = makepivot.MakeCollection('weak pulsar candidates', 'sedfig', 'weak_pulsar_candidate.csv', refresh=True) self.pulsar_pivot_info = """<p> These can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight. """ % pc.cId except Exception, msg: self.pulsar_pivot_info = '<p>No pivot output; job failed %s' % msg
def flag_proc(self): """ Flagged source summary: %(flagged_link)s """ # Generate summary table for flagged sources (except t = self.df[(self.df.flags & 7 > 0) & ( self.df.ts > 10 )]['ra dec ts fitqual pull0 eflux pindex beta cutoff index2 flags roiname' .split()] t.to_csv('flagged_sources.csv') print 'wrote %d sources to flagged_sources.csv' % len(t) num = [sum(self.df.flags & 2**b > 0) for b in range(4)] flagtable = pd.DataFrame( dict(number=num, description=('tails', 'poor fits', 'low energy bad', 'poor localization'))) flagtable.index.name = 'bit' self.flagged_link = """\ <p>A number of these sources have been flagged to indicate potential issues with the spectral fit. The flag bits and number flagged as such are: %s<br> """ % html_table(flagtable, href=False) try: pc = makepivot.MakeCollection( 'flagged sources %s' % os.path.split(os.getcwd())[-1], 'sedfig', 'flagged_sources.csv') self.flagged_link += """\ <p>These can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % pc.cId except Exception, msg: print "**** Failed to make pivot table, perhaps need to run sedinfo first: %s" % msg
def collection(self, name='pulsar candidates v2', refresh=False): """Pivot collection. %(pivot_link)s """ from uw.utilities import makepivot cid = 263 cid =makepivot.MakeCollection(name, 'sedfig', self.cvsname, refresh=refresh).cId self.pivot_link = """\ <p>The data set, with SED images can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % cid
def delta_ts(self, dmax=10, dmin=-1, pivotit=True): """ Delta TS Plots of the TS for the gtlike fit spectra determined with the pointlike analysis, compared with the pointlike value.<br> Mismatches: %(over_ts)d with gtlike worse by %(dmax)d; %(under_ts)d with pointlike worse by %(dmin)d.<br> <br><b>Top left</b>": Scatter plot of $\Delta$ TS with the pointlike TS. <br><b>Top middle</b>: Histogram of $\Delta$ TS. <br><b>Top right</b>: Distribution with respect to galactic latitude, with the subset with discrepancies shown. <br><b>Bottom left</b> <br><b>Bottom middle</b> For strong sources, this checks the possibilty that the origin of discrepancies for the gtlike fits applied to the pointlike data is a consequence of a exposure bias. If so, there would be a correlation of the ratio of fluxes at the gtlike pivot energy with $\Delta$ TS. <br>%(mismatch_table)s <br>%(pivot_info)s """ df = self.dfx delta = df.ts_delta mismatch = (delta > dmax) + (delta < dmin) self.dmax = dmax self.dmin = dmin df['logflux'] = np.log10(np.asarray(df.flux, float)) fixme = df[mismatch][ 'name ts ts_gtlike glat plane fitqual ts_delta ts_gt ts_pt logflux flux_ratio freebits beta roiname' .split()].sort_index(by='roiname') fixme.index = fixme.name fixme.index.name = 'name' self.mismatch_table = html_table( fixme, columns={}, name=self.plotfolder + '/mismatch', heading='Table of poor matches with delta_ts<%d or >%d' % (dmin, dmax), href=True, float_format=FloatFormat(2)) fixme.to_csv('gtlike_mismatch.csv') print 'wrote %d entries to gtlike_mismatch.csv' % len(fixme) version = os.path.split(os.getcwd())[-1] if pivotit: try: pc = makepivot.MakeCollection('gtlike mismatch %s/%s' % (version, self.catname), 'gtlike/sed', 'gtlike_mismatch.csv', refresh=True) self.pivot_info = """<p> These can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight. """ % pc.cId except Exception, msg: self.pivot_info = '<p>No pivot output; job failed %s' % msg
def make_collection(self): self.collection_html = '' try: t = makepivot.MakeCollection('flagged localizations %s' % self.skymodel, 'tsmap_fail', 'flagged_localizations.csv', refresh=True) makepivot.set_format(t.cId) self.collection_html = """ <p>The images and associated values can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % t.cId except Exception, msg: print "**** Failed to make pivot table: %s" % msg
def chisq_plots(self, use10=True, unweight=False, hsize=(1.0, 0.7, 1.5, 0.7), vmin=0, vmax=50, bcut=10, grid_flag=True, makecollection=False): """ chi squared plots chi squared distribution <p>Only for bins below 10 GeV. <p>%(bad_roi_link)s <p>%(bad_roi_html)s """ fig, axs = self.subplot_array(hsize, figsize=(11, 5)) if unweight: chisq = self.rois.uchisq else: chisq = self.rois.chisq if not use10 else self.rois.chisq10 chisqtxt = r'$\chi^2$' # make a table of the bad ones, sorted by chisq bad_rois = self.rois[chisq > vmax]['glat glon chisq'.split()] bad_rois['chisq'] = chisq bad_rois.sort_values(by='chisq', ascending=False) bad_rois.to_csv('bad_rois.csv') if not self.skymodel.startswith('month') and makecollection: try: pc = makepivot.MakeCollection( 'bad rois %s' % os.path.split(os.getcwd())[-1], 'countfig', 'bad_rois.csv') self.bad_roi_link = """\ <p>A list of %d bad ROIs, with chisq>%.0f, can examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % (len(bad_rois), vmax, pc.cId) except Exception, msg: self.bad_roi_link = 'Failed to create Pivot: {}'.format(msg) print self.bad_roi_link
def load_moment_analysis(self, make_collection=True): """ check results of moment analysis """ m =self.df.moment has_moment = [x is not None for x in m] print 'Found %d sources with moment analysls' % sum(has_moment) mdf = pd.DataFrame(m[has_moment]) u = np.array([list(x) for x in mdf.moment]) self.dfm=md= pd.DataFrame(u, index=mdf.index, columns='rax decx ax bx angx size peak_fract'.split()) md['locqual'] = self.df.locqual md['ts'] = self.df.ts md['delta_ts'] = self.df.delta_ts md['roiname'] = self.df.roiname md['a'] = self.df.a # generate the angular difference of fit vs. current positions delta=[] for n,t in self.df.iterrows(): ellipse = t['ellipse'] delta.append(np.degrees(SkyDir(*ellipse[:2]).difference(t['skydir'])) if ellipse is not None else None ) self.df['delta'] = delta md['delta']= self.df.delta filename = 'moment_localizations.csv' md.to_csv(filename) print 'Write file %s' % filename if not make_collection: return md # now make a collection with images and data self.moment_collection_html='' try: t = makepivot.MakeCollection('moment analysis localizations %s'%self.skymodel, 'tsmap_fail', 'moment_localizations.csv', refresh=True) makepivot.set_format(t.cId) self.moment_collection_html=""" <p>The images and associated values can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % t.cId except Exception, msg: print "**** Failed to make moment pivot table: %s" % msg
def load_moment_analysis(self): """ check results of moment analysis """ m = self.df.moment has_moment = [x is not None for x in m] print 'Found %d sources with moment analysls' % sum(has_moment) mdf = pd.DataFrame(m[has_moment]) u = np.array([list(x) for x in mdf.moment]) self.dfm = md = pd.DataFrame( u, index=mdf.index, columns='rax decx ax bx angx size peak_fract'.split()) md['locqual'] = self.df.locqual md['ts'] = self.df.ts md['delta_ts'] = self.df.delta_ts md['roiname'] = self.df.roiname md['a'] = self.df.a filename = 'moment_localizations.csv' md.to_csv(filename) print 'Write file %s' % filename # no make a collection with images and data self.moment_collection_html = '' try: t = makepivot.MakeCollection('moment analysis localizations %s' % self.skymodel, 'tsmap_fail', 'moment_localizations.csv', refresh=True) makepivot.set_format(t.cId) self.moment_collection_html = """ <p>The images and associated values can be examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % t.cId except Exception, msg: print "**** Failed to make moment pivot table: %s" % msg
def make_collection(self): """Pivot table with all sources Inputs are: <ul><li> a table of sources, a <a href="../../%(csv_file)s?download=True">csv file</a> <li><a href="../../%(sedfig_zip)s?download=true">zip file</a> containing all SED plots. </ul> <br> %(full_pivot_link)s """ self.make_facets() assert os.path.exists( self.csv_file), 'Did not find the file %s' % self.csv_file self.sedfig_zip = 'sedfig.zip' try: pc = makepivot.MakeCollection('all sources %s' % self.skymodel, 'sedfig', self.csv_file) self.full_pivot_link = """\ <p>These can be examined with a <a href="http://%s/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % (makepivot.http_host, pc.cId) except Exception, msg: print "**** Failed to make pivot table, perhaps need to run sedinfo first: %s" % msg
def chisq_plots(self, use10=False, unweight=True, hsize=(1.0, 0.7, 1.5, 0.7), vmin=0, vmax=50, bcut=10, grid_flag=True): """ chi squared plots chi squared distribution <p>Note that this chi squared is modified by the unweighting factors. %(bad_roi_link)s """ fig, axs = self.subplot_array(hsize, figsize=(11, 5)) if unweight: chisq = self.rois.uchisq else: chisq = self.rois.chisq if not use10 else self.rois.chisq10 chisqtxt = r'$\chi^2$' # make a table of the bad ones bad_rois = self.rois[chisq > vmax]['glat glon chisq'.split()] bad_rois['chisq'] = chisq bad_rois.to_csv('bad_rois.csv') pc = makepivot.MakeCollection( 'bad rois %s' % os.path.split(os.getcwd())[-1], 'countfig', 'bad_rois.csv') self.bad_roi_link = """\ <p>A list of %d bad ROIs, with chisq>%.0f, can examined with a <a href="http://deeptalk.phys.washington.edu/PivotWeb/SLViewer.html?cID=%d">Pivot browser</a>, which requires Silverlight.""" % (len(bad_rois), vmax, pc.cId) def chisky(ax): self.basic_skyplot(ax, self.rois.glon, self.rois.singlat, chisq, s=55, marker='D', vmin=vmin, vmax=vmax, edgecolor='none', colorbar=True, cbtext=chisqtxt) def chihist(ax, htype='stepfilled'): bins = np.linspace(0, vmax, 26) lolat = np.abs(self.rois.glat) < bcut ax.hist(chisq.clip(0, vmax), bins, label='all: mean=%.0f' % chisq.mean(), histtype=htype) ax.hist(chisq.clip(0, vmax)[lolat], bins, color='orange', label='|b|<%d (%.0f)' % (bcut, chisq[lolat].mean()), histtype=htype) ax.legend(loc='upper right', prop=dict(size=10)) plt.setp(ax, xlabel=chisqtxt, xlim=(0, vmax)) ax.grid(grid_flag) for f, ax in zip((chihist, chisky), axs.flatten()): f(ax) return fig