def ft1_time_range(ft1filelistname): '''Return the FT1 time range.''' ft1start = sys.maxint ft1stop = 0 if ft1filelistname.startswith('@'): ft1filelist = open(ft1filelistname[1:], 'r') while ft1filelist: ft1filename = ft1filelist.readline() if len(ft1filename) == 0: break ft1file = pyfits.open(ft1filename) ft1filestart = float(ft1file[0].header['TSTART']) if ft1filestart < ft1start: ft1start = ft1filestart ft1filestop = float(ft1file[0].header['TSTOP']) if ft1filestop > ft1stop: ft1stop = ft1filestop else: ft1file = pyfits.open(ft1filelistname) ft1filestart = float(ft1file[0].header['TSTART']) if ft1filestart < ft1start: ft1start = ft1filestart ft1filestop = float(ft1file[0].header['TSTOP']) if ft1filestop > ft1stop: ft1stop = ft1filestop ft1range = {'START': ft1start, 'STOP': ft1stop} return ft1range
def __init__(self, fitsfiles, extension=1): import sys from astro.io import fits as pyfits # # If fitsfile is not a list or tuple of file names, assume # it's a single file name and put it into a single element # tuple. # if type(fitsfiles) != type([]) and type(fitsfiles) != type(()): fitsfiles = (fitsfiles, ) # # Process each file named in the list or tuple. # columnData = {} for i, file in enumerate(fitsfiles): #print "adding", file table = pyfits.open(file.strip(" ")) if i == 0: self.names = table[extension].columns.names for name in self.names: myData = table[extension].data.field(name) if myData.dtype.name.find('float') == 0: myData = np.array(myData, dtype=np.float) if myData.dtype.name.find('int') == 0: myData = np.array(myData, dtype=np.int) if i == 0: columnData[name] = myData else: columnData[name] = _cat((columnData[name], myData)) # # Add these columns to the internal dictionary. # self.__dict__.update(columnData)
def add_angsep_column(filename, ra=0., dec=0.): '''Add a angular separation column between the photon direction and (ra,dec). ___arguments___: filename : name of your FT1 file ra : Right Ascension (deg) dec : Declination (deg) ''' file = pyfits.open(filename) RA = numpy.asarray(file['EVENTS'].data.field('RA')) DEC = numpy.asarray(file['EVENTS'].data.field('DEC')) angsep = [] for it in range(len(RA)): angsep.append(angular_separation(ra, dec, RA[it], DEC[it])) colname = 'ANGSEP' # clobber old values try: file['EVENTS'].data.field(colname)[:] = angsep print 'Clobbered old ANGSEP column.' except: cols = file['EVENTS'].columns newcols = cols.add_col( pyfits.Column(name=colname, format='D', array=angsep)) table = pyfits.new_table(newcols, header=file['EVENTS'].header) table.name = 'EVENTS' file[1] = table file.writeto(filename, clobber=True)
def load_photon_mjds(ft1name): # Load photon times (presumed to be barycentered) from FT1 file hdulist = pyfits.open(ft1name) ft1hdr = hdulist[1].header ft1dat = hdulist[1].data MJDREF = -1 TIMEZERO = -1 if ft1hdr['TIMESYS'] != 'TDB': print "# !!!!!!!!!!!!!!!!!!!!!!!!!!!!" print "# !!!!!!!!! WARNING !!!!!!!!!! TIMESYS is NOT TDB! You probably want to barycenter your photons!!!" print "# !!!!!!!!!!!!!!!!!!!!!!!!!!!!" time.sleep(5) # Collect TIMEZERO and MJDREF try: TIMEZERO = ft1hdr['TIMEZERO'] except KeyError: TIMEZERO = ft1hdr['TIMEZERI'] + ft1hdr['TIMEZERF'] #print >>outfile, "# TIMEZERO = ",TIMEZERO try: MJDREF = ft1hdr['MJDREF'] except KeyError: # Here I have to work around an issue where the MJDREFF key is stored # as a string in the header and uses the "1.234D-5" syntax for floats, which # is not supported by Python MJDREF = ft1hdr['MJDREFI'] + float(ft1hdr['MJDREFF'].replace( 'D', 'E')) #print >>outfile, "# MJDREF = ",MJDREF mjds = ft1dat.field('TIME') / 86400.0 + MJDREF + TIMEZERO return mjds
def get_header_erange(filename): '''Return the FT1 energy range from the HEADER: emin, emax. ___arguments___: filename (name of FITS file) ''' file = pyfits.open(filename) num = file[1].header['NDSKEYS'] header = file[1].header right = 'ENERGY' i = 1 keynum = 0 while i <= num: word = 'DSTYP%i' % i test = file[1].header[word] if test == right: keynum = i i = num i += 1 if keynum == 0: print 'Error: No energy keyword found in fits header. Exiting...' exit() keyword = 'DSVAL%i' % keynum emin, emax = header[keyword].split(':') return float(emin), float(emax)
def __init__(self): ff = sorted( glob.glob( os.path.expandvars( '$FERMI/catalog/srcid/cat/Pulsars_BigFile_*.fits')) ) t = pyfits.open(ff[-1]) self.d = pd.DataFrame(t[1].data)
def setup(self, **kw): fnames = glob.glob('hptables_pts*.fits') assert len(fnames) == 1, 'expect one hptable_pts*.fits file' self.fname = fnames[0] self.tables = pd.DataFrame(pyfits.open(self.fname)[1].data) self.tsname = 'pts' self.plotfolder = 'pulsar_ts' self.seedfile, self.seedroot, self.title = 'pseeds.txt', 'PSEED', 'pulsar' self.bmin = 0 ## self.make_seeds(kw.pop('refresh', False))
def ft2_time_range(ft2filename): ft2file = pyfits.open(ft2filename) # NOTE: header information unreliable if using ftmerge, so we need to look into the data for FT2. #ft2start=ft2file[0].header['TSTART'] #ft2stop=ft2file[0].header['TSTOP'] # NOTE: using Aous's method of getting information from the data fields for FT2 sc_data = ft2file[1].data ft2start = sc_data.field('START')[0] ft2stop = sc_data.field('STOP')[-1] ft2range = {'START': ft2start, 'STOP': ft2stop} return ft2range
def summary(self): """Summary %(summary_html)s """ # load the catalogs used srcid_path = sys.path[0] = os.path.expandvars('$FERMI/catalog/srcid/') import classes cats = dict() for module_name in classes.__all__: cd = dict() # this is equivalent to "from classes import module_name" module = __import__('classes.' + module_name, fromlist=['classes']) for var in 'catid catname prob_prior prob_thres figure_of_merit max_counterparts new_quantity selection'.split( ): cd[var] = module.__dict__[var] cats[module_name] = cd if cd['catname'].find('*') > 0: try: t = glob.glob(srcid_path + '/cat/' + cd['catname'])[-1] cd['catname'] = os.path.split(t)[-1] except: print 'File %s not found' % s self.catdf = pd.DataFrame(cats).T self.catdf['objects'] = [ len(pyfits.open(srcid_path + 'cat/' + fname)[1].data) for fname in self.catdf.catname ] # make dict of catnames with values the number of associations t = dict() for c in self.df10.acat: if c not in t: t[c] = 1 else: t[c] += 1 self.catdf['associations'] = pd.DataFrame( t.items(), index=t.keys(), columns='name associations'.split())['associations'] self.summary_html = html_table( self.catdf[~pd.isnull(self.catdf.associations)][ 'objects associations prob_prior prob_thres catname'.split()], dict(objects='Objects,number of entries in table', prob_prior='Prior,prior probability', prob_thres='Threshold,threshold probability', catname='Catalog,name of the FITS file in the folder %s' % (srcid_path + '/cat')), heading='<br>Table of catalogs with associations', name=self.plotfolder + '/associated_catalogs', href=False, maxlines=100)
def __init__( self, catname='3FGL-v13r3_v6r9p1_3lacv12p1_v7.fits', ): if catname[0] != '/': catname = os.path.expandvars('$FERMI/catalog/' + catname) assert os.path.exists(catname), 'Did not find file %s' % catname self.ft = ft = pyfits.open(catname)[1].data def nickfix(n): return n if n[:3] != 'PSR' else 'PSR ' + n[3:] self.index_3fgl = map(nickfix, [x.strip() for x in ft.NickName_3FGL]) #Source_Name id_prob = [np.nan] * len(ft) try: id_prob = ft.ID_Probability_v6r9p1[:, 0] ## should find that suffix except: print 'warning: id_prob not set' self.get_oldname = OldName() self.cat = pd.DataFrame( dict( name3=ft.Source_Name_3FGL_1, nickname=map(nickfix, ft.NickName_3FGL), ra=ft.RAJ2000, dec=ft.DEJ2000, ts=ft.Test_Statistic, #skydir=cat_skydirs, #glat=glat, glon=glon, #pivot=ft.Pivot_Energy, flux=ft.Flux_Density, #modelname=ft.SpectrumType, id_prob=id_prob, a95=ft.Conf_95_SemiMajor, b95=ft.Conf_95_SemiMinor, ang95=ft.Conf_95_PosAng, flags=np.asarray(ft.Flags_3FGL, int), ), columns='name3 nickname ra dec ts a95 b95 ang95 id_prob flags'. split(), # this to order them index=self.index_3fgl, )
def setup(self, **kw): self.nside = nside = kw.pop('nside', 512) self.tsname = tsname = kw.pop('tsname', 'ts') self.title = 'TS Table analysis' fnames = glob.glob('hptables_*_%d.fits' % nside) assert len(fnames) > 0, 'did not find hptables_*_%d.fits file' % nside self.fname = None for fname in fnames: t = fname.split('_') if tsname in t: print 'Found table %s in file %s' % (tsname, fname) self.fname = fname break assert self.fname is not None, 'Table %s not found in files %s' % ( tsname, fnames) self.tables = pd.DataFrame(pyfits.open(self.fname)[1].data) print 'loaded file %s ' % self.fname self.plotfolder = 'hptables_%s' % tsname self.seedfile, self.seedroot, self.bmin = \ 'seeds_%s.txt'%tsname, '%s%s'%( (tsname[-1]).upper(),self.skymodel[-3:] ) , 0
def ait_plots(self, pattern='hptables*.fits', **kwargs): show_kw_dict=dict( kde=dict(nocolorbar=True, scale='log',vmin=4.5,vmax=7.5, cmap=colormaps.sls), ts = dict(nocolorbar=False, vmin=10, vmax=25), galactic = dict(nocolorbar=True, scale='log', cmap=colormaps.sls), counts = dict(scale='log'), ) show_kw_default = dict(vmin=kwargs.pop('vmin',None), vmax=kwargs.pop('vmax',None)) dpi = kwargs.pop('dpi', 120) infits = glob.glob(pattern) if len(infits)==0: raise Exception('No files match pattern %s' %pattern) hp="<p>Click on any of the thumbnails below to see an expanded version. All are AIT projections." for filename in infits: try: t = pyfits.open(filename)[1].data except Exception, msg: raise Exception('Failed to open fits file "%s": %s'% (filename, msg)) nside = int(np.sqrt(len(t)/12.)) hrow = '\n<table><tr>' mrow = '</tr>\n<tr>' for table in t.dtype.names: print 'processing table %s, length=%d' % (table, len(table)) outfile = self._check_exist(table+'_ait.png') if outfile is not None: dm = display_map.DisplayMap(t.field(table)) show_kw=show_kw_dict.get(table, show_kw_default) dm.fill_ait(show_kw=show_kw, **kwargs) plt.savefig(outfile, bbox_inches='tight', dpi=dpi) make_thumbnail(outfile) print 'wrote %s image and thumbnail' % outfile else: print 'using existing %s and thumbnail' % table hrow += '<td>%s</td>' % table mrow += """\n<td><a href="%(path)s_ait.png"> <img alt="%(path)s_ait.png" src="%(path)s_ait_thumbnail.png" /></a> </td>""" % dict(path=table)
def get_header_position(filename): '''return the Position keyword in the FT1 header and extract the ra, dec, and radius values. ___arguments___: filename (name of FITS file) ''' file = pyfits.open(filename) num = file[1].header['NDSKEYS'] header = file[1].header right = 'POS(RA,DEC)' i = 1 keynum = 0 while i <= num: #this step is necessary since it is not clear that the POS key word will have the same number always word = 'DSTYP%i' % i test = file[1].header[word] if test == right: keynum = i i = num i += 1 if keynum == 0: #DSKEYS start numbering at 1, if this value hasn't been updated, KEYword doesn't exist print 'Error: No position keyword found in fits header (assuming position is RA and DEC. Exiting...' exit() keyword = 'DSVAL%i' % keynum try: ra, dec, rad = header[keyword].strip('circle()').split( ',' ) #gets rid of the circle and parenthesis part and splits around the comma return float(ra), float(dec), float(rad) except ValueError: ra, dec, rad = header[keyword].strip('CIRCLE()').split( ',' ) #gets rid of the circle and parenthesis part and splits around the comma return float(ra), float(dec), float(rad)
def pulsar_check(self): """LAT pulsar check %(atable)s """ # compare with LAT pulsar catalog tt = set(self.df.name[self.df.psr]) pulsar_lat_catname = sorted( glob.glob( os.path.expandvars( '$FERMI/catalog/srcid/cat/obj-pulsar-lat_*')))[-1] print 'opening LAT catalog file %s' % pulsar_lat_catname pp = pyfits.open(pulsar_lat_catname)[1].data lat = pd.DataFrame(pp, index=[n.strip() for n in pp.Source_Name]) lat['ts'] = self.df[self.df.psr]['ts'] lat['ROI_index'] = [ Band(12).index(SkyDir(float(ra), float(dec))) for ra, dec in zip(lat.RAJ2000, lat.DEJ2000) ] lat['skydir'] = [ SkyDir(float(ra), float(dec)) for ra, dec in zip(lat.RAJ2000, lat.DEJ2000) ] #map(SkyDir, np.array(lat.RAJ2000,float), np.array(lat.DEJ2000,float)) lat['sourcedir'] = self.df.skydir[self.df.psr] lat['delta'] = [ np.degrees(s.difference(t)) if not type(t) == float else np.nan for s, t in zip(lat.skydir, lat.sourcedir) ] far = lat.delta > 0.25 dc2names = set(pp.Source_Name) print 'sources with exp cutoff not in LAT catalog:', np.asarray( list(tt.difference(dc2names))) print 'Catalog entries not found:', list(dc2names.difference(tt)) missing = np.array([np.isnan(x) or x < 10. for x in lat.ts]) # this used to work but now generates 'endian' message #latsel = lat[missing]['RAJ2000 DEJ2000 ts ROI_index'.split()] missing_names = lat.index[missing] cols = 'RAJ2000 DEJ2000 ts ROI_index'.split() self.latsel = latsel = pd.DataFrame(np.array( [lat[id][missing] for id in cols]), index=cols, columns=missing_names).T #psrx = np.array([x in 'pulsar_fom pulsar_low msp pulsar_big'.split() for x in self.df.acat]) #print '%d sources found in other pulsar catalogs' % sum(psrx) self.atable = '<h4>Compare with LAT pulsar catalog: %s</h4>' % os.path.split( pulsar_lat_catname)[-1] self.atable += '<p>Sources fit with exponential cutoff not in catalog %s' % np.asarray( list(tt.difference(dc2names))) self.atable += html_table( latsel, dict(ts='TS,Test Statistic', ROI_index='ROI Index,Index of the ROI, a HEALPix ring index'), heading= '<p>%d LAT catalog entries not in the model (TS shown as NaN), or too weak.' % sum(missing), name=self.plotfolder + '/missing', maxlines=20, float_format=(FloatFormat(2))) if sum(far) > 0: far_names = lat.index[far] cols = 'delta ts RAJ2000 DEJ2000 ROI_index'.split() latfar = pd.DataFrame(np.array([lat[id][far] for id in cols]), index=cols, columns=far_names).T print 'LAT pulsar catalog entries found more than 0.25 deg from catalog:' print far_names self.atable += '<p>Pulsars located > 0.25 deg from nominal'\ + latfar.to_html(float_format=FloatFormat(2))
def setup( self, cat='3FGL-v13r3_v6r9p1_3lacv12p1_v7.fits', #'gll_psc4yearsource_v9_assoc_v6r3p0.fit', #gll_psc_v06.fit', catname='3FGL', **kw): super(SourceComparison, self).setup(**kw) self.catname = catname self.plotfolder = 'comparison_%s' % catname if not os.path.exists('plots/' + self.plotfolder): os.mkdir('plots/' + self.plotfolder) if cat[0] != '/': cat = os.path.expandvars('$FERMI/catalog/' + cat) assert os.path.exists(cat), 'Did not find file %s' % cat ft = pyfits.open(cat)[1].data self.ft = ft # temp print 'loaded FITS catalog file %s with %d entries' % (cat, len(ft)) id_prob = [np.nan] * len(ft) try: id_prob = ft.ID_Probability_v6r9p1[:, 0] ## should find that suffix except: print 'warning: id_prob not set' cat_skydirs = map(lambda x, y: SkyDir(float(x), float(y)), ft.RAJ2000, ft.DEJ2000) glat = [s.b() for s in cat_skydirs] glon = [s.l() for s in cat_skydirs] def nickfix(n): return n if n[:3] != 'PSR' else 'PSR ' + n[3:] index = map(nickfix, [x.strip() for x in ft.NickName_3FGL]) #Source_Name self.cat = pd.DataFrame( dict( name3=ft.Source_Name_3FGL_1, nickname=map(nickfix, ft.NickName_3FGL), ra=ft.RAJ2000, dec=ft.DEJ2000, ts=ft.Test_Statistic, skydir=cat_skydirs, glat=glat, glon=glon, #pivot=ft.Pivot_Energy, flux=ft.Flux_Density, #modelname=ft.SpectrumType, id_prob=id_prob, a95=ft.Conf_95_SemiMajor, b95=ft.Conf_95_SemiMinor, ang95=ft.Conf_95_PosAng, flags=np.asarray(ft.Flags_3FGL, int), ), columns= 'name3 nickname ra dec glat glon skydir ts a95 b95 ang95 id_prob flags' .split(), # this to order them index=index, ) self.cat.index.name = 'name' self.cat['pt_flags'] = self.df.flags self.cat['pt_ts'] = self.df.ts self.cat['pt_ra'] = self.df.ra self.cat['pt_dec'] = self.df.dec def find_close(A, B): """ helper function: make a DataFrame with A index containg columns of the name of the closest entry in B, and its distance A, B : DataFrame objects each with a skydir column """ def mindist(a): d = map(a.difference, B.skydir.values) n = np.argmin(d) return (B.index[n], np.degrees(d[n])) return pd.DataFrame(map(mindist, A.skydir.values), index=A.index, columns=('otherid', 'distance')) if catname == '2FGL' or catname == '3FGL': print 'generating closest distance to catalog "%s"' % cat closedf = find_close(self.df, self.cat) self.df['closest'] = closedf['distance'] self.df['close_name'] = closedf.otherid #closest = np.degrees(np.array([min(map(sdir.difference, cat_skydirs))for sdir in self.df.skydir.values])) #self.df['closest'] = closest closedf.to_csv( os.path.join('plots', self.plotfolder, 'comparison_%s.csv' % catname)) closest2 = np.degrees( np.array([ min(map(sdir.difference, self.df.skydir.values)) for sdir in cat_skydirs ])) self.cat['closest'] = closest2
def run(**kwargs): if (len(kwargs.keys()) == 0): #Nothing specified, the user needs just help! thisCommand.getHelp() return pass #Get parameters values thisCommand.setParValuesFromDictionary(kwargs) try: ra = thisCommand.getParValue('ra') dec = thisCommand.getParValue('dec') particlemodel = thisCommand.getParValue('particle_model') galacticmodel = thisCommand.getParValue('galactic_model') sourcemodel = thisCommand.getParValue('source_model') filteredeventfile = thisCommand.getParValue('filteredeventfile') xmlmodel = thisCommand.getParValue('xmlmodel') triggername = thisCommand.getParValue('triggername') ft2file = thisCommand.getParValue('ft2file') clobber = _yesOrNoToBool(thisCommand.getParValue('clobber')) verbose = _yesOrNoToBool(thisCommand.getParValue('verbose')) except KeyError as err: print("\n\nERROR: Parameter %s not found or incorrect! \n\n" % (err.args[0])) #Print help print thisCommand.getHelp() return pass #Get the IRF from the event file try: f = pyfits.open(filteredeventfile) except: raise GtBurstException( 31, "Could not open filtered event file %s" % (filteredeventfile)) tstart = float(f[0].header['_TMIN']) tstop = float(f[0].header['_TMAX']) irf = str(f[0].header['_IRF']) ra = float(f[0].header['_ROI_RA']) dec = float(f[0].header['_ROI_DEC']) roi = float(f[0].header['_ROI_RAD']) #Lookup table for the models models = {} if (particlemodel == 'isotr with pow spectrum'): models[ 'isotr with pow spectrum'] = LikelihoodComponent.IsotropicPowerlaw( ) elif (particlemodel == 'isotr template'): models['isotr template'] = LikelihoodComponent.IsotropicTemplate(irf) pass if (galacticmodel == 'template'): models['template'] = LikelihoodComponent.GalaxyAndExtragalacticDiffuse( irf, ra, dec, 2.5 * roi) elif (galacticmodel == 'template (fixed norm.)'): models[ 'template (fixed norm.)'] = LikelihoodComponent.GalaxyAndExtragalacticDiffuse( irf, ra, dec, 2.5 * roi) models['template (fixed norm.)'].fixNormalization() pass deltat = numpy.sum(f['GTI'].data.field('STOP') - f['GTI'].data.field('START')) f.close() triggertime = dataHandling.getTriggerTime(filteredeventfile) if (irf.lower().find('source') >= 0 and particlemodel != 'isotr template'): raise GtBurstException( 6, "Do not use '%s' as model for the particle background in SOURCE class. Use '%s' instead." % (particlemodel, 'isotropic template')) modelsToUse = [ LikelihoodComponent.PointSource(ra, dec, triggername, sourcemodel) ] if (particlemodel != 'none'): if (particlemodel == 'bkge'): if (ft2file == None or ft2file == ''): raise ValueError( "If you want to use the BKGE, you have to provide an FT2 file!" ) modelsToUse.append( LikelihoodComponent.BKGETemplate(filteredeventfile, ft2file, tstart, tstop, triggername, triggertime)) else: modelsToUse.append(models[particlemodel]) if (galacticmodel != 'none'): modelsToUse.append(models[galacticmodel]) xml = LikelihoodComponent.LikelihoodModel() xml.addSources(*modelsToUse) xml.writeXML(xmlmodel) xml.add2FGLsources(ra, dec, float(roi) + 8.0, xmlmodel, deltat) dataHandling._writeParamIntoXML(xmlmodel, IRF=irf, OBJECT=triggername, RA=ra, DEC=dec) return 'xmlmodel', xmlmodel