def ft1_time_range(ft1filelistname):
'''Return the FT1 time range.'''
ft1start = sys.maxint
ft1stop = 0
if ft1filelistname.startswith('@'):
ft1filelist = open(ft1filelistname[1:], 'r')
while ft1filelist:
ft1filename = ft1filelist.readline()
if len(ft1filename) == 0:
break
ft1file = pyfits.open(ft1filename)
ft1filestart = float(ft1file[0].header['TSTART'])
if ft1filestart < ft1start:
ft1start = ft1filestart
ft1filestop = float(ft1file[0].header['TSTOP'])
if ft1filestop > ft1stop:
ft1stop = ft1filestop
else:
ft1file = pyfits.open(ft1filelistname)
ft1filestart = float(ft1file[0].header['TSTART'])
if ft1filestart < ft1start:
ft1start = ft1filestart
ft1filestop = float(ft1file[0].header['TSTOP'])
if ft1filestop > ft1stop:
ft1stop = ft1filestop
ft1range = {'START': ft1start, 'STOP': ft1stop}
return ft1range
def __init__(self, fitsfiles, extension=1):
import sys
from astro.io import fits as pyfits
#
# If fitsfile is not a list or tuple of file names, assume
# it's a single file name and put it into a single element
# tuple.
#
if type(fitsfiles) != type([]) and type(fitsfiles) != type(()):
fitsfiles = (fitsfiles, )
#
# Process each file named in the list or tuple.
#
columnData = {}
for i, file in enumerate(fitsfiles):
#print "adding", file
table = pyfits.open(file.strip(" "))
if i == 0:
self.names = table[extension].columns.names
for name in self.names:
myData = table[extension].data.field(name)
if myData.dtype.name.find('float') == 0:
myData = np.array(myData, dtype=np.float)
if myData.dtype.name.find('int') == 0:
myData = np.array(myData, dtype=np.int)
if i == 0:
columnData[name] = myData
else:
columnData[name] = _cat((columnData[name], myData))
#
# Add these columns to the internal dictionary.
#
self.__dict__.update(columnData)
def add_angsep_column(filename, ra=0., dec=0.):
'''Add a angular separation column between the photon direction and (ra,dec).
___arguments___:
filename : name of your FT1 file
ra : Right Ascension (deg)
dec : Declination (deg)
'''
file = pyfits.open(filename)
RA = numpy.asarray(file['EVENTS'].data.field('RA'))
DEC = numpy.asarray(file['EVENTS'].data.field('DEC'))
angsep = []
for it in range(len(RA)):
angsep.append(angular_separation(ra, dec, RA[it], DEC[it]))
colname = 'ANGSEP'
# clobber old values
try:
file['EVENTS'].data.field(colname)[:] = angsep
print 'Clobbered old ANGSEP column.'
except:
cols = file['EVENTS'].columns
newcols = cols.add_col(
pyfits.Column(name=colname, format='D', array=angsep))
table = pyfits.new_table(newcols, header=file['EVENTS'].header)
table.name = 'EVENTS'
file[1] = table
file.writeto(filename, clobber=True)
def load_photon_mjds(ft1name):
# Load photon times (presumed to be barycentered) from FT1 file
hdulist = pyfits.open(ft1name)
ft1hdr = hdulist[1].header
ft1dat = hdulist[1].data
MJDREF = -1
TIMEZERO = -1
if ft1hdr['TIMESYS'] != 'TDB':
print "# !!!!!!!!!!!!!!!!!!!!!!!!!!!!"
print "# !!!!!!!!! WARNING !!!!!!!!!! TIMESYS is NOT TDB! You probably want to barycenter your photons!!!"
print "# !!!!!!!!!!!!!!!!!!!!!!!!!!!!"
time.sleep(5)
# Collect TIMEZERO and MJDREF
try:
TIMEZERO = ft1hdr['TIMEZERO']
except KeyError:
TIMEZERO = ft1hdr['TIMEZERI'] + ft1hdr['TIMEZERF']
#print >>outfile, "# TIMEZERO = ",TIMEZERO
try:
MJDREF = ft1hdr['MJDREF']
except KeyError:
# Here I have to work around an issue where the MJDREFF key is stored
# as a string in the header and uses the "1.234D-5" syntax for floats, which
# is not supported by Python
MJDREF = ft1hdr['MJDREFI'] + float(ft1hdr['MJDREFF'].replace(
'D', 'E'))
#print >>outfile, "# MJDREF = ",MJDREF
mjds = ft1dat.field('TIME') / 86400.0 + MJDREF + TIMEZERO
return mjds
def get_header_erange(filename):
'''Return the FT1 energy range from the HEADER: emin, emax.
___arguments___: filename (name of FITS file)
'''
file = pyfits.open(filename)
num = file[1].header['NDSKEYS']
header = file[1].header
right = 'ENERGY'
i = 1
keynum = 0
while i <= num:
word = 'DSTYP%i' % i
test = file[1].header[word]
if test == right:
keynum = i
i = num
i += 1
if keynum == 0:
print 'Error: No energy keyword found in fits header. Exiting...'
exit()
keyword = 'DSVAL%i' % keynum
emin, emax = header[keyword].split(':')
return float(emin), float(emax)
def __init__(self):
ff = sorted(
glob.glob(
os.path.expandvars(
'$FERMI/catalog/srcid/cat/Pulsars_BigFile_*.fits'))
)
t = pyfits.open(ff[-1])
self.d = pd.DataFrame(t[1].data)
def setup(self, **kw):
fnames = glob.glob('hptables_pts*.fits')
assert len(fnames) == 1, 'expect one hptable_pts*.fits file'
self.fname = fnames[0]
self.tables = pd.DataFrame(pyfits.open(self.fname)[1].data)
self.tsname = 'pts'
self.plotfolder = 'pulsar_ts'
self.seedfile, self.seedroot, self.title = 'pseeds.txt', 'PSEED', 'pulsar'
self.bmin = 0 ##
self.make_seeds(kw.pop('refresh', False))
def ft2_time_range(ft2filename):
ft2file = pyfits.open(ft2filename)
# NOTE: header information unreliable if using ftmerge, so we need to look into the data for FT2.
#ft2start=ft2file[0].header['TSTART']
#ft2stop=ft2file[0].header['TSTOP']
# NOTE: using Aous's method of getting information from the data fields for FT2
sc_data = ft2file[1].data
ft2start = sc_data.field('START')[0]
ft2stop = sc_data.field('STOP')[-1]
ft2range = {'START': ft2start, 'STOP': ft2stop}
return ft2range
def summary(self):
"""Summary
%(summary_html)s
"""
# load the catalogs used
srcid_path = sys.path[0] = os.path.expandvars('$FERMI/catalog/srcid/')
import classes
cats = dict()
for module_name in classes.__all__:
cd = dict()
# this is equivalent to "from classes import module_name"
module = __import__('classes.' + module_name, fromlist=['classes'])
for var in 'catid catname prob_prior prob_thres figure_of_merit max_counterparts new_quantity selection'.split(
):
cd[var] = module.__dict__[var]
cats[module_name] = cd
if cd['catname'].find('*') > 0:
try:
t = glob.glob(srcid_path + '/cat/' + cd['catname'])[-1]
cd['catname'] = os.path.split(t)[-1]
except:
print 'File %s not found' % s
self.catdf = pd.DataFrame(cats).T
self.catdf['objects'] = [
len(pyfits.open(srcid_path + 'cat/' + fname)[1].data)
for fname in self.catdf.catname
]
# make dict of catnames with values the number of associations
t = dict()
for c in self.df10.acat:
if c not in t: t[c] = 1
else: t[c] += 1
self.catdf['associations'] = pd.DataFrame(
t.items(), index=t.keys(),
columns='name associations'.split())['associations']
self.summary_html = html_table(
self.catdf[~pd.isnull(self.catdf.associations)][
'objects associations prob_prior prob_thres catname'.split()],
dict(objects='Objects,number of entries in table',
prob_prior='Prior,prior probability',
prob_thres='Threshold,threshold probability',
catname='Catalog,name of the FITS file in the folder %s' %
(srcid_path + '/cat')),
heading='<br>Table of catalogs with associations',
name=self.plotfolder + '/associated_catalogs',
href=False,
maxlines=100)
def __init__(
self,
catname='3FGL-v13r3_v6r9p1_3lacv12p1_v7.fits',
):
if catname[0] != '/':
catname = os.path.expandvars('$FERMI/catalog/' + catname)
assert os.path.exists(catname), 'Did not find file %s' % catname
self.ft = ft = pyfits.open(catname)[1].data
def nickfix(n):
return n if n[:3] != 'PSR' else 'PSR ' + n[3:]
self.index_3fgl = map(nickfix,
[x.strip()
for x in ft.NickName_3FGL]) #Source_Name
id_prob = [np.nan] * len(ft)
try:
id_prob = ft.ID_Probability_v6r9p1[:,
0] ## should find that suffix
except:
print 'warning: id_prob not set'
self.get_oldname = OldName()
self.cat = pd.DataFrame(
dict(
name3=ft.Source_Name_3FGL_1,
nickname=map(nickfix, ft.NickName_3FGL),
ra=ft.RAJ2000,
dec=ft.DEJ2000,
ts=ft.Test_Statistic,
#skydir=cat_skydirs,
#glat=glat, glon=glon,
#pivot=ft.Pivot_Energy, flux=ft.Flux_Density,
#modelname=ft.SpectrumType,
id_prob=id_prob,
a95=ft.Conf_95_SemiMajor,
b95=ft.Conf_95_SemiMinor,
ang95=ft.Conf_95_PosAng,
flags=np.asarray(ft.Flags_3FGL, int),
),
columns='name3 nickname ra dec ts a95 b95 ang95 id_prob flags'.
split(), # this to order them
index=self.index_3fgl,
)
def setup(self, **kw):
self.nside = nside = kw.pop('nside', 512)
self.tsname = tsname = kw.pop('tsname', 'ts')
self.title = 'TS Table analysis'
fnames = glob.glob('hptables_*_%d.fits' % nside)
assert len(fnames) > 0, 'did not find hptables_*_%d.fits file' % nside
self.fname = None
for fname in fnames:
t = fname.split('_')
if tsname in t:
print 'Found table %s in file %s' % (tsname, fname)
self.fname = fname
break
assert self.fname is not None, 'Table %s not found in files %s' % (
tsname, fnames)
self.tables = pd.DataFrame(pyfits.open(self.fname)[1].data)
print 'loaded file %s ' % self.fname
self.plotfolder = 'hptables_%s' % tsname
self.seedfile, self.seedroot, self.bmin = \
'seeds_%s.txt'%tsname, '%s%s'%( (tsname[-1]).upper(),self.skymodel[-3:] ) , 0
def ait_plots(self, pattern='hptables*.fits', **kwargs):
show_kw_dict=dict(
kde=dict(nocolorbar=True, scale='log',vmin=4.5,vmax=7.5, cmap=colormaps.sls),
ts = dict(nocolorbar=False, vmin=10, vmax=25),
galactic = dict(nocolorbar=True, scale='log', cmap=colormaps.sls),
counts = dict(scale='log'),
)
show_kw_default = dict(vmin=kwargs.pop('vmin',None), vmax=kwargs.pop('vmax',None))
dpi = kwargs.pop('dpi', 120)
infits = glob.glob(pattern)
if len(infits)==0:
raise Exception('No files match pattern %s' %pattern)
hp="<p>Click on any of the thumbnails below to see an expanded version. All are AIT projections."
for filename in infits:
try:
t = pyfits.open(filename)[1].data
except Exception, msg:
raise Exception('Failed to open fits file "%s": %s'% (filename, msg))
nside = int(np.sqrt(len(t)/12.))
hrow = '\n<table><tr>'
mrow = '</tr>\n<tr>'
for table in t.dtype.names:
print 'processing table %s, length=%d' % (table, len(table))
outfile = self._check_exist(table+'_ait.png')
if outfile is not None:
dm = display_map.DisplayMap(t.field(table))
show_kw=show_kw_dict.get(table, show_kw_default)
dm.fill_ait(show_kw=show_kw, **kwargs)
plt.savefig(outfile, bbox_inches='tight', dpi=dpi)
make_thumbnail(outfile)
print 'wrote %s image and thumbnail' % outfile
else:
print 'using existing %s and thumbnail' % table
hrow += '<td>%s</td>' % table
mrow += """\n<td><a href="%(path)s_ait.png">
<img alt="%(path)s_ait.png"
src="%(path)s_ait_thumbnail.png" /></a> </td>""" % dict(path=table)
def get_header_position(filename):
'''return the Position keyword in the FT1 header and extract the ra, dec, and radius values.
___arguments___: filename (name of FITS file)
'''
file = pyfits.open(filename)
num = file[1].header['NDSKEYS']
header = file[1].header
right = 'POS(RA,DEC)'
i = 1
keynum = 0
while i <= num: #this step is necessary since it is not clear that the POS key word will have the same number always
word = 'DSTYP%i' % i
test = file[1].header[word]
if test == right:
keynum = i
i = num
i += 1
if keynum == 0: #DSKEYS start numbering at 1, if this value hasn't been updated, KEYword doesn't exist
print 'Error: No position keyword found in fits header (assuming position is RA and DEC. Exiting...'
exit()
keyword = 'DSVAL%i' % keynum
try:
ra, dec, rad = header[keyword].strip('circle()').split(
','
) #gets rid of the circle and parenthesis part and splits around the comma
return float(ra), float(dec), float(rad)
except ValueError:
ra, dec, rad = header[keyword].strip('CIRCLE()').split(
','
) #gets rid of the circle and parenthesis part and splits around the comma
return float(ra), float(dec), float(rad)
def pulsar_check(self):
"""LAT pulsar check
%(atable)s
"""
# compare with LAT pulsar catalog
tt = set(self.df.name[self.df.psr])
pulsar_lat_catname = sorted(
glob.glob(
os.path.expandvars(
'$FERMI/catalog/srcid/cat/obj-pulsar-lat_*')))[-1]
print 'opening LAT catalog file %s' % pulsar_lat_catname
pp = pyfits.open(pulsar_lat_catname)[1].data
lat = pd.DataFrame(pp, index=[n.strip() for n in pp.Source_Name])
lat['ts'] = self.df[self.df.psr]['ts']
lat['ROI_index'] = [
Band(12).index(SkyDir(float(ra), float(dec)))
for ra, dec in zip(lat.RAJ2000, lat.DEJ2000)
]
lat['skydir'] = [
SkyDir(float(ra), float(dec))
for ra, dec in zip(lat.RAJ2000, lat.DEJ2000)
]
#map(SkyDir, np.array(lat.RAJ2000,float), np.array(lat.DEJ2000,float))
lat['sourcedir'] = self.df.skydir[self.df.psr]
lat['delta'] = [
np.degrees(s.difference(t)) if not type(t) == float else np.nan
for s, t in zip(lat.skydir, lat.sourcedir)
]
far = lat.delta > 0.25
dc2names = set(pp.Source_Name)
print 'sources with exp cutoff not in LAT catalog:', np.asarray(
list(tt.difference(dc2names)))
print 'Catalog entries not found:', list(dc2names.difference(tt))
missing = np.array([np.isnan(x) or x < 10. for x in lat.ts])
# this used to work but now generates 'endian' message
#latsel = lat[missing]['RAJ2000 DEJ2000 ts ROI_index'.split()]
missing_names = lat.index[missing]
cols = 'RAJ2000 DEJ2000 ts ROI_index'.split()
self.latsel = latsel = pd.DataFrame(np.array(
[lat[id][missing] for id in cols]),
index=cols,
columns=missing_names).T
#psrx = np.array([x in 'pulsar_fom pulsar_low msp pulsar_big'.split() for x in self.df.acat])
#print '%d sources found in other pulsar catalogs' % sum(psrx)
self.atable = '<h4>Compare with LAT pulsar catalog: %s</h4>' % os.path.split(
pulsar_lat_catname)[-1]
self.atable += '<p>Sources fit with exponential cutoff not in catalog %s' % np.asarray(
list(tt.difference(dc2names)))
self.atable += html_table(
latsel,
dict(ts='TS,Test Statistic',
ROI_index='ROI Index,Index of the ROI, a HEALPix ring index'),
heading=
'<p>%d LAT catalog entries not in the model (TS shown as NaN), or too weak.'
% sum(missing),
name=self.plotfolder + '/missing',
maxlines=20,
float_format=(FloatFormat(2)))
if sum(far) > 0:
far_names = lat.index[far]
cols = 'delta ts RAJ2000 DEJ2000 ROI_index'.split()
latfar = pd.DataFrame(np.array([lat[id][far] for id in cols]),
index=cols,
columns=far_names).T
print 'LAT pulsar catalog entries found more than 0.25 deg from catalog:'
print far_names
self.atable += '<p>Pulsars located > 0.25 deg from nominal'\
+ latfar.to_html(float_format=FloatFormat(2))
def setup(
self,
cat='3FGL-v13r3_v6r9p1_3lacv12p1_v7.fits', #'gll_psc4yearsource_v9_assoc_v6r3p0.fit', #gll_psc_v06.fit',
catname='3FGL',
**kw):
super(SourceComparison, self).setup(**kw)
self.catname = catname
self.plotfolder = 'comparison_%s' % catname
if not os.path.exists('plots/' + self.plotfolder):
os.mkdir('plots/' + self.plotfolder)
if cat[0] != '/':
cat = os.path.expandvars('$FERMI/catalog/' + cat)
assert os.path.exists(cat), 'Did not find file %s' % cat
ft = pyfits.open(cat)[1].data
self.ft = ft # temp
print 'loaded FITS catalog file %s with %d entries' % (cat, len(ft))
id_prob = [np.nan] * len(ft)
try:
id_prob = ft.ID_Probability_v6r9p1[:,
0] ## should find that suffix
except:
print 'warning: id_prob not set'
cat_skydirs = map(lambda x, y: SkyDir(float(x), float(y)), ft.RAJ2000,
ft.DEJ2000)
glat = [s.b() for s in cat_skydirs]
glon = [s.l() for s in cat_skydirs]
def nickfix(n):
return n if n[:3] != 'PSR' else 'PSR ' + n[3:]
index = map(nickfix,
[x.strip() for x in ft.NickName_3FGL]) #Source_Name
self.cat = pd.DataFrame(
dict(
name3=ft.Source_Name_3FGL_1,
nickname=map(nickfix, ft.NickName_3FGL),
ra=ft.RAJ2000,
dec=ft.DEJ2000,
ts=ft.Test_Statistic,
skydir=cat_skydirs,
glat=glat,
glon=glon,
#pivot=ft.Pivot_Energy, flux=ft.Flux_Density,
#modelname=ft.SpectrumType,
id_prob=id_prob,
a95=ft.Conf_95_SemiMajor,
b95=ft.Conf_95_SemiMinor,
ang95=ft.Conf_95_PosAng,
flags=np.asarray(ft.Flags_3FGL, int),
),
columns=
'name3 nickname ra dec glat glon skydir ts a95 b95 ang95 id_prob flags'
.split(), # this to order them
index=index,
)
self.cat.index.name = 'name'
self.cat['pt_flags'] = self.df.flags
self.cat['pt_ts'] = self.df.ts
self.cat['pt_ra'] = self.df.ra
self.cat['pt_dec'] = self.df.dec
def find_close(A, B):
""" helper function: make a DataFrame with A index containg columns of the
name of the closest entry in B, and its distance
A, B : DataFrame objects each with a skydir column
"""
def mindist(a):
d = map(a.difference, B.skydir.values)
n = np.argmin(d)
return (B.index[n], np.degrees(d[n]))
return pd.DataFrame(map(mindist, A.skydir.values),
index=A.index,
columns=('otherid', 'distance'))
if catname == '2FGL' or catname == '3FGL':
print 'generating closest distance to catalog "%s"' % cat
closedf = find_close(self.df, self.cat)
self.df['closest'] = closedf['distance']
self.df['close_name'] = closedf.otherid
#closest = np.degrees(np.array([min(map(sdir.difference, cat_skydirs))for sdir in self.df.skydir.values]))
#self.df['closest'] = closest
closedf.to_csv(
os.path.join('plots', self.plotfolder,
'comparison_%s.csv' % catname))
closest2 = np.degrees(
np.array([
min(map(sdir.difference, self.df.skydir.values))
for sdir in cat_skydirs
]))
self.cat['closest'] = closest2
def run(**kwargs):
if (len(kwargs.keys()) == 0):
#Nothing specified, the user needs just help!
thisCommand.getHelp()
return
pass
#Get parameters values
thisCommand.setParValuesFromDictionary(kwargs)
try:
ra = thisCommand.getParValue('ra')
dec = thisCommand.getParValue('dec')
particlemodel = thisCommand.getParValue('particle_model')
galacticmodel = thisCommand.getParValue('galactic_model')
sourcemodel = thisCommand.getParValue('source_model')
filteredeventfile = thisCommand.getParValue('filteredeventfile')
xmlmodel = thisCommand.getParValue('xmlmodel')
triggername = thisCommand.getParValue('triggername')
ft2file = thisCommand.getParValue('ft2file')
clobber = _yesOrNoToBool(thisCommand.getParValue('clobber'))
verbose = _yesOrNoToBool(thisCommand.getParValue('verbose'))
except KeyError as err:
print("\n\nERROR: Parameter %s not found or incorrect! \n\n" %
(err.args[0]))
#Print help
print thisCommand.getHelp()
return
pass
#Get the IRF from the event file
try:
f = pyfits.open(filteredeventfile)
except:
raise GtBurstException(
31, "Could not open filtered event file %s" % (filteredeventfile))
tstart = float(f[0].header['_TMIN'])
tstop = float(f[0].header['_TMAX'])
irf = str(f[0].header['_IRF'])
ra = float(f[0].header['_ROI_RA'])
dec = float(f[0].header['_ROI_DEC'])
roi = float(f[0].header['_ROI_RAD'])
#Lookup table for the models
models = {}
if (particlemodel == 'isotr with pow spectrum'):
models[
'isotr with pow spectrum'] = LikelihoodComponent.IsotropicPowerlaw(
)
elif (particlemodel == 'isotr template'):
models['isotr template'] = LikelihoodComponent.IsotropicTemplate(irf)
pass
if (galacticmodel == 'template'):
models['template'] = LikelihoodComponent.GalaxyAndExtragalacticDiffuse(
irf, ra, dec, 2.5 * roi)
elif (galacticmodel == 'template (fixed norm.)'):
models[
'template (fixed norm.)'] = LikelihoodComponent.GalaxyAndExtragalacticDiffuse(
irf, ra, dec, 2.5 * roi)
models['template (fixed norm.)'].fixNormalization()
pass
deltat = numpy.sum(f['GTI'].data.field('STOP') -
f['GTI'].data.field('START'))
f.close()
triggertime = dataHandling.getTriggerTime(filteredeventfile)
if (irf.lower().find('source') >= 0 and particlemodel != 'isotr template'):
raise GtBurstException(
6,
"Do not use '%s' as model for the particle background in SOURCE class. Use '%s' instead."
% (particlemodel, 'isotropic template'))
modelsToUse = [
LikelihoodComponent.PointSource(ra, dec, triggername, sourcemodel)
]
if (particlemodel != 'none'):
if (particlemodel == 'bkge'):
if (ft2file == None or ft2file == ''):
raise ValueError(
"If you want to use the BKGE, you have to provide an FT2 file!"
)
modelsToUse.append(
LikelihoodComponent.BKGETemplate(filteredeventfile, ft2file,
tstart, tstop, triggername,
triggertime))
else:
modelsToUse.append(models[particlemodel])
if (galacticmodel != 'none'):
modelsToUse.append(models[galacticmodel])
xml = LikelihoodComponent.LikelihoodModel()
xml.addSources(*modelsToUse)
xml.writeXML(xmlmodel)
xml.add2FGLsources(ra, dec, float(roi) + 8.0, xmlmodel, deltat)
dataHandling._writeParamIntoXML(xmlmodel,
IRF=irf,
OBJECT=triggername,
RA=ra,
DEC=dec)
return 'xmlmodel', xmlmodel