def add(self, others, output, binfile, ltcube):
"""Combine this DataSpec instance with another and return the result
The two instances must have consistent definitions (DSS, binning,
and livetime), and must have non-overlapping Gtis. The binfiles and
ltcubes will be combined and written to the provided destinations;
perhaps in the future, I will come up with sensible defaults.
"""
if not hasattr(others, '__iter__'):
others = [others]
for other in others:
exc = self.check_consistency(other)
if exc is not None:
raise (exc)
binfile = os.path.expandvars(binfile)
ltcube = os.path.expandvars(ltcube)
gti = skymaps.Gti(self.gti)
ft1 = self.ft1files
ft2 = self.ft2files
bpd = skymaps.BinnedPhotonData(self.binfile)
for other in others:
gti.intersection(other.gti)
ft1 += other.ft1files
ft2 += other.ft2files
bpd.add(skymaps.BinnedPhotonData(other.binfile))
if gti.computeOntime() > 0:
raise DataError("DataSpec instances have overlapping GTIs")
ft2 = sorted(list(set(ft2)))
bpd.write(binfile)
fitstools.merge_lt([self.ltcube] + [other.ltcube for other in others],
outfile=ltcube,
weighted=self.use_weighted_livetime)
dssman.DSSEntries(self.binfile, header_key=0).write(binfile,
header_key=0)
dssman.DSSEntries(self.ltcube, header_key=0).write(ltcube,
header_key=0)
#TODO: move the copying of DSS entries into the merge_bpd and merge_lt functions
gti_mask = skymaps.Gti(self.gti_mask)
for other in others:
gti_mask.combine(other.gti_mask)
kwargs = dict(ft1=ft1,
ft2=ft2,
binfile=binfile,
ltcube=ltcube,
gti_mask=gti_mask,
binsperdec=self.binsperdec,
mc_src_id=self.mc_src_id,
mc_energy=self.mc_energy,
use_weighted_livetime=self.use_weighted_livetime,
livetime_buffer=self.livetime_buffer,
livetime_pixelsize=self.livetime_pixelsize,
clobber=False)
return DataSpec(output, **kwargs)
def _get_GTI(self):
""" Apply GTI cuts and get resulting merged GTI."""
# get GTI for a set of FT1 files
gti = skymaps.Gti(self.ft1files[0])
for ef in self.ft1files[1:]:
gti.combine(skymaps.Gti(ef))
# apply mask if specified
if self.gti_mask is not None:
before = gti.computeOntime()
gti.intersection(self.gti_mask)
if not self.quiet:
print('applied gti mask, before, after times: {0:.1f}, {1:.1f}'
.format(before, gti.computeOntime()))
return gti
def _make_ltcube(self):
""" Generate the livetime cube."""
#TODO -- unify weighted livetime
import sys
self.quiet = False
self.ltcube = self.ltcube or self._make_default_name(prefix='lt')
roi_info = self.dss.roi_info() if hasattr(self.dss,
'roi_info') else None
if (roi_info is None) or (roi_info[2] > 90):
# full sky ltcube
roi_dir = skymaps.SkyDir(0, 0)
exp_radius = 180
else:
roi_dir = skymaps.SkyDir(roi_info[0], roi_info[1])
exp_radius = roi_info[2] + self.livetime_buffer
if self.zenith_cut is None:
self.zenith_cut = get_default('ZENITH_ANGLE', None)
print 'Warning: using default zenith_cut, {}'.format(
self.zenith_cut)
# raise DataManException('zenith cut not specified')
zenithcut = self.zenith_cut.get_bounds()[1]
if not self.quiet:
if exp_radius == 180:
print 'Constructing all-sky livetime cube'
else:
print(
'Constructing livetime cube about RA,Dec = ({0:0.3f},{1:0.3f}) with a radius of {2:0.3f} deg.'
.format(roi_dir.ra(), roi_dir.dec(), exp_radius))
for i in xrange(1 + self.use_weighted_livetime):
#print('on iteration {0}'.format(i))
sys.stdout.flush()
lt = skymaps.LivetimeCube(cone_angle=exp_radius,
dir=roi_dir,
zcut=np.cos(np.radians(zenithcut)),
pixelsize=self.livetime_pixelsize,
quiet=self.quiet,
weighted=i)
for hf in self.ft2files:
if not self.quiet:
print('checking FT2 file {0}...'.format(hf)),
lt_gti = skymaps.Gti(hf, 'SC_DATA')
if not ((lt_gti.maxValue() < self.gti.minValue()) or
(lt_gti.minValue() > self.gti.maxValue())):
lt.load(hf, self.gti)
if not self.quiet: print 'loaded'
else:
if not self.quiet: print 'not in Gti range'
# write out ltcube
extension = 'WEIGHTED_EXPOSURE' if i else 'EXPOSURE'
lt.write(self.ltcube, extension, not bool(i))
if self.dss is not None:
self.dss.write(self.ltcube, header_key=0)
# write some info to livetime file
f = pyfits.open(self.ltcube)
f[0]._header['RADIUS'] = exp_radius
f[0]._header['PIXSIZE'] = self.livetime_pixelsize
f.writeto(self.ltcube, overwrite=True)
f.close()
def _GTI_setup(self):
"""Create the GTI object that will be used to filter photon events and
to calculate the livetime.
If the user has provided ltcube or binfile arguments, the saved GTI
will be used. The ltcube GTI takes precedence over the binfile GTI;
if both are provided, it is the users responsibility to ensure that
the two sets of GTIs are consistent.
Generally, one will have run gtmktime on the FT1 file(s) provided
to filter out large excursions in rocking angle, and/or make a
a cut for when a (small) ROI is too close to the horizon.
gti_mask is an optional kwarg to provide an additional GTI object that
can, e.g., be used to filter out GRBs. An intersection with the GTIs
from the FT1 files and this gti_mask is made.
"""
if not self.recalcgti:
if self.ltcube is not None and os.path.exists(self.ltcube):
if not self.quiet: print('Using gti from %s' % (self.ltcube))
return skymaps.Gti(self.ltcube)
elif self.binfile is not None and os.path.exists(self.binfile):
if not self.quiet: print('Using gti from %s' % (self.binfile))
return skymaps.Gti(self.binfile)
if len(self.ft1files) == 0 or not os.path.exists(self.ft1files[0]):
raise PixelDataException(
'Cannot process GTI: no ft1 files found, nor %s nor %s' %
(self.ltcube, self.binfile))
gti = skymaps.Gti(self.ft1files[0])
# take the union of the GTI in each FT1 file
for ef in self.ft1files[1:]:
gti.combine(skymaps.Gti(ef))
tmax = self.tstop if self.tstop > 0 else gti.maxValue()
gti = gti.applyTimeRangeCut(self.tstart, tmax) #save gti for later use
if 'gti_mask' in self.__dict__ and self.gti_mask is not None:
before = gti.computeOntime()
gti.intersection(self.gti_mask)
if not self.quiet:
print 'applied gti mask, before, after times: %.1f, %.1f' % (
before, gti.computeOntime())
return gti
def _check_ltcube(self):
""" Verify ltcube exists and is consistent with any existing data cuts.
"""
#if os.path.exists(self.ltcube) and self.legacy :
# print('Accepting ltcube without dss checks since legacy specified')
# return True
if self.clobber or (not os.path.exists(self.ltcube or '')):
print 'checking ltcube: failed clobber on %s' % self.ltcube
return False
# check for presence of important history
# eew -- primary header doesn't have info about extensions, so just
# open the file and use that handle to check header keys and
# extensions.
# h = pyfits.getheader(self.ltcube,0)
lt = pyfits.open(self.ltcube)
try:
lt[0].header['RADIUS']
lt[0].header['PIXSIZE']
except KeyError:
if not self.quiet:
print 'no header info in ltcube?' #pass #return False
# check for weighted extension if we are using it
if self.use_weighted_livetime:
#try: h['WEIGHTED_EXPOSURE']
#except KeyError: return False
try:
assert (len(lt) > 3)
except AssertionError:
print 'fail len(lt)>3:%d' % len(lt)
return False
#
# DSS check
#
dss = dssman.DSSEntries(self.ltcube, header_key=0)
if dss is None or len(dss) == 0:
if self.legacy:
if not self.quiet:
print 'Accepting ltcube without DSS info since legacy specified'
dss = self.dss
else:
raise DataManException('DSS found in ltcube %s' % self.ltcube)
if dss != self.dss:
print 'Failed dss comparison:\n ltcube %s,\n FT1 %s' % (dss,
self.dss)
return False
#
# compare GTI with that found in FT1 or binfile
#
gti = skymaps.Gti(self.ltcube)
if (gti.minValue != self.gti.minValue) or (gti.computeOntime() !=
self.gti.computeOntime()):
print 'Failed gti check:\n ltcube: %s \n binfile: %s' % (gti,
self.gti)
return self.legacy #ignore if legacy, for now
if (not self.quiet): print('Verified ltcube {0}'.format(self.ltcube))
return True
def _check_ltcube(self):
""" Verify ltcube exists and is consistent with any existing data cuts.
"""
#if os.path.exists(self.ltcube) and self.legacy :
# print('Accepting ltcube without dss checks since legacy specified')
# return True
ltcubes = glob.glob(self.ltcube)
if len(ltcubes)==0:
#if self.clobber or (not os.path.exists(self.ltcube or '')):
print 'Checking ltcube: will generate new {}'.format(self.ltcube)
return False
# check for presence of important history
# eew -- primary header doesn't have info about extensions, so just
# open the file and use that handle to check header keys and
# extensions.
# h = pyfits.getheader(self.ltcube,0)
lt = pyfits.open(ltcubes[0])
# DISABLE this: seems to be normal
# try:
# lt[0].header['RADIUS']; lt[0].header['PIXSIZE']
# except KeyError:
# if not self.quiet: print 'no header info in ltcube?' #pass #return False
# check for weighted extension if we are using it
if self.use_weighted_livetime:
#try: h['WEIGHTED_EXPOSURE']
#except KeyError: return False
try: assert(len(lt)>3)
except AssertionError:
print 'fail len(lt)>3:%d' %len(lt)
return False
#
# DSS check
#
nodss=False
dss = dssman.DSSEntries(ltcubes[0],header_key=0)
if dss is None or len(dss)==0:
nodss=True
if True: # Permamently allow this self.legacy:
#if not self.quiet: print 'Accepting ltcube without DSS info since legacy specified'
dss=self.dss
else:
raise DataManException('DSS found in ltcube %s' % ltcubes[0])
if dss != self.dss:
print 'Failed dss comparison:\n ltcube %s,\n FT1 %s' % (dss, self.dss)
return False
#
# compare GTI with that found in FT1 or binfile
#
gti = skymaps.Gti(ltcubes[0])
tdiff = gti.computeOntime() - self.gti.computeOntime()
if (gti.minValue()!=self.gti.minValue()) or abs(tdiff)>1:
print 'Failed gti check, time difference %.1f:\n ltcube: %s \n binfile: %s' % (tdiff, gti, self.gti)
return True #self.legacy #ignore if legacy, for now
if (not self.quiet): print('Verified ltcube {} {}'.format(ltcubes[0],
'(without DSS)' if nodss else ''))
return True
def _process_dataset(self,
dataset,
binfile=None,
interval=None,
month=None,
quiet=False):
""" Parse the dataset as a string lookup key, or a dict
interval: string
name of a
month: sub spec.
"""
if isinstance(dataset, dict):
dataset['event_class_bit'] = dict(source=2, clean=3,
extraclean=4)[dataset.get(
'event_class',
'source').lower()]
dataset.setdefault('theta_cut', 66.422)
self.name = dataset.get('data_name', '(noname)')
self.dict_file = 'input dict'
return DataSpecification(os.path.expandvars('$FERMI/data'),
interval=None,
gti_mask=None,
**dataset)
self.name = dataset
folders = ['.'] + glob.glob(
os.path.join(os.path.expandvars('$FERMI'), 'data'))
for folder in folders:
self.dict_file = dict_file = os.path.join(folder, 'dataspec.py')
if not os.path.exists(dict_file):
continue
try:
ldict = eval(open(dict_file).read())
except Exception, msg:
raise DataSetError('Data dictionary file %s not valid: %s' %
(dict_file, msg))
if interval is not None and not (interval.startswith('month_')
or interval.startswith('year_')):
# note that the underscore is to specify the designated month or year, with their own files
try:
pyfile = os.path.join(folder, 'intervals.py')
idict = eval(open(pyfile).read()) if os.path.exists(
pyfile) else Interval()
gr = idict[interval]
gti_mask = skymaps.Gti([gr[0]], [gr[1]])
if True: #self.verbose:
if not quiet:
print 'apply gti mask %s, %s' % (interval,
gti_mask)
except Exception, msg:
raise DataSetError('Interval dictionary file %s, key %s, problem: %s'\
% (pyfile, interval, msg))
def _check_binfile(self):
""" Verify binfile exists and is consistent with any existing data cuts.
Return False if fails any, print message
(why not an exception?)
"""
if self.binfile is None:
raise DataManException('No bin file specified')
if not os.path.exists(self.binfile):
print 'File %s not found' % self.binfile
sys.stdout.flush()
return False
if self.clobber or self.binfile is None:
print 'self.clobber or self.binfile is None'
sys.stdout.flush()
return False
#
# Check DSS keywords
#
dss = dssman.DSSEntries(self.binfile, header_key=0)
if (dss is None):
print("dss is None")
sys.stdout.flush()
return False
if self.dss is None:
if not self.quiet: print('Extracting DSS from existing binfile')
self.dss = dss
if self.dss != dss:
print 'File %s Failed DSS keyword check: expected \n%s \nbut found \n%s' % (
self.binfile, self.dss, dss)
sys.stdout.flush()
return False
#
# Check GTI
#
gti = skymaps.Gti(self.binfile)
if not self.quiet: print 'GTI from binfile', gti
if gti is None:
raise DataManException('GTI not found in the binfile %s' %
self.binfile)
self.gti = gti
if (gti.minValue != self.gti.minValue) or (gti.computeOntime() !=
self.gti.computeOntime()):
print 'File %s Failed GTI check: \n expect %s \n found %s' % (
self.binfile, self.gti, gti)
return self.legacy # ignore if legacy, for now
if (not self.quiet): print('Verified binfile {0}'.format(self.binfile))
return True
def pointlike_ltcube(evfile,scfile,outfile,dcostheta,binsz, zmax, cone_angle,dir,quiet=False):
""" Run the pointlike ltcube code. """
gti = skymaps.Gti(evfile)
for weighted,clobber in [[False,True],[True,False]]:
lt = skymaps.LivetimeCube(
cone_angle = cone_angle,
dir = dir,
zcut = math.cos(math.radians(zmax)),
pixelsize = binsz,
quiet = quiet,
weighted = weighted)
lt.load(scfile,gti)
extension = 'WEIGHTED_EXPOSURE' if weighted else 'EXPOSURE'
lt.write(outfile,extension,clobber)
fix_pointlike_ltcube(outfile)
def get_livetime(self, pixelsize=1.0, weighted=False, clobber=True):
gti = self.gti
if self.ltcube is not None and os.path.exists(self.ltcube):
try:
lt = skymaps.LivetimeCube(self.ltcube, weighted=weighted)
if not self.quiet:
print 'loaded LivetimeCube %s ' % self.ltcube
return lt
except RuntimeError:
if not self.quiet:
ext = 'WEIGHTED_EXPOSURE' if weighted else 'EXPOSURE'
print(
'no extension %s in file %s: will generate it from the ft2files'
% (ext, self.ltcube))
elif not self.quiet:
print 'LivetimeCube file %s does not exist: will generate it from the ft2 files' % self.ltcube
if self.roi_dir is None:
# no roi specified: use full sky
self.roi_dir = skymaps.SkyDir(0, 0)
self.exp_radius = 180
lt = skymaps.LivetimeCube(cone_angle=self.exp_radius,
dir=self.roi_dir,
zcut=math.cos(math.radians(self.zenithcut)),
pixelsize=pixelsize,
quiet=self.quiet,
weighted=weighted)
for hf in self.ft2files:
if not self.quiet: print 'loading FT2 file %s' % hf,
lt_gti = skymaps.Gti(hf, 'SC_DATA')
if not ((lt_gti.maxValue() < self.gti.minValue()) or
(lt_gti.minValue() > self.gti.maxValue())):
lt.load(hf, gti)
# write out ltcube if requested
if self.ltcube is not None:
extension = 'WEIGHTED_EXPOSURE' if weighted else 'EXPOSURE'
lt.write(self.ltcube, extension, clobber)
return lt
def overlaps(f):
fgti = skymaps.Gti(f)
inrange = lambda t: t >= self.gti.minValue(
) and t <= self.gti.maxValue()
return inrange(fgti.minValue()) or inrange(fgti.maxValue())
