def add(self, others, output, binfile, ltcube): """Combine this DataSpec instance with another and return the result The two instances must have consistent definitions (DSS, binning, and livetime), and must have non-overlapping Gtis. The binfiles and ltcubes will be combined and written to the provided destinations; perhaps in the future, I will come up with sensible defaults. """ if not hasattr(others, '__iter__'): others = [others] for other in others: exc = self.check_consistency(other) if exc is not None: raise (exc) binfile = os.path.expandvars(binfile) ltcube = os.path.expandvars(ltcube) gti = skymaps.Gti(self.gti) ft1 = self.ft1files ft2 = self.ft2files bpd = skymaps.BinnedPhotonData(self.binfile) for other in others: gti.intersection(other.gti) ft1 += other.ft1files ft2 += other.ft2files bpd.add(skymaps.BinnedPhotonData(other.binfile)) if gti.computeOntime() > 0: raise DataError("DataSpec instances have overlapping GTIs") ft2 = sorted(list(set(ft2))) bpd.write(binfile) fitstools.merge_lt([self.ltcube] + [other.ltcube for other in others], outfile=ltcube, weighted=self.use_weighted_livetime) dssman.DSSEntries(self.binfile, header_key=0).write(binfile, header_key=0) dssman.DSSEntries(self.ltcube, header_key=0).write(ltcube, header_key=0) #TODO: move the copying of DSS entries into the merge_bpd and merge_lt functions gti_mask = skymaps.Gti(self.gti_mask) for other in others: gti_mask.combine(other.gti_mask) kwargs = dict(ft1=ft1, ft2=ft2, binfile=binfile, ltcube=ltcube, gti_mask=gti_mask, binsperdec=self.binsperdec, mc_src_id=self.mc_src_id, mc_energy=self.mc_energy, use_weighted_livetime=self.use_weighted_livetime, livetime_buffer=self.livetime_buffer, livetime_pixelsize=self.livetime_pixelsize, clobber=False) return DataSpec(output, **kwargs)
def _get_GTI(self): """ Apply GTI cuts and get resulting merged GTI.""" # get GTI for a set of FT1 files gti = skymaps.Gti(self.ft1files[0]) for ef in self.ft1files[1:]: gti.combine(skymaps.Gti(ef)) # apply mask if specified if self.gti_mask is not None: before = gti.computeOntime() gti.intersection(self.gti_mask) if not self.quiet: print('applied gti mask, before, after times: {0:.1f}, {1:.1f}' .format(before, gti.computeOntime())) return gti
def _make_ltcube(self): """ Generate the livetime cube.""" #TODO -- unify weighted livetime import sys self.quiet = False self.ltcube = self.ltcube or self._make_default_name(prefix='lt') roi_info = self.dss.roi_info() if hasattr(self.dss, 'roi_info') else None if (roi_info is None) or (roi_info[2] > 90): # full sky ltcube roi_dir = skymaps.SkyDir(0, 0) exp_radius = 180 else: roi_dir = skymaps.SkyDir(roi_info[0], roi_info[1]) exp_radius = roi_info[2] + self.livetime_buffer if self.zenith_cut is None: self.zenith_cut = get_default('ZENITH_ANGLE', None) print 'Warning: using default zenith_cut, {}'.format( self.zenith_cut) # raise DataManException('zenith cut not specified') zenithcut = self.zenith_cut.get_bounds()[1] if not self.quiet: if exp_radius == 180: print 'Constructing all-sky livetime cube' else: print( 'Constructing livetime cube about RA,Dec = ({0:0.3f},{1:0.3f}) with a radius of {2:0.3f} deg.' .format(roi_dir.ra(), roi_dir.dec(), exp_radius)) for i in xrange(1 + self.use_weighted_livetime): #print('on iteration {0}'.format(i)) sys.stdout.flush() lt = skymaps.LivetimeCube(cone_angle=exp_radius, dir=roi_dir, zcut=np.cos(np.radians(zenithcut)), pixelsize=self.livetime_pixelsize, quiet=self.quiet, weighted=i) for hf in self.ft2files: if not self.quiet: print('checking FT2 file {0}...'.format(hf)), lt_gti = skymaps.Gti(hf, 'SC_DATA') if not ((lt_gti.maxValue() < self.gti.minValue()) or (lt_gti.minValue() > self.gti.maxValue())): lt.load(hf, self.gti) if not self.quiet: print 'loaded' else: if not self.quiet: print 'not in Gti range' # write out ltcube extension = 'WEIGHTED_EXPOSURE' if i else 'EXPOSURE' lt.write(self.ltcube, extension, not bool(i)) if self.dss is not None: self.dss.write(self.ltcube, header_key=0) # write some info to livetime file f = pyfits.open(self.ltcube) f[0]._header['RADIUS'] = exp_radius f[0]._header['PIXSIZE'] = self.livetime_pixelsize f.writeto(self.ltcube, overwrite=True) f.close()
def _GTI_setup(self): """Create the GTI object that will be used to filter photon events and to calculate the livetime. If the user has provided ltcube or binfile arguments, the saved GTI will be used. The ltcube GTI takes precedence over the binfile GTI; if both are provided, it is the users responsibility to ensure that the two sets of GTIs are consistent. Generally, one will have run gtmktime on the FT1 file(s) provided to filter out large excursions in rocking angle, and/or make a a cut for when a (small) ROI is too close to the horizon. gti_mask is an optional kwarg to provide an additional GTI object that can, e.g., be used to filter out GRBs. An intersection with the GTIs from the FT1 files and this gti_mask is made. """ if not self.recalcgti: if self.ltcube is not None and os.path.exists(self.ltcube): if not self.quiet: print('Using gti from %s' % (self.ltcube)) return skymaps.Gti(self.ltcube) elif self.binfile is not None and os.path.exists(self.binfile): if not self.quiet: print('Using gti from %s' % (self.binfile)) return skymaps.Gti(self.binfile) if len(self.ft1files) == 0 or not os.path.exists(self.ft1files[0]): raise PixelDataException( 'Cannot process GTI: no ft1 files found, nor %s nor %s' % (self.ltcube, self.binfile)) gti = skymaps.Gti(self.ft1files[0]) # take the union of the GTI in each FT1 file for ef in self.ft1files[1:]: gti.combine(skymaps.Gti(ef)) tmax = self.tstop if self.tstop > 0 else gti.maxValue() gti = gti.applyTimeRangeCut(self.tstart, tmax) #save gti for later use if 'gti_mask' in self.__dict__ and self.gti_mask is not None: before = gti.computeOntime() gti.intersection(self.gti_mask) if not self.quiet: print 'applied gti mask, before, after times: %.1f, %.1f' % ( before, gti.computeOntime()) return gti
def _check_ltcube(self): """ Verify ltcube exists and is consistent with any existing data cuts. """ #if os.path.exists(self.ltcube) and self.legacy : # print('Accepting ltcube without dss checks since legacy specified') # return True if self.clobber or (not os.path.exists(self.ltcube or '')): print 'checking ltcube: failed clobber on %s' % self.ltcube return False # check for presence of important history # eew -- primary header doesn't have info about extensions, so just # open the file and use that handle to check header keys and # extensions. # h = pyfits.getheader(self.ltcube,0) lt = pyfits.open(self.ltcube) try: lt[0].header['RADIUS'] lt[0].header['PIXSIZE'] except KeyError: if not self.quiet: print 'no header info in ltcube?' #pass #return False # check for weighted extension if we are using it if self.use_weighted_livetime: #try: h['WEIGHTED_EXPOSURE'] #except KeyError: return False try: assert (len(lt) > 3) except AssertionError: print 'fail len(lt)>3:%d' % len(lt) return False # # DSS check # dss = dssman.DSSEntries(self.ltcube, header_key=0) if dss is None or len(dss) == 0: if self.legacy: if not self.quiet: print 'Accepting ltcube without DSS info since legacy specified' dss = self.dss else: raise DataManException('DSS found in ltcube %s' % self.ltcube) if dss != self.dss: print 'Failed dss comparison:\n ltcube %s,\n FT1 %s' % (dss, self.dss) return False # # compare GTI with that found in FT1 or binfile # gti = skymaps.Gti(self.ltcube) if (gti.minValue != self.gti.minValue) or (gti.computeOntime() != self.gti.computeOntime()): print 'Failed gti check:\n ltcube: %s \n binfile: %s' % (gti, self.gti) return self.legacy #ignore if legacy, for now if (not self.quiet): print('Verified ltcube {0}'.format(self.ltcube)) return True
def _check_ltcube(self): """ Verify ltcube exists and is consistent with any existing data cuts. """ #if os.path.exists(self.ltcube) and self.legacy : # print('Accepting ltcube without dss checks since legacy specified') # return True ltcubes = glob.glob(self.ltcube) if len(ltcubes)==0: #if self.clobber or (not os.path.exists(self.ltcube or '')): print 'Checking ltcube: will generate new {}'.format(self.ltcube) return False # check for presence of important history # eew -- primary header doesn't have info about extensions, so just # open the file and use that handle to check header keys and # extensions. # h = pyfits.getheader(self.ltcube,0) lt = pyfits.open(ltcubes[0]) # DISABLE this: seems to be normal # try: # lt[0].header['RADIUS']; lt[0].header['PIXSIZE'] # except KeyError: # if not self.quiet: print 'no header info in ltcube?' #pass #return False # check for weighted extension if we are using it if self.use_weighted_livetime: #try: h['WEIGHTED_EXPOSURE'] #except KeyError: return False try: assert(len(lt)>3) except AssertionError: print 'fail len(lt)>3:%d' %len(lt) return False # # DSS check # nodss=False dss = dssman.DSSEntries(ltcubes[0],header_key=0) if dss is None or len(dss)==0: nodss=True if True: # Permamently allow this self.legacy: #if not self.quiet: print 'Accepting ltcube without DSS info since legacy specified' dss=self.dss else: raise DataManException('DSS found in ltcube %s' % ltcubes[0]) if dss != self.dss: print 'Failed dss comparison:\n ltcube %s,\n FT1 %s' % (dss, self.dss) return False # # compare GTI with that found in FT1 or binfile # gti = skymaps.Gti(ltcubes[0]) tdiff = gti.computeOntime() - self.gti.computeOntime() if (gti.minValue()!=self.gti.minValue()) or abs(tdiff)>1: print 'Failed gti check, time difference %.1f:\n ltcube: %s \n binfile: %s' % (tdiff, gti, self.gti) return True #self.legacy #ignore if legacy, for now if (not self.quiet): print('Verified ltcube {} {}'.format(ltcubes[0], '(without DSS)' if nodss else '')) return True
def _process_dataset(self, dataset, binfile=None, interval=None, month=None, quiet=False): """ Parse the dataset as a string lookup key, or a dict interval: string name of a month: sub spec. """ if isinstance(dataset, dict): dataset['event_class_bit'] = dict(source=2, clean=3, extraclean=4)[dataset.get( 'event_class', 'source').lower()] dataset.setdefault('theta_cut', 66.422) self.name = dataset.get('data_name', '(noname)') self.dict_file = 'input dict' return DataSpecification(os.path.expandvars('$FERMI/data'), interval=None, gti_mask=None, **dataset) self.name = dataset folders = ['.'] + glob.glob( os.path.join(os.path.expandvars('$FERMI'), 'data')) for folder in folders: self.dict_file = dict_file = os.path.join(folder, 'dataspec.py') if not os.path.exists(dict_file): continue try: ldict = eval(open(dict_file).read()) except Exception, msg: raise DataSetError('Data dictionary file %s not valid: %s' % (dict_file, msg)) if interval is not None and not (interval.startswith('month_') or interval.startswith('year_')): # note that the underscore is to specify the designated month or year, with their own files try: pyfile = os.path.join(folder, 'intervals.py') idict = eval(open(pyfile).read()) if os.path.exists( pyfile) else Interval() gr = idict[interval] gti_mask = skymaps.Gti([gr[0]], [gr[1]]) if True: #self.verbose: if not quiet: print 'apply gti mask %s, %s' % (interval, gti_mask) except Exception, msg: raise DataSetError('Interval dictionary file %s, key %s, problem: %s'\ % (pyfile, interval, msg))
def _check_binfile(self): """ Verify binfile exists and is consistent with any existing data cuts. Return False if fails any, print message (why not an exception?) """ if self.binfile is None: raise DataManException('No bin file specified') if not os.path.exists(self.binfile): print 'File %s not found' % self.binfile sys.stdout.flush() return False if self.clobber or self.binfile is None: print 'self.clobber or self.binfile is None' sys.stdout.flush() return False # # Check DSS keywords # dss = dssman.DSSEntries(self.binfile, header_key=0) if (dss is None): print("dss is None") sys.stdout.flush() return False if self.dss is None: if not self.quiet: print('Extracting DSS from existing binfile') self.dss = dss if self.dss != dss: print 'File %s Failed DSS keyword check: expected \n%s \nbut found \n%s' % ( self.binfile, self.dss, dss) sys.stdout.flush() return False # # Check GTI # gti = skymaps.Gti(self.binfile) if not self.quiet: print 'GTI from binfile', gti if gti is None: raise DataManException('GTI not found in the binfile %s' % self.binfile) self.gti = gti if (gti.minValue != self.gti.minValue) or (gti.computeOntime() != self.gti.computeOntime()): print 'File %s Failed GTI check: \n expect %s \n found %s' % ( self.binfile, self.gti, gti) return self.legacy # ignore if legacy, for now if (not self.quiet): print('Verified binfile {0}'.format(self.binfile)) return True
def pointlike_ltcube(evfile,scfile,outfile,dcostheta,binsz, zmax, cone_angle,dir,quiet=False): """ Run the pointlike ltcube code. """ gti = skymaps.Gti(evfile) for weighted,clobber in [[False,True],[True,False]]: lt = skymaps.LivetimeCube( cone_angle = cone_angle, dir = dir, zcut = math.cos(math.radians(zmax)), pixelsize = binsz, quiet = quiet, weighted = weighted) lt.load(scfile,gti) extension = 'WEIGHTED_EXPOSURE' if weighted else 'EXPOSURE' lt.write(outfile,extension,clobber) fix_pointlike_ltcube(outfile)
def get_livetime(self, pixelsize=1.0, weighted=False, clobber=True): gti = self.gti if self.ltcube is not None and os.path.exists(self.ltcube): try: lt = skymaps.LivetimeCube(self.ltcube, weighted=weighted) if not self.quiet: print 'loaded LivetimeCube %s ' % self.ltcube return lt except RuntimeError: if not self.quiet: ext = 'WEIGHTED_EXPOSURE' if weighted else 'EXPOSURE' print( 'no extension %s in file %s: will generate it from the ft2files' % (ext, self.ltcube)) elif not self.quiet: print 'LivetimeCube file %s does not exist: will generate it from the ft2 files' % self.ltcube if self.roi_dir is None: # no roi specified: use full sky self.roi_dir = skymaps.SkyDir(0, 0) self.exp_radius = 180 lt = skymaps.LivetimeCube(cone_angle=self.exp_radius, dir=self.roi_dir, zcut=math.cos(math.radians(self.zenithcut)), pixelsize=pixelsize, quiet=self.quiet, weighted=weighted) for hf in self.ft2files: if not self.quiet: print 'loading FT2 file %s' % hf, lt_gti = skymaps.Gti(hf, 'SC_DATA') if not ((lt_gti.maxValue() < self.gti.minValue()) or (lt_gti.minValue() > self.gti.maxValue())): lt.load(hf, gti) # write out ltcube if requested if self.ltcube is not None: extension = 'WEIGHTED_EXPOSURE' if weighted else 'EXPOSURE' lt.write(self.ltcube, extension, clobber) return lt
def overlaps(f): fgti = skymaps.Gti(f) inrange = lambda t: t >= self.gti.minValue( ) and t <= self.gti.maxValue() return inrange(fgti.minValue()) or inrange(fgti.maxValue())