def make_pictures(config_dir='.', image_folder = 'extended_images'): """ make a folder with images for all extended sources """ from uw.like2 import configuration from skymaps import Band if not os.path.exists(image_folder): os.mkdir(image_folder) cf = configuration.Configuration(config_dir, quiet=True, postpone=True) class Bandlite(object): def __init__(self, roi_dir, event_type=1, energy =133.352): self.event_type=event_type self.energy=energy self.sd=roi_dir self.psf=cf.psfman(event_type,energy) self.exposure = cf.exposureman(event_type,energy) self.radius =5 ecat = ExtendedCatalog(cf.extended) for name in ecat.names: print 'processing %s ...' % name , source = ecat.lookup(name) b12 = Band(12); roi_index = b12.index(source.skydir); roi_dir = b12.dir(roi_index) conv = convolution.ExtendedConvolver(source, Bandlite(roi_dir=roi_dir)) conv.create_grid() fig = conv.show_source() fig.savefig('%s/%s_map.png' %(image_folder,name.replace(' ','_'))) print
def __init__(self, table, sources=None, imshow_kw=dict(interpolation='bilinear', ), **kwargs): """table : string or iterable If a string, the name of a pickled file sources : None or a string if a string, the name of a pickled rec with name, ra, dec fields """ if type(table)==types.StringType: self.v = pickle.load(open(table)) print 'Loaded HEALpix table from file %s' %table else: self.v=table self.nside = int(np.sqrt(len(self.v)/12)) assert len(self.v)==12*self.nside**2, 'size of map not consistent with expected nside %d' % nside self.band = Band(self.nside) self.imshow_kw=imshow_kw self.scale = kwargs.pop('scale', lambda x: x) if type(self.scale) == types.StringTypes: if self.scale=='sqrt': self.scale= lambda x: np.sqrt(max(x,0)) elif self.scale=='log': self.scale=lambda x: np.log10(max(x,0.1)) else: raise Exception, 'unrecognized scale function, %s' %self.scale self.ZEA_kw = kwargs.pop('ZEA_kw', dict(galactic=True, size=10, pixelsize=0.1)) if sources is not None: self.sources = pickle.load(open(sources)) print 'loaded %d sources from %s' % (len(self.sources),sources) else:self.sources=None self.map_path = kwargs.pop('map_path',None)
def __init__(self, table_name='kde', index=860, nside=nside, **kwargs): """ table_name : index : nside : """ folder = '%s_table_%d' % (table_name, nside) assert os.path.exists(folder), 'Folder %s not found' % folder self.hmap = pickle.load(open('%s/HP12_%04d.pickle' % (folder, index))) self.index = index self.hpindices = list(make_index_table(12, nside)[index]) self.hpdict = dict((self.hpindices[i], self.hmap[i]) for i in range(len(self.hpindices))) self.center = Band(12).dir(index) self.indexfun = Band(nside).index self.ZEA_kw = kwargs.pop('ZEA_kw', dict(galactic=True, size=10, pixelsize=0.05)) self.imshow_kw = dict(interpolation='bilinear', ) self.scale = kwargs.pop('scale', lambda x: x) if type(self.scale) == types.StringTypes: if self.scale == 'sqrt': self.scale = lambda x: np.sqrt(max(x, 0)) elif self.scale == 'log': self.scale = lambda x: np.log10(max(x, 0.1)) else: raise Exception, 'unrecognized scale function, %s' % self.scale
def check_converge(month, tol=10, add_neighbors=True, log=None): """ check for convergence, ROI that have been updated month: int or string if int, intrepret as a month, else a folder """ from pointlike import IntVector from skymaps import Band outdir = 'month%02d' % month if type(month) == types.IntType else month #print '%s:' %outdir r = roirec(outdir) if r is None: return diff = r.loglike - r.prevlike nisnan = sum(np.isnan(diff)) if nisnan > 0: print 'warning: %d NaN values in likelihoods: ignoring them' % nisnan diff[np.isnan(diff)] = 0 dmin, dmax = diff.min(), diff.max() rmin, rmax = list(diff).index(dmin), list(diff).index(dmax) changed = set(np.arange(1728)[np.abs(diff) > tol]) print >> log, '\tpass %d: %d changed > %d, min, max: %d(#%d) %d(#%d)' % ( max(r.niter), len(changed), tol, dmin, rmin, dmax, rmax), if not add_neighbors: return list(changed) nbrs = set() b12 = Band(12) for x in changed: v = IntVector() b12.findNeighbors(int(x), v) # int is tricky for n in v: nbrs.add(n) q = list(changed.union(nbrs)) print >> log, ' (total %d)' % len(q) if log is not None: log.flush() return q
def __init__(self, other, nside=256): self.name=other.name self.vec = other.vec self.nside =nside self._nside = int(np.sqrt(len(self.vec)/12) ) # internal self.dirfun = Band(self.nside).dir # external direction from index self._indexfun = Band(self._nside).index #internal index from direction
def load(self, filename): d = load(file(filename)) self.inds = d['inds'] self.band1 = Band(d['nside1']) self.band2 = Band(d['nside2']) self.img = d['img'] self.index = d['index'] self.previous_index = -1
def __init__(self, name, skyfun, nside): """ skyfun : function of position """ self.name = name self.skyfun = skyfun self.nside = nside self.dirfun = Band(self.nside).dir self._indexfun = Band(self.nside).index
def average(self, radius=1.0): """ average over all pixels within radius""" iv = IntVector() hp = Healpix(self.nside, Healpix.RING, SkyDir.GALACTIC) band = Band(self.nside) newvec = [0] * len(self.vec) for i in range(len(self.vec)): hp.query_disc(band.dir(i), np.radians(radius), iv) newvec[i] = np.array([self.vec[j] for j in iv]).mean() self.vec = newvec
def __init__(self, outdir, filename, nside=512, fieldname='ts'): full_filename = os.path.join(outdir, filename) band = Band(nside) self.sdir = lambda index: band.dir(index) assert os.path.exists( full_filename), 'file, %s, not found' % full_filename self.rts = pyfits.open(full_filename)[1].data.field(fieldname) assert len(self.rts) == 12 * ( nside)**2, 'wrong nside in file %s: expect %d' % (filename, nside) self.glat = None # np.array([sdir(i).b() for i in range(len(self.rts))])
def __init__(self, nside, inds, vals): self.band = Band(nside) #s = np.argsort(inds) #self.inds = inds[s] #self.vals = vals[s] self.vals = np.empty(12 * nside**2) self.vals[:] = np.nan self.vals[inds] = vals from libpointlike import IntVector self.iv = IntVector()
def smooth(self, a=0.6): """ simple-minded smooth using nearest neighbors """ newvec = [0]*len(self.vec) band = Band(self.nside) iv = IntVector() # needed for findNeighbors: first 4 share sides b=0.25*(1-a) for i in range(len(self.vec)): band.findNeighbors(i,iv) newvec[i] = a*self.vec[i] + b*sum([self.vec[j] for j in iv[:4]]) self.vec= newvec
def __getitem__(self, index): """ return a skymaps.Band C++ object corresponding to the band index This object implements query_disk for extracting the pixels within a given ROI. """ from skymaps import Band b = self.bands[index] bb = Band(int(b.nside), int(b.event_type), b.e_min, b.e_max, 0, 0) # this is an unfortunate loop: need to consider an interface for adding a set of pixels for pix, cnt in self.pixels[index]: bb.add(int(pix), int(cnt)) return bb
def select_seeds_in_roi(roi, fn='seeds/seeds_all.csv'): """ Read seeds from csv file, return those in the given ROI roi : int or Process instance if the latter, look up index from roi direction. direction """ if type(roi) != int: roi = Band(12).index(roi.roi_dir) seeds = pd.read_csv(fn, index_col=0) seeds['skydir'] = map(SkyDir, seeds.ra, seeds.dec) seeds.index.name = 'name' sel = np.array(map(Band(12).index, seeds.skydir)) == roi return seeds[sel]
def fill_zea(self, index, fignum=12, axes=None, show_kw=None, **kwargs): """ index: integer, or a SkyDir the HP12 index if integer figmun: integer used if axes is None show_kw : dict override imshow keywords kwargs size pixelsize galactic """ if axes is None: plt.close(fignum) fig = plt.figure(fignum, figsize=(6, 6)) axes = fig.gca() if type(index) == types.IntType: sdir = Band(12).dir(index) title = 'HP12_%4d' % index else: sdir = index title = 'l = %.1f, b=%.1f' % (sdir.l(), sdir.b()) title = kwargs.pop('title', title) kw = self.ZEA_kw kw.update(kwargs) zea = image.ZEA(sdir, **kw) zea.grid() zea.fill(self.get_pyskyfun()) zea.imshow(**(show_kw if show_kw is not None else self.imshow_kw)) zea.colorbar() if title is not None: axes.set_title(title) if self.sources is not None: count = 0 for s in self.sources: sdir = SkyDir(s.ra, s.dec) if not zea.inside(sdir): continue count += 1 inside = self.band.index(sdir) == index zea.plot_source(s.name, sdir, symbol='*' if inside else 'd', markersize=14 if inside else 8, color='w') print 'found %d sources to plot' % count if self.map_path is not None: fout = os.path.join(self.map_path, hpname(index) + '.png') plt.savefig(fout, bbox_inches='tight') print 'saved figure to %s' % fout plt.draw_if_interactive() return zea
def make_index_table(nside, subnside, usefile=True): filename = 'index_table_%02d_%03d.pickle' % (nside, subnside) if os.path.exists(filename) and usefile: return pickle.load(open(filename)) print 'generating index table for nside, subnside= %d %d' % (nside, subnside) band, subband = Band(nside), Band(subnside) npix, nsubpix = 12 * nside**2, 12 * subnside**2 t = np.array([band.index(subband.dir(i)) for i in xrange(nsubpix)]) a = np.arange(nsubpix) index_table = [a[t == i] for i in xrange(npix)] if usefile: pickle.dump(index_table, open(filename, 'w')) return index_table
def setup_from_roi(self, hr, factor): band1 = hr.band band2 = Band(int(round(band1.nside() * factor))) rd = band1.dir(hr.index) # get pixels within a radius 10% greater than the base Healpix diagonal dimension radius = (np.pi / (3 * band1.nside()**2))**0.5 * 2**0.5 * 1.1 wsdl = WeightedSkyDirList(band2, rd, radius, True) # then cut down to just the pixels inside the base Healpixel inds = np.asarray([band1.index(x) for x in wsdl]) mask = inds == hr.index dirs = [wsdl[i] for i in xrange(len(mask)) if mask[i]] inds = np.asarray([band2.index(x) for x in dirs]).astype(int) # sanity check if abs(float(mask.sum()) / factor**2 - 1) > 0.01: print 'Warning: number of pixels found does not agree with expectations!' # calculate TS for each sub-pixel tsc = TSCalc(hr.roi) if self.bright_sources is not None: ts_vals = self.ts_vals = [deque(), deque(), deque()] bsm = np.asarray([ x.source_name in self.bright_sources for x in hr.roi.psm.models ]) for d in dirs: ts_vals[0].append(tsc(d)) ts_vals[1].append(tsc(d, repeat_diffuse=True)) ts_vals[2].append( tsc(d, repeat_diffuse=True, bright_source_mask=bsm)) else: self.ts_vals = ts_vals = [deque(), deque()] for d in dirs: ts_vals[0].append(tsc(d)) ts_vals[1].append(tsc(d, repeat_diffuse=True)) self.band1 = band1 self.band2 = band2 sorting = np.argsort(inds) self.inds = inds[sorting] for i in range(len(ts_vals)): ts_vals[i] = np.asarray(ts_vals[i])[sorting] self.previous_index = -1 self.index = hr.index self.mode = 0 self.ts = self.ts_vals[0]
def make_index_table(nside=12, subnside=512, usefile=True): """create, and/or use a table to convert between different nside pixelizations """ filename = os.path.expandvars('$FERMI/misc/index_table_%02d_%03d.pickle' % (nside, subnside) ) if os.path.exists(filename) and usefile: return pickle.load(open(filename)) print 'generating index table for nside, subnside= %d %d' % (nside, subnside) band, subband = Band(nside), Band(subnside) npix, nsubpix = 12*nside**2, 12*subnside**2 t=np.array([band.index(subband.dir(i)) for i in xrange(nsubpix)]) a = np.arange(nsubpix) index_table = [a[t==i] for i in xrange(npix)] if usefile: pickle.dump(index_table, open(filename,'w')) return index_table
def load(self, filename): d = load(file(filename)) self.inds = d['inds'] self.band1 = Band(d['nside1']) self.band2 = Band(d['nside2']) if 'ts_vals' in d.keys(): self.ts_vals = d['ts_vals'] else: dim = (self.band2.nside() / self.band1.nside())**2 self.ts_vals = np.empty([3, dim]) self.ts_vals[:] = np.nan self.index = d['index'] self.ts = self.ts_vals[0] self.previous_index = -1 self.mode = 0
def __init__(self, glob_expansion): """glob_expansion: an expression that can be expanded by glob into a list of the pickles of HealpixTSMaps that this class will access N.B. it is necessary that these objects all have been constructed with the same settings of base nside and sub-grid factor! """ tsmaps = [HealpixKDEMap(pickle=x) for x in glob(glob_expansion)] nside1 = tsmaps[0].band1.nside() self.band1 = Band(nside1) self.tsmaps = [False] * 12 * nside1**2 for tsmap in tsmaps: self.tsmaps[tsmap.index] = tsmap self.scale = 0
def pulsar_check(self): """LAT pulsar check %(atable)s """ # compare with LAT pulsar catalog lat = self.lcatdf lat['ts'] = self.df[self.df.psr]['ts'] lat['aprob'] = self.df[self.df.psr]['aprob'] lat['ROI_index'] = [ Band(12).index(SkyDir(float(ra), float(dec))) for ra, dec in zip(lat.RAJ2000, lat.DEJ2000) ] lat['skydir'] = [ SkyDir(float(ra), float(dec)) for ra, dec in zip(lat.RAJ2000, lat.DEJ2000) ] lat['sourcedir'] = self.df.skydir[self.df.psr] lat['delta'] = [ np.degrees(s.difference(t)) if not type(t) == float else np.nan for s, t in zip(lat.skydir, lat.sourcedir) ] far = lat.delta > 0.25 dc2names = set(self.lcatdf.index) tt = set(self.df.name[self.df.psr]) print 'Catalog entries not found:', list(dc2names.difference(tt)) missing = np.array([np.isnan(x) or x < 10. for x in lat.ts]) missing |= np.array((lat.aprob == 0) & (lat.ts < 1000)) missing_names = lat.index[missing] cols = 'RAJ2000 DEJ2000 ts delta ROI_index'.split() self.latsel = latsel = pd.DataFrame(np.array( [lat[id][missing] for id in cols]), index=cols, columns=missing_names).T self.atable = '<h4>Compare with LAT pulsar catalog: {}</h4>'.format( self.version) label_info = dict( ts='TS,Test Statistic', delta='delta,distance to fit position (deg)', ROI_index='ROI Index,Index of the ROI, a HEALPix ring index') self.atable += html_table( latsel.query('ts<10'), label_info, heading='<p>LAT catalog entries with weak or no fit (TS<10)', name=self.plotfolder + '/weak', maxlines=20, float_format=(FloatFormat(2))) self.atable += html_table( latsel.query('ts>10'), label_info, heading= '<p>LAT catalog entries with nearby, but unassociated source ', name=self.plotfolder + '/far', maxlines=20, float_format=(FloatFormat(2)))
def __call__(self, band, nside, index, ps_dir): """Return an array of fractional overlap for a point source at location skydir.""" # pick an nside commensurate with PSF and that will fit evenly within base pixel scale = band.psf.parent_psf.scale_func[band.ct](band.e) / 5 sample_nside = (np.pi / 3)**0.5 / scale p0 = np.log(sample_nside / nside) / np.log(2) last_overlap = None overlaps = [] for addon in [0]: sample_nside = min(8192, nside * 2**(int(p0) + addon)) geo = (4 * np.pi / 12 / sample_nside**2) #print sample_nside,index # temporary code for testing! if True: from skymaps import Band band.b = Band(sample_nside) wsdl = WeightedSkyDirList(band.b, nside, index, True) vals = np.empty(len(wsdl)) band.psf.cpsf.wsdl_val(vals, ps_dir, wsdl) overlap = vals.sum() * geo #print overlap overlaps.append(overlap) #if last_overlap is not None: # new_est = (2*last_overlap+4*overlap)/6 # print 'Updated estimate: ',new_est last_overlap = overlap #print ('%.6f\t'*(len(overlaps)))%(tuple(overlaps)) return overlaps[-1], sample_nside
class FromCCube(HParray): """Manage the gtlike HEALPix counts map """ def __init__(self, filename): """ load array from a HEALPix-format FITS file """ assert os.path.exists(filename), 'File "{}" not found'.format(filename) self.hdus = pyfits.open(filename) self.data = self.hdus[1].data self.nside = self.hdus[1].header['NSIDE'] # expect to find circle in header hdr = self.hdus[0].header try: dstypes = sorted( filter(lambda n: n.startswith('DSTYP'), hdr.keys())) i = [hdr[x] for x in dstypes].index('POS(RA,DEC)') i circ = hdr['DSVAL{}'.format(i)] ra, dec, self.radius = np.array(circ[7:-1].split(','), float) except Exception, msg: print 'failed to parse file {}: expected header to have a DSVAL with POS(RA,DEC)'.format( filename), msg raise self.center = SkyDir(ra, dec) self.indexfun = Band(self.nside).index self.lookup = dict(zip(self.data.PIX, self.data.CHANNEL1))
def load_skyspect(fn = r'T:\data\galprop\ring_21month_P6v11.fits', # r'D:\fermi\data\galprop\gll_iem_v02.fit', nside=192, show_kw = dict(fun=np.log10, cmap='hot'), ): """ load a galactic diffuse distribution. Save the HEALpix respresentation at an energy (1 GeV default) fn : string filename for the FITS representaion of a SKySpectrum nside: int HEALpix nside to use for represenation -- note that 192 is 12*16, about 0.25 deg show_kw : dict fun: weighting function, cmap, vmin, vmax """ t = SkyImage(fn) galname = os.path.split(fn)[-1] print '%s: nx, ny, layers: %d %d %d' %(galname, t.naxis1(), t.naxis2(), t.layers()) hpdir = Band(nside).dir dmap = map(lambda i:t(hpdir(i)), xrange(12*nside**2)) tdm=DisplayMap(dmap) tdm.fill_ait(fignum=12, source_kw=dict(edgecolor='w',), show_kw=show_kw ) plt.title(galname+' (1 GeV)') sfn = galname.split('.')[0]+'.png' plt.savefig(galname.split('.')[0]+'.png', bbox_inches='tight', pad_inches=0) print 'saved figure to %s' % sfn return tdm
def __init__(self, nside=12): """ nside: integer the HEALpix nside parameter """ self.v = np.zeros(12*nside**2, float) self.index = Band(nside).index
def process_band(self, iband, verbose=False): self.select_band(iband) dflux= self.dflux.astype(float) if verbose: m = dflux.mean(axis=2); s = dflux.std(axis=2)/m ids = 'Front Back'.split() if dflux.shape[0]==2 else ['Front'] print 'mean\n', pd.DataFrame(m, index=ids, columns='gal iso'.split()) print 'rms/mean\n',pd.DataFrame(s, index=ids, columns='gal iso'.split()) x=self.fa[iband,:] if verbose: print 'Fits\n', x a64=Band(64).pixelArea() f=(dflux*a64); # iso/gal ratio r = f[:,1,:]/f[:,0,:]; if verbose: print 'ratio\n', r # adjusted fits per (flux, pixel) af = x[0] + r*(x[1]-1); if verbose: print 'adjusted fits\n',af # isolate the galactic flux f_gal = f[:,0]; if verbose: print 'galactic fit\n',f_gal # weighted sum of the adjusted fits by the galactic flux waf = (f_gal*af).sum(axis=0) / f_gal.sum(axis=0); if verbose: print 'weighted fit\n',self.waf return waf
def plot(self, title='', axes=None, fignum=30, ait_kw={}, **kwargs): """ make an AIT skyplot from the array title : string set the figure title ait_kw : dict to set kwargs for image.AIT, perhaps pixelsize Other args passed to imshow, for example norm: to make a log plot, from matplotlib.colors import LogNorm plot(norm=LogNorm(vmin=1,vmax=10)) It returns a image.AIT object, which has a colorbar member that can be adjusted. """ cbtext = kwargs.pop('cbtext', '') ait_kw.update(cbtext=cbtext) band = Band(self.nside) def skyplotfun(v): skydir = SkyDir(Hep3Vector(v[0], v[1], v[2])) return self(skydir) if axes is None: plt.close(fignum) fig = plt.figure(fignum, figsize=(12, 6)) axes = plt.gca() ait = image.AIT(PySkyFunction(skyplotfun), axes=axes, **ait_kw) ait.imshow(title=title, **kwargs) return ait
class MultiHealpixKDEMap(object): def __init__(self, glob_expansion): """glob_expansion: an expression that can be expanded by glob into a list of the pickles of HealpixTSMaps that this class will access N.B. it is necessary that these objects all have been constructed with the same settings of base nside and sub-grid factor! """ tsmaps = [HealpixKDEMap(pickle=x) for x in glob(glob_expansion)] nside1 = tsmaps[0].band1.nside() self.band1 = Band(nside1) self.tsmaps = [False] * 12 * nside1**2 for tsmap in tsmaps: self.tsmaps[tsmap.index] = tsmap self.scale = 0 def __call__(self, v, skydir=None): sd = skydir or SkyDir(Hep3Vector(v[0], v[1], v[2])) id = self.band1.index(sd) tsmap = self.tsmaps[id] if not tsmap: return np.nan return tsmap.multipix_call(sd) def make_zea(self, center, size=10, pixelsize=0.02, galactic=False): z = ZEA(center, size=size, pixelsize=pixelsize, galactic=galactic) z.fill(PySkyFunction(self)) self.z = z
def skyplot(crec, title='', axes=None, fignum=30, ait_kw={}, **kwargs): """ make an AIT skyplot of a HEALpix array crec : array must be sorted according to the HEALpix index title : string set the figure title ait_kw : dict to set kwargs for image.AIT, perhaps pixelsize Other args passed to imshow """ n = len(crec) nside = int(np.sqrt(n/12)) assert n==12*nside**2, 'wrong length to be healpix array' band = Band(nside) def skyplotfun(v): skydir = SkyDir(Hep3Vector(v[0],v[1],v[2])) index = band.index(skydir) return crec[index] if axes is None: plt.close(fignum) fig = plt.figure(fignum, figsize=(12,6)) ait=image.AIT(PySkyFunction(skyplotfun) ,axes=axes, **ait_kw) ait.imshow(title=title, **kwargs) return ait
def __init__(self, outdir, nside, roi_nside=12, **kwargs): self.subdirfun = Band(nside).dir disc_info = kwargs.pop('disc_info', ()) # allow arbitraty disk, rather than just within a ROI active boundary if disc_info: l, b, r = disc_info vec = healpy.dir2vec(l, b, lonlat=True) index_list = healpy.query_disc(nside=nside, vec=vec, radius=np.radians(r), nest=False) self.pos_list = [self.subdirfun(int(i)) for i in index_list] else: self.index_table = make_index_table(roi_nside, nside) self.skyfuns = kwargs.pop( 'skyfuns', ( ( ResidualTS, 'ts', dict(photon_index=2.3), ), (KdeMap, 'kde', dict()), ), ) self.subdirs = [ os.path.join(outdir, name + '_table_%d' % nside) for s, name, kw in self.skyfuns ] for subdir in self.subdirs: if not os.path.exists(subdir): os.makedirs(subdir)
def __init__(self, roi, roi_index=None, diffuse_fits=None, nside=64): """ """ self.roi = roi if roi_index is not None: print "setting up index {}".format(roi_index) roi.setup_roi(roi_index) # Process object external for now else: roi_index = int(roi.name[-4:]) self.ri = roi_index self.pdirs = map(Band(nside).dir, maps.make_index_table(12,nside)[roi_index]) print 'Processing ROI index {}'.format(roi_index) # load diffuse fits for this model if diffuse_fits is None: files = sorted(glob.glob('diffuse_fit/*.pickle')) assert len(files)>0, 'no files found' if len(files)<1728: msg= "found {} files, expected 1728".format(len(files)) print msg raise Exception(msg) # return as an array 1728x8x2, last being (gal,iso) self.fa = (pd.read_pickle(files[roi_index]).values[:,:2]).astype(float) print 'Loaded diffuse fits for this RoI' else: self.fa = diffuse_fits.values[:,:2].astype(float) print 'Using fits just generated'