def toy_background(self, mc_source_id=2, seed=None): """ Quick uniform background generation. """ logger.info("Running toy background simulation...") size = 20000 nstar = np.random.poisson(size) #np.random.seed(0) logger.info("Simulating %i background stars..." % nstar) ### # Random points from roi pixels ### idx = np.random.randint(len(self.roi.pixels)-1,size=nstar) ### pix = self.roi.pixels[idx] # Random points drawn from subpixels logger.info("Generating uniform positions...") idx = np.random.randint(0, len(self.subpix) - 1, size=nstar) lon, lat = pix2ang(self.nside_subpixel, self.subpix[idx]) pix = ang2pix(self.nside_pixel, lon, lat) lon, lat = pix2ang(self.nside_pixel, pix) # Single color #mag_1 = 19.05*np.ones(len(pix)) #mag_2 = 19.10*np.ones(len(pix)) # Uniform in color logger.info("Generating uniform CMD...") mag_1 = np.random.uniform(self.config['mag']['min'], self.config['mag']['max'], size=nstar) color = np.random.uniform(self.config['color']['min'], self.config['color']['max'], size=nstar) mag_2 = mag_1 - color # There is probably a better way to do this step without creating the full HEALPix map mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel)) mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse mag_lim_1 = mask[pix] mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel)) mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse mag_lim_2 = mask[pix] #mag_err_1 = 1.0*np.ones(len(pix)) #mag_err_2 = 1.0*np.ones(len(pix)) mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1) mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2) mc_source_id = mc_source_id * numpy.ones(len(mag_1)) select = (mag_lim_1 > mag_1) & (mag_lim_2 > mag_2) hdu = ugali.observation.catalog.makeHDU( self.config, mag_1[select], mag_err_1[select], mag_2[select], mag_err_2[select], lon[select], lat[select], mc_source_id[select]) catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data) return catalog
def toy_background(self,mc_source_id=2,seed=None): """ Quick uniform background generation. """ logger.info("Running toy background simulation...") size = 20000 nstar = np.random.poisson(size) #np.random.seed(0) logger.info("Simulating %i background stars..."%nstar) ### # Random points from roi pixels ### idx = np.random.randint(len(self.roi.pixels)-1,size=nstar) ### pix = self.roi.pixels[idx] # Random points drawn from subpixels logger.info("Generating uniform positions...") idx = np.random.randint(0,len(self.subpix)-1,size=nstar) lon,lat = pix2ang(self.nside_subpixel,self.subpix[idx]) pix = ang2pix(self.nside_pixel, lon, lat) lon,lat = pix2ang(self.nside_pixel,pix) # Single color #mag_1 = 19.05*np.ones(len(pix)) #mag_2 = 19.10*np.ones(len(pix)) # Uniform in color logger.info("Generating uniform CMD...") mag_1 = np.random.uniform(self.config['mag']['min'],self.config['mag']['max'],size=nstar) color = np.random.uniform(self.config['color']['min'],self.config['color']['max'],size=nstar) mag_2 = mag_1 - color # There is probably a better way to do this step without creating the full HEALPix map mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel)) mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse mag_lim_1 = mask[pix] mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel)) mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse mag_lim_2 = mask[pix] #mag_err_1 = 1.0*np.ones(len(pix)) #mag_err_2 = 1.0*np.ones(len(pix)) mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1) mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2) mc_source_id = mc_source_id * numpy.ones(len(mag_1)) select = (mag_lim_1>mag_1)&(mag_lim_2>mag_2) hdu = ugali.observation.catalog.makeHDU(self.config,mag_1[select],mag_err_1[select], mag_2[select],mag_err_2[select], lon[select],lat[select],mc_source_id[select]) catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data) return catalog
def _parse_coords(self,opts): """ Parse target coordinates in various ways... """ # The coordinates are mutually exclusive, so # shouldn't have to worry about over-writing them. if 'coords' in vars(opts): return radius = vars(opts).get('radius',0) gal = None if vars(opts).get('gal') is not None: gal = opts.gal elif vars(opts).get('cel') is not None: gal = cel2gal(*opts.cel) elif vars(opts).get('hpx') is not None: gal = pix2ang(*opts.hpx) if gal is not None: opts.coords = [(gal[0],gal[1],radius)] opts.names = [vars(opts).get('name','')] else: opts.coords = None opts.names = None if vars(opts).get('targets') is not None: opts.names,opts.coords = self.parse_targets(opts.targets) if vars(opts).get('radius') is not None: opts.coords['radius'] = vars(opts).get('radius')
def randomPositions(input, nside_pix, n=1): """ Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates. input is either a (1) full HEALPix mask of booleans, or (2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels. nside_pix is meant to be at coarser resolution than the input mask or catalog object positions so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in. Return the longitude and latitude of the random positions (deg) and the total area (deg^2). """ input = numpy.array(input) if len(input.shape) == 1: if healpy.npix2nside(len(input)) < nside_pix: logger.warning( 'Expected coarser resolution nside_pix in skymap.randomPositions' ) subpix = numpy.nonzero( input )[0] # All the valid pixels in the mask at the NSIDE for the input mask lon, lat = pix2ang(healpy.npix2nside(len(input)), subpix) elif len(input.shape) == 2: lon, lat = input[0], input[1] # All catalog object positions else: logger.warning( 'Unexpected input dimensions for skymap.randomPositions') pix = surveyPixel(lon, lat, nside_pix) # Area with which the random points are thrown area = len(pix) * healpy.nside2pixarea(nside_pix, degrees=True) # Create mask at the coarser resolution mask = numpy.tile(False, healpy.nside2npix(nside_pix)) mask[pix] = True # Estimate the number of points that need to be thrown based off # coverage fraction of the HEALPix mask coverage_fraction = float(numpy.sum(mask)) / len(mask) n_throw = int(n / coverage_fraction) lon, lat = [], [] count = 0 while len(lon) < n: lon_throw = numpy.random.uniform(0., 360., n_throw) lat_throw = numpy.degrees( numpy.arcsin(numpy.random.uniform(-1., 1., n_throw))) pix_throw = ugali.utils.healpix.angToPix(nside_pix, lon_throw, lat_throw) cut = mask[pix_throw].astype(bool) lon = numpy.append(lon, lon_throw[cut]) lat = numpy.append(lat, lat_throw[cut]) count += 1 if count > 10: raise RuntimeError('Too many loops...') return lon[0:n], lat[0:n], area
def _parse_coords(self, opts): """ Parse target coordinates in various ways... """ # The coordinates are mutually exclusive, so # shouldn't have to worry about over-writing them. if 'coords' in vars(opts): return radius = vars(opts).get('radius', 0) gal = None if vars(opts).get('gal') is not None: gal = opts.gal elif vars(opts).get('cel') is not None: gal = cel2gal(*opts.cel) elif vars(opts).get('hpx') is not None: gal = pix2ang(*opts.hpx) if gal is not None: opts.coords = [(gal[0], gal[1], radius)] opts.names = [vars(opts).get('name', '')] else: opts.coords = None opts.names = None if vars(opts).get('targets') is not None: opts.names, opts.coords = self.parse_targets(opts.targets) if vars(opts).get('radius') is not None: opts.coords['radius'] = vars(opts).get('radius')
def randomPositions(input, nside_pix, n=1): """ Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates. nside_pix is meant to be at coarser resolution than the input mask or catalog object positions so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in. Return the longitude and latitude of the random positions and the total area (deg^2). Probably there is a faster algorithm, but limited much more by the simulation and fitting time than by the time it takes to generate random positions within the mask. """ input = numpy.array(input) if len(input.shape) == 1: subpix = numpy.nonzero( input )[0] # All the valid pixels in the mask at the NSIDE for the input mask lon, lat = pix2ang(healpy.npix2nside(len(input)), subpix) elif len(input.shape) == 2: lon, lat = input[0], input[1] # All catalog object positions else: logger.warning( 'Unexpected input dimensions for skymap.randomPositions') pix = surveyPixel(lon, lat, nside_pix) # Area with which the random points are thrown area = len(pix) * healpy.nside2pixarea(nside_pix, degrees=True) lon = [] lat = [] for ii in range(0, n): # Choose an unmasked pixel at random, which is OK because HEALPix is an equal area scheme pix_ii = pix[numpy.random.randint(0, len(pix))] lon_ii, lat_ii = ugali.utils.projector.pixToAng(nside_pix, pix_ii) projector = ugali.utils.projector.Projector(lon_ii, lat_ii) inside = False while not inside: # Apply random offset arcminToDegree = 1 / 60. resolution = arcminToDegree * healpy.nside2resol(nside_pix, arcmin=True) x = 2. * (numpy.random.rand() - 0.5) * resolution # Using factor 2 to be conservative y = 2. * (numpy.random.rand() - 0.5) * resolution lon_candidate, lat_candidate = projector.imageToSphere(x, y) # Make sure that the random position does indeed fall within the randomly selected pixel if ugali.utils.projector.angToPix(nside_pix, lon_candidate, lat_candidate) == pix_ii: inside = True lon.append(lon_candidate) lat.append(lat_candidate) return numpy.array(lon), numpy.array(lat), area
def randomPositions(input, nside_pix, n=1): """ Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates. Parameters: ----------- input : (1) full HEALPix mask of booleans, or (2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels. nside_pix : nside_pix is meant to be at coarser resolution than the input mask or catalog object positions so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in. Returns: -------- lon,lat,area : Return the longitude and latitude of the random positions (deg) and the total area (deg^2). """ input = np.array(input) if len(input.shape) == 1: if hp.npix2nside(len(input)) < nside_pix: logger.warning('Expected coarser resolution nside_pix in skymap.randomPositions') subpix = np.nonzero(input)[0] # All the valid pixels in the mask at the NSIDE for the input mask lon, lat = pix2ang(hp.npix2nside(len(input)), subpix) elif len(input.shape) == 2: lon, lat = input[0], input[1] # All catalog object positions else: logger.warning('Unexpected input dimensions for skymap.randomPositions') pix = surveyPixel(lon, lat, nside_pix) # Area with which the random points are thrown area = len(pix) * hp.nside2pixarea(nside_pix, degrees=True) # Create mask at the coarser resolution mask = np.tile(False, hp.nside2npix(nside_pix)) mask[pix] = True # Estimate the number of points that need to be thrown based off # coverage fraction of the HEALPix mask coverage_fraction = float(np.sum(mask)) / len(mask) n_throw = int(n / coverage_fraction) lon, lat = [], [] count = 0 while len(lon) < n: lon_throw = np.random.uniform(0., 360., n_throw) lat_throw = np.degrees(np.arcsin(np.random.uniform(-1., 1., n_throw))) pix_throw = ugali.utils.healpix.angToPix(nside_pix, lon_throw, lat_throw) cut = mask[pix_throw].astype(bool) lon = np.append(lon, lon_throw[cut]) lat = np.append(lat, lat_throw[cut]) count += 1 if count > 10: raise RuntimeError('Too many loops...') return lon[0:n], lat[0:n], area
def __new__(cls, nside, pixels): # Input array is an already formed ndarray instance # We first cast to be our class type obj = np.asarray(pixels).view(cls) # add the new attribute to the created instance obj._nside = nside obj._pix = pixels obj._lon,obj._lat = pix2ang(nside,pixels) # Finally, we must return the newly created object: return obj
def __new__(cls, nside, pixels): # Input array is an already formed ndarray instance # We first cast to be our class type obj = np.asarray(pixels).view(cls) # add the new attribute to the created instance obj._nside = nside obj._pix = pixels obj._lon, obj._lat = pix2ang(nside, pixels) # Finally, we must return the newly created object: return obj
def parse_targets(filename): """ Load a text file with target coordinates. Returns an array of target locations in Galactic coordinates. File description: [NAME] [LON] [LAT] [RADIUS] [COORD] The values of LON and LAT will depend on COORD: COORD = [GAL | CEL | HPX ], LON = [GLON | RA | NSIDE] LAT = [GLAT | DEC | PIX ] """ base,ext = os.path.splitext(filename) if (ext=='.fits'): import fitsio data = fitsio.read(filename) elif (ext=='.txt'): from numpy.lib import NumpyVersion if NumpyVersion(np.__version__) < '1.14.0': data = np.genfromtxt(filename,names=True,dtype=None) else: data = np.genfromtxt(filename,names=True,dtype=None,encoding=None) #data = np.genfromtxt(filename,unpack=True,usecols=list(range(5)),dtype=object,names=True) elif (ext=='.yaml'): import yaml data = [(k,v['kernel']['lon']['value'],v['kernel']['lat']['value'],0.5,'CEL') for k,v in yaml.load(open(filename)).items()] data = np.rec.fromrecords(data,names=['name','lon','lat','radius','coord']) else: msg = "Unrecognized file type: %s"%filename raise IOError(msg) data = np.atleast_1d(data) data.dtype.names = list(map(str.lower,data.dtype.names)) # Deal with one-line input files #if data.ndim == 1: data = np.array([data]).T names = data['name'] out = data[['lon','lat','radius']].copy() coord = np.char.lower(data['coord']) gal = (coord=='gal') cel = (coord=='cel') hpx = (coord=='hpx') if cel.any(): glon,glat = cel2gal(data['lon'][cel],data['lat'][cel]) out['lon'][cel] = glon out['lat'][cel] = glat if hpx.any(): glon,glat = pix2ang(data['lat'][hpx],data['lon'][hpx]) out['lon'][hpx] = glon out['lat'][hpx] = glat return names,out.view(np.ndarray)
def randomPositions(input, nside_pix, n=1): """ Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates. nside_pix is meant to be at coarser resolution than the input mask or catalog object positions so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in. Return the longitude and latitude of the random positions and the total area (deg^2). Probably there is a faster algorithm, but limited much more by the simulation and fitting time than by the time it takes to generate random positions within the mask. """ input = numpy.array(input) if len(input.shape) == 1: subpix = numpy.nonzero(input)[0] # All the valid pixels in the mask at the NSIDE for the input mask lon, lat = pix2ang(healpy.npix2nside(len(input)), subpix) elif len(input.shape) == 2: lon, lat = input[0], input[1] # All catalog object positions else: logger.warning('Unexpected input dimensions for skymap.randomPositions') pix = surveyPixel(lon, lat, nside_pix) # Area with which the random points are thrown area = len(pix) * healpy.nside2pixarea(nside_pix, degrees=True) lon = [] lat = [] for ii in range(0, n): # Choose an unmasked pixel at random, which is OK because HEALPix is an equal area scheme pix_ii = pix[numpy.random.randint(0, len(pix))] lon_ii, lat_ii = ugali.utils.projector.pixToAng(nside_pix, pix_ii) projector = ugali.utils.projector.Projector(lon_ii, lat_ii) inside = False while not inside: # Apply random offset arcminToDegree = 1 / 60. resolution = arcminToDegree * healpy.nside2resol(nside_pix, arcmin=True) x = 2. * (numpy.random.rand() - 0.5) * resolution # Using factor 2 to be conservative y = 2. * (numpy.random.rand() - 0.5) * resolution lon_candidate, lat_candidate = projector.imageToSphere(x, y) # Make sure that the random position does indeed fall within the randomly selected pixel if ugali.utils.projector.angToPix(nside_pix, lon_candidate, lat_candidate) == pix_ii: inside = True lon.append(lon_candidate) lat.append(lat_candidate) return numpy.array(lon), numpy.array(lat), area
def parse_targets(filename): """ Load a text file with target coordinates. Returns an array of target locations in Galactic coordinates. File description: [NAME] [LON] [LAT] [RADIUS] [COORD] The values of LON and LAT will depend on COORD: COORD = [GAL | CEL | HPX ], LON = [GLON | RA | NSIDE] LAT = [GLAT | DEC | PIX ] """ base, ext = os.path.splitext(filename) if (ext == '.fits'): import fitsio data = fitsio.read(filename) else: from numpy.lib import NumpyVersion if NumpyVersion(np.__version__) < '1.14.0': data = np.genfromtxt(filename, names=True, dtype=None) else: data = np.genfromtxt(filename, names=True, dtype=None, encoding=None) #data = np.genfromtxt(filename,unpack=True,usecols=list(range(5)),dtype=object,names=True) data = np.atleast_1d(data) data.dtype.names = list(map(str.lower, data.dtype.names)) # Deal with one-line input files #if data.ndim == 1: data = np.array([data]).T names = data['name'] out = data[['lon', 'lat', 'radius']].copy() coord = np.char.lower(data['coord']) gal = (coord == 'gal') cel = (coord == 'cel') hpx = (coord == 'hpx') if cel.any(): glon, glat = cel2gal(data['lon'][cel], data['lat'][cel]) out['lon'][cel] = glon out['lat'][cel] = glat if hpx.any(): glon, glat = pix2ang(data['lat'][hpx], data['lon'][hpx]) out['lon'][hpx] = glon out['lat'][hpx] = glat return names, out.view(np.ndarray)
def draw_maglim_pixel(skymap,**kwargs): nside = healpy.npix2nside(len(skymap)) pix = np.where(skymap > 0) if len(pix[0]) == 0: logger.warn("No maglims found") return ra,dec = pix2ang(nside,pix) ra_center,dec_center = np.median(ra),np.median(dec) vmin,vmax = maglim_range(skymap) kwargs.setdefault('rot',(ra_center, dec_center, 0.)) kwargs.setdefault('min',vmin) kwargs.setdefault('max',vmax) healpy.gnomview(skymap,**kwargs)
def calc_surface_intensity(self, factor=10): """Calculate the surface intensity for each pixel in the interior region of the ROI. Pixels are adaptively subsampled around the kernel centroid out to a radius of 'factor * max_pixrad'. Parameters: ----------- factor : the radius of the oversample region in units of max_pixrad Returns: -------- surface_intensity : the surface intensity at each pixel """ # First we calculate the surface intensity at native resolution pixels = self.roi.pixels_interior nside_in = self.config['coords']['nside_pixel'] surface_intensity = self.kernel.pdf(pixels.lon, pixels.lat) # Then we recalculate the surface intensity around the kernel # centroid at higher resolution for i in np.arange(1, 5): # Select pixels within the region of interest nside_out = 2**i * nside_in radius = factor * np.degrees(hp.max_pixrad(nside_out)) pix = ang2disc(nside_in, self.kernel.lon, self.kernel.lat, radius, inclusive=True) # Select pix within the interior region of the ROI idx = ugali.utils.healpix.index_pix_in_pixels(pix, pixels) pix = pix[(idx >= 0)] idx = idx[(idx >= 0)] # Reset the surface intensity for the subsampled pixels subpix = ugali.utils.healpix.ud_grade_ipix(pix, nside_in, nside_out) pix_lon, pix_lat = pix2ang(nside_out, subpix) surface_intensity[idx] = np.mean(self.kernel.pdf(pix_lon, pix_lat), axis=1) return surface_intensity
def rms_photometry(catfile,nside=64,band=None,plot=False): """ Calculate photometric repeatability """ if not os.path.exists(catfile): msg = "Couldn't find %s"%catfile raise Exception(msg) columns = ['RA','DEC'] spread,nepochs = bfields(['WAVG_SPREAD_MODEL','NEPOCHS'],band) mag,magerr,magrms = bfields(['WAVG_MAG_PSF','WAVG_MAGERR_PSF','WAVG_MAGRMS_PSF'],band) columns += [spread, nepochs, mag, magerr, magrms] # Hack to get pixel location hpx = int(catfile.split('_')[-1].split('.')[0]) #hpx = ang2pix(NSIDE, cat['RA'], cat['DEC']) ra,dec = pix2ang(NSIDE, hpx) msg = '%s (RA,DEC) = %.2f,%.2f'%(os.path.basename(catfile),ra,dec) print(msg) #print "Getting coadd catalog: DES" cat = load_infiles([catfile],columns) # Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5 sel = (np.fabs(cat[spread]) < 0.002) & \ (cat[mag] > 16) & (cat[mag] < 18) &\ (cat[magrms] < 90) &\ (cat[nepochs] > 1) cat = cat[sel] if len(cat) == 0: msg = "WARNING: No objects passing selection in: %s"%catfile print(msg) return np.array([],dtype=int), np.array([]) pix = ang2pix(nside,cat['RA'],cat['DEC']) upix = np.unique(pix) stat = nd.median(cat[magrms],labels=pix,index=upix) if False: plt.figure() plt.hist(cat[magrms],bins=50) import pdb; pdb.set_trace() return upix,stat
def parse_targets(filename): """ Load a text file with target coordinates. Returns an array of target locations in Galactic coordinates. File description: [NAME] [LON] [LAT] [RADIUS] [COORD] The values of LON and LAT will depend on COORD: COORD = [GAL | CEL | HPX ], LON = [GLON | RA | NSIDE] LAT = [GLAT | DEC | PIX ] """ base,ext = os.path.splitext(filename) if ext == '.fits': f = pyfits.open(filename) data = f[1].data.view(np.recarray) data = recfuncs.append_fields(data,'RADIUS',np.zeros(len(data)),usemask=False) return data['NAME'],data[['GLON','GLAT','RADIUS']] elif (ext=='.txt') or (ext=='.dat'): data = np.loadtxt(filename,unpack=True,usecols=list(range(5)),dtype=object) # Deal with one-line input files if data.ndim == 1: data = np.array([data]).T names = data[0] out = data[1:4].astype(float) lon,lat,radius = out coord = np.array([s.lower() for s in data[4]]) gal = (coord=='gal') cel = (coord=='cel') hpx = (coord=='hpx') if cel.any(): glon,glat = cel2gal(lon[cel],lat[cel]) out[0][cel] = glon out[1][cel] = glat if hpx.any(): glon,glat = pix2ang(lat[hpx],lon[hpx]) out[0][hpx] = glon out[1][hpx] = glat return names,out.T
def parse_targets(filename): """ Load a text file with target coordinates. Returns an array of target locations in Galactic coordinates. File description: [NAME] [LON] [LAT] [RADIUS] [COORD] The values of LON and LAT will depend on COORD: COORD = [GAL | CEL | HPX ], LON = [GLON | RA | NSIDE] LAT = [GLAT | DEC | PIX ] """ base,ext = os.path.splitext(filename) if ext == '.fits': f = pyfits.open(filename) data = f[1].data.view(np.recarray) data = recfuncs.append_fields(data,'RADIUS',np.zeros(len(data)),usemask=False) return data['NAME'],data[['GLON','GLAT','RADIUS']] elif (ext=='.txt') or (ext=='.dat'): data = np.loadtxt(filename,unpack=True,usecols=range(5),dtype=object) # Deal with one-line input files if data.ndim == 1: data = np.array([data]).T names = data[0] out = data[1:4].astype(float) lon,lat,radius = out coord = np.array([s.lower() for s in data[4]]) gal = (coord=='gal') cel = (coord=='cel') hpx = (coord=='hpx') if cel.any(): glon,glat = cel2gal(lon[cel],lat[cel]) out[0][cel] = glon out[1][cel] = glat if hpx.any(): glon,glat = pix2ang(lat[hpx],lon[hpx]) out[0][hpx] = glon out[1][hpx] = glat return names,out.T
def coarseFootprint(input, nside_pix): """ Generate a coarse healpix mask of booleans from a finer healpix mask or a set of (lon, lat) coordinates. Parameters: ----------- input : (1) full HEALPix mask of booleans, or (2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels. nside_pix : nside_pix is meant to be at coarser (or equivalent) resolution than the input mask or catalog object positions so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in. Returns: -------- lon,lat,area : Return the longitude and latitude of the random positions (deg) and the total area (deg^2). """ input = np.array(input) if len(input.shape) == 1: if hp.npix2nside(len(input)) < nside_pix: logger.warning( 'Expected coarser resolution nside_pix in skymap.randomPositions' ) subpix = np.nonzero( input )[0] # All the valid pixels in the mask at the NSIDE for the input mask lon, lat = pix2ang(hp.npix2nside(len(input)), subpix) elif len(input.shape) == 2: lon, lat = input[0], input[1] # All catalog object positions else: logger.warning( 'Unexpected input dimensions for skymap.randomPositions') pix = surveyPixel(lon, lat, nside_pix) # Area with which the random points are thrown area = len(pix) * hp.nside2pixarea(nside_pix, degrees=True) # Create mask at the coarser resolution mask = np.tile(False, hp.nside2npix(nside_pix)) mask[pix] = True return mask
def calc_surface_intensity(self, factor=10): """Calculate the surface intensity for each pixel in the interior region of the ROI. Pixels are adaptively subsampled around the kernel centroid out to a radius of 'factor * max_pixrad'. Parameters: ----------- factor : the radius of the oversample region in units of max_pixrad Returns: -------- surface_intensity : the surface intensity at each pixel """ # First we calculate the surface intensity at native resolution pixels = self.roi.pixels_interior nside_in = self.config['coords']['nside_pixel'] surface_intensity = self.kernel.pdf(pixels.lon,pixels.lat) # Then we recalculate the surface intensity around the kernel # centroid at higher resolution for i in np.arange(1,5): # Select pixels within the region of interest nside_out = 2**i * nside_in radius = factor*np.degrees(hp.max_pixrad(nside_out)) pix = ang2disc(nside_in,self.kernel.lon,self.kernel.lat, radius,inclusive=True) # Select pix within the interior region of the ROI idx = ugali.utils.healpix.index_pix_in_pixels(pix,pixels) pix = pix[(idx >= 0)]; idx = idx[(idx >= 0)] # Reset the surface intensity for the subsampled pixels subpix = ugali.utils.healpix.ud_grade_ipix(pix,nside_in,nside_out) pix_lon,pix_lat = pix2ang(nside_out,subpix) surface_intensity[idx]=np.mean(self.kernel.pdf(pix_lon,pix_lat),axis=1) return surface_intensity
def allSkyCoordinates(nside): """ Generate a set of coordinates at the centers of pixels of resolutions nside across the full sky. """ lon,lat = pix2ang(nside, np.arange(0, hp.nside2npix(nside))) return lon, lat
def findObjects(pixels, values, nside, zvalues, rev, good): """ Characterize labelled candidates in a multi-dimensional HEALPix map. Parameters: values : (Sparse) HEALPix array of data values nside : HEALPix dimensionality pixels : Pixel values associated to (sparse) HEALPix array zvalues : Values of the z-dimension (usually distance modulus) rev : Reverse indices for pixels in each "island" good : Array containg labels for each "island" Returns: objs : numpy.recarray of object characteristics """ ngood = len(good) objs = numpy.recarray((ngood,), dtype=[('LABEL','i4'), ('NPIX','i4'), ('VAL_MAX','f4'), ('IDX_MAX','i4'), ('ZIDX_MAX','i4'), ('PIX_MAX','i4'), ('X_MAX','f4'), ('Y_MAX','f4'), ('Z_MAX','f4'), ('X_CENT','f4'), ('Y_CENT','f4'), ('Z_CENT','f4'), ('X_BARY','f4'), ('Y_BARY','f4'), ('Z_BARY','f4'), ('CUT','i2'),]) objs['CUT'][:] = 0 shape = values.shape ncol = shape[1] for i in range(0,ngood): logger.debug("i=%i",i) # This code could use some cleanup... indices=rev[rev[good[i]]:rev[good[i]+1]] npix = len(indices) idx = indices // ncol # This is the spatial index zidx = indices % ncol # This is the distance index pix = pixels[idx] # This is the healpix pixel xval,yval = pix2ang(nside, pix) zval = zvalues[zidx] objs[i]['LABEL'] = good[i] objs[i]['NPIX'] = npix logger.debug("LABEL=%i"%objs[i]['LABEL']) logger.debug("NPIX=%i"%objs[i]['NPIX']) island = values[idx,zidx] idxmax = island.argmax() xval_max,yval_max,zval_max = xval[idxmax],yval[idxmax],zval[idxmax] objs[i]['VAL_MAX'] = island[idxmax] objs[i]['IDX_MAX'] = idx[idxmax] objs[i]['ZIDX_MAX'] = zidx[idxmax] objs[i]['PIX_MAX'] = pix[idxmax] objs[i]['X_MAX'] = xval_max objs[i]['Y_MAX'] = yval_max objs[i]['Z_MAX'] = zval_max proj = Projector(xval_max,yval_max) xpix,ypix = proj.sphereToImage(xval,yval) # Projected centroid x_cent,y_cent,zval_cent = numpy.average([xpix,ypix,zval],axis=1) xval_cent, yval_cent = proj.imageToSphere(x_cent,y_cent) objs[i]['X_CENT'] = xval_cent objs[i]['Y_CENT'] = yval_cent objs[i]['Z_CENT'] = zval_cent # Projected barycenter weights=[island,island,island] x_bary,y_bary,zval_bary = numpy.average([xpix,ypix,zval],weights=weights,axis=1) xval_bary,yval_bary = proj.imageToSphere(x_bary, y_bary) objs[i]['X_BARY'] = xval_bary objs[i]['Y_BARY'] = yval_bary objs[i]['Z_BARY'] = zval_bary return objs
def submit(self, pixels, queue=None, debug=False, configfile=None): """ Submit the likelihood job for the given pixel(s). """ # For backwards compatibility batch = self.config['scan'].get('batch', self.config['batch']) queue = batch.get('default', 'medium') if queue is None else queue # Need to develop some way to take command line arguments... self.batch = ugali.utils.batch.batchFactory(queue, **batch.get(queue, {})) self.batch.max_jobs = self.config['scan'].get('max_jobs', 200) if np.isscalar(pixels): pixels = np.array([pixels]) outdir = mkdir(self.config['output']['likedir']) logdir = mkdir(join(outdir, 'log')) subdir = mkdir(join(outdir, 'sub')) # Save the current configuation settings; avoid writing # file multiple times if configfile passed as argument. if configfile is None: shutil.copy(self.config.filename, outdir) configfile = join(outdir, os.path.basename(self.config.filename)) lon, lat = pix2ang(self.nside_likelihood, pixels) commands = [] chunk = self.config['scan'].get('chunk', 25) istart = 0 logger.info('=== Submit Likelihood ===') for ii, pix in enumerate(pixels): msg = ' (%i/%i) pixel=%i nside=%i; (lon, lat) = (%.2f, %.2f)' msg = msg % (ii + 1, len(pixels), pix, self.nside_likelihood, lon[ii], lat[ii]) logger.info(msg) # Create outfile name outfile = self.config.likefile % ( pix, self.config['coords']['coordsys'].lower()) outbase = os.path.basename(outfile) jobname = batch.get('jobname', 'ugali') # Submission command sub = not os.path.exists(outfile) cmd = self.command(outfile, configfile, pix) commands.append([ii, cmd, lon[ii], lat[ii], sub]) if chunk == 0: # No chunking command = cmd submit = sub logfile = join(logdir, os.path.splitext(outbase)[0] + '.log') elif (len(commands) % chunk == 0) or (ii + 1 == len(pixels)): # End of chunk, create submission script commands = np.array(commands, dtype=object) istart, iend = commands[0][0], commands[-1][0] subfile = join(subdir, 'submit_%08i_%08i.sh' % (istart, iend)) logfile = join(logdir, 'submit_%08i_%08i.log' % (istart, iend)) command = "sh %s" % subfile submit = np.any(commands[:, -1]) if submit: self.write_script(subfile, commands) else: # Not end of chunk continue commands = [] # Actual job submission if not submit: logger.info(self.skip) continue else: job = self.batch.submit(command, jobname, logfile) logger.info(" " + job) time.sleep(0.5)
def findObjects(pixels, values, nside, zvalues, rev, good): """ Characterize labelled candidates in a multi-dimensional HEALPix map. Parameters: values : (Sparse) HEALPix array of data values nside : HEALPix dimensionality pixels : Pixel values associated to (sparse) HEALPix array zvalues : Values of the z-dimension (usually distance modulus) rev : Reverse indices for pixels in each "island" good : Array containg labels for each "island" Returns: objs : numpy.recarray of object characteristics """ ngood = len(good) objs = numpy.recarray((ngood, ), dtype=[ ('LABEL', 'i4'), ('NPIX', 'i4'), ('VAL_MAX', 'f4'), ('IDX_MAX', 'i4'), ('ZIDX_MAX', 'i4'), ('PIX_MAX', 'i4'), ('X_MAX', 'f4'), ('Y_MAX', 'f4'), ('Z_MAX', 'f4'), ('X_CENT', 'f4'), ('Y_CENT', 'f4'), ('Z_CENT', 'f4'), ('X_BARY', 'f4'), ('Y_BARY', 'f4'), ('Z_BARY', 'f4'), ('CUT', 'i2'), ]) objs['CUT'][:] = 0 shape = values.shape ncol = shape[1] for i in range(0, ngood): logger.debug("i=%i", i) # This code could use some cleanup... indices = rev[rev[good[i]]:rev[good[i] + 1]] npix = len(indices) idx = indices // ncol # This is the spatial index zidx = indices % ncol # This is the distance index pix = pixels[idx] # This is the healpix pixel xval, yval = pix2ang(nside, pix) zval = zvalues[zidx] objs[i]['LABEL'] = good[i] objs[i]['NPIX'] = npix logger.debug("LABEL=%i" % objs[i]['LABEL']) logger.debug("NPIX=%i" % objs[i]['NPIX']) island = values[idx, zidx] idxmax = island.argmax() xval_max, yval_max, zval_max = xval[idxmax], yval[idxmax], zval[ idxmax] objs[i]['VAL_MAX'] = island[idxmax] objs[i]['IDX_MAX'] = idx[idxmax] objs[i]['ZIDX_MAX'] = zidx[idxmax] objs[i]['PIX_MAX'] = pix[idxmax] objs[i]['X_MAX'] = xval_max objs[i]['Y_MAX'] = yval_max objs[i]['Z_MAX'] = zval_max proj = Projector(xval_max, yval_max) xpix, ypix = proj.sphereToImage(xval, yval) # Projected centroid x_cent, y_cent, zval_cent = numpy.average([xpix, ypix, zval], axis=1) xval_cent, yval_cent = proj.imageToSphere(x_cent, y_cent) objs[i]['X_CENT'] = xval_cent objs[i]['Y_CENT'] = yval_cent objs[i]['Z_CENT'] = zval_cent # Projected barycenter weights = [island, island, island] x_bary, y_bary, zval_bary = numpy.average([xpix, ypix, zval], weights=weights, axis=1) xval_bary, yval_bary = proj.imageToSphere(x_bary, y_bary) objs[i]['X_BARY'] = xval_bary objs[i]['Y_BARY'] = yval_bary objs[i]['Z_BARY'] = zval_bary return objs
def background(self, mc_source_id=2, seed=None): """ Create a simulation of the background stellar population. Because some stars have been clipped to generate the CMD, this function tends to slightly underestimate (~1%) the background as compared to the true catalog. The simulation of background object colors relies on the data-derived CMD. As such, it is a binned random generator and thus has some fundamental limitations. - The expected number of counts per bin is drawn ra There are a few limitations of this procedure: - Colors are drawn from the CMD of the background annulus - The number of stars per CMD bin is randomized according to the CMD - The colors/mags are then uniformly distributed within the bin - This leads to trouble with large bins when the cloud-in-cells algorithm is applied to the simulated data - The positions are chosen randomly over the spherical cap of the ROI - Objects that are outside of the WARNING: The cloud-in-cells method of generating the CMD leads to some difficulties since it disperses objects from high-density zones to low density zones. - Magnitudes are not randomized according to their errors """ if seed is not None: np.random.seed(seed) self._setup_cmd() # Randomize the number of stars per bin according to Poisson distribution nstar_per_bin = numpy.random.poisson(lam=self.bkg_lambda) nstar = nstar_per_bin.sum() logger.info("Simulating %i background stars..." % nstar) if not self.config['simulate'].get('uniform'): logger.info("Generating colors from background CMD.") # Distribute the stars within each CMD bin delta_color = self.bkg_centers_color[1] - self.bkg_centers_color[0] delta_mag = self.bkg_centers_mag[1] - self.bkg_centers_mag[0] # Distribute points within each color-mag bins xx, yy = np.meshgrid(self.bkg_centers_color, self.bkg_centers_mag) color = numpy.repeat(xx.flatten(), repeats=nstar_per_bin.flatten()) color += numpy.random.uniform(-delta_color / 2., delta_color / 2., size=nstar) mag_1 = numpy.repeat(yy.flatten(), repeats=nstar_per_bin.flatten()) mag_1 += numpy.random.uniform(-delta_mag / 2., delta_mag / 2., size=nstar) else: # Uniform color-magnitude distribution logger.info("Generating uniform CMD.") mag_1 = np.random.uniform(self.config['mag']['min'], self.config['mag']['max'], size=nstar) color = np.random.uniform(self.config['color']['min'], self.config['color']['max'], size=nstar) mag_2 = mag_1 - color # Random points drawn from healpix subpixels logger.info("Generating uniform positions...") idx = np.random.randint(0, len(self.subpix) - 1, size=nstar) lon, lat = pix2ang(self.nside_subpixel, self.subpix[idx]) nside_pixel = self.nside_pixel pix = ang2pix(nside_pixel, lon, lat) # There is probably a better way to do this step without creating the full HEALPix map mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel)) mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse mag_lim_1 = mask[pix] mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel)) mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse mag_lim_2 = mask[pix] mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1) mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2) mc_source_id = mc_source_id * numpy.ones(len(mag_1)) # ADW: Should magnitudes be randomized by the erros? #mag_1 += (numpy.random.normal(size=len(mag_1)) * mag_err_1) #mag_2 += (numpy.random.normal(size=len(mag_2)) * mag_err_2) select = (mag_lim_1 > mag_1) & (mag_lim_2 > mag_2) ### # Make sure objects lie within the original cmd (should be done later...) ### select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, color, mag_1, ### self.roi.bins_color, self.roi.bins_mag) > 0) logger.info("Clipping %i simulated background stars..." % (~select).sum()) hdu = ugali.observation.catalog.makeHDU( self.config, mag_1[select], mag_err_1[select], mag_2[select], mag_err_2[select], lon[select], lat[select], mc_source_id[select]) catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data) return catalog
def calculate(self, infile, field=1, simple=False): logger.info("Calculating magnitude limit from %s"%infile) #manglefile = self.config['mangle']['infile_%i'%field] #footfile = self.config['data']['footprint'] #try: # footprint = fitsio.read(footfile)['I'].ravel() #except: # logger.warn("Couldn't open %s; will try again."%footfile) # footprint = footfile mag_column = self.config['catalog']['mag_%i_field'%field] magerr_column = self.config['catalog']['mag_err_%i_field'%field] # For simple maglims release = self.config['data']['release'].lower() band = self.config['catalog']['mag_%i_band'%field] pixel_pix_name = 'PIX%i'%self.nside_pixel data = fitsio.read(infile,columns=[pixel_pix_name]) #mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int') mask_maglims = np.zeros(hp.nside2npix(self.nside_mask)) out_pixels = np.zeros(0,dtype='int') out_maglims = np.zeros(0) # Find the objects in each pixel pixel_pix = data[pixel_pix_name] mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask) count = Counter(mask_pix) pixels = sorted(count.keys()) pix_digi = np.digitize(mask_pix,pixels).argsort() idx = 0 min_num = 500 signal_to_noise = 10. magerr_lim = 1/signal_to_noise for pix in pixels: # Calculate the magnitude limit in each pixel num = count[pix] objs = data[pix_digi[idx:idx+num]] idx += num if simple: # Set constant magnitude limits logger.debug("Simple magnitude limit for %s"%infile) mask_maglims[pix] = MAGLIMS[release][band] elif num < min_num: logger.info('Found <%i objects in pixel %i'%(min_num,pix)) mask_maglims[pix] = 0 else: mag = objs[mag_column] magerr = objs[magerr_column] # Estimate the magnitude limit as suggested by: # https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document # (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc) maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)]) # Alternative method to estimate the magnitude limit by fitting median #mag_min, mag_max = mag.min(),mag.max() #mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086? #x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins) #x,y = x[~np.isnan(y)],y[~np.isnan(y)] #magerr_med = interp1d(x,y) #mag0 = np.median(x) #maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False) # Median from just objects near magerr cut mask_maglims[pix] = maglim logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix])) subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel) maglims = np.zeros(len(subpix)) + mask_maglims[pix] out_pixels = np.append(out_pixels,subpix) out_maglims = np.append(out_maglims,maglims) # Remove empty pixels logger.info("Removing empty pixels") idx = np.nonzero(out_maglims > 0)[0] out_pixels = out_pixels[idx] out_maglims = out_maglims[idx] # Remove pixels outside the footprint if self.footfile: logger.info("Checking footprint against %s"%self.footfile) glon,glat = pix2ang(self.nside_pixel,out_pixels) ra,dec = gal2cel(glon,glat) footprint = inFootprint(self.footprint,ra,dec) idx = np.nonzero(footprint)[0] out_pixels = out_pixels[idx] out_maglims = out_maglims[idx] logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims))) return out_pixels,out_maglims
def calculate(self, infile, field=1, simple=False): logger.info("Calculating magnitude limit from %s"%infile) #manglefile = self.config['mangle']['infile_%i'%field] footfile = self.config['data']['footprint'] mag_column = self.config['catalog']['mag_%i_field'%field] magerr_column = self.config['catalog']['mag_err_%i_field'%field] # For simple maglims release = self.config['data']['release'].lower() band = self.config['catalog']['mag_%i_band'%field] f = pyfits.open(infile) header = f[1].header data = f[1].data #mask_pixels = numpy.arange( healpy.nside2npix(self.nside_mask), dtype='int') mask_maglims = numpy.zeros( healpy.nside2npix(self.nside_mask) ) out_pixels = numpy.zeros(0,dtype='int') out_maglims = numpy.zeros(0) # Find the objects in each pixel pixel_pix = data['PIX%i'%self.nside_pixel] mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask) count = Counter(mask_pix) pixels = sorted(count.keys()) pix_digi = numpy.digitize(mask_pix,pixels).argsort() idx = 0 min_num = 500 signal_to_noise = 10. magerr_lim = 1/signal_to_noise for pix in pixels: # Calculate the magnitude limit in each pixel num = count[pix] objs = data[pix_digi[idx:idx+num]] idx += num if simple: # Set constant magnitude limits logger.debug("Simple magnitude limit for %s"%infile) mask_maglims[pix] = MAGLIMS[release][band] elif num < min_num: logger.info('Found <%i objects in pixel %i'%(min_num,pix)) mask_maglims[pix] = 0 else: mag = objs[mag_column] magerr = objs[magerr_column] # Estimate the magnitude limit as suggested by: # https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document # (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc) maglim = numpy.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)]) # Alternative method to estimate the magnitude limit by fitting median #mag_min, mag_max = mag.min(),mag.max() #mag_bins = numpy.arange(mag_min,mag_max,0.1) #0.1086? #x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins) #x,y = x[~numpy.isnan(y)],y[~numpy.isnan(y)] #magerr_med = interp1d(x,y) #mag0 = numpy.median(x) #maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False) # Median from just objects near magerr cut mask_maglims[pix] = maglim logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix])) subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel) maglims = numpy.zeros(len(subpix)) + mask_maglims[pix] out_pixels = numpy.append(out_pixels,subpix) out_maglims = numpy.append(out_maglims,maglims) # Remove empty pixels logger.info("Removing empty pixels") idx = numpy.nonzero(out_maglims > 0)[0] out_pixels = out_pixels[idx] out_maglims = out_maglims[idx] # Remove pixels outside the footprint logger.info("Checking footprint against %s"%footfile) glon,glat = pix2ang(self.nside_pixel,out_pixels) ra,dec = gal2cel(glon,glat) footprint = inFootprint(footfile,ra,dec) idx = numpy.nonzero(footprint)[0] out_pixels = out_pixels[idx] out_maglims = out_maglims[idx] logger.info("MAGLIM = %.3f +/- %.3f"%(numpy.mean(out_maglims),numpy.std(out_maglims))) return out_pixels,out_maglims
def background(self,mc_source_id=2,seed=None): """ Create a simulation of the background stellar population. Because some stars have been clipped to generate the CMD, this function tends to slightly underestimate (~1%) the background as compared to the true catalog. The simulation of background object colors relies on the data-derived CMD. As such, it is a binned random generator and thus has some fundamental limitations. - The expected number of counts per bin is drawn ra There are a few limitations of this procedure: - Colors are drawn from the CMD of the background annulus - The number of stars per CMD bin is randomized according to the CMD - The colors/mags are then uniformly distributed within the bin - This leads to trouble with large bins when the cloud-in-cells algorithm is applied to the simulated data - The positions are chosen randomly over the spherical cap of the ROI - Objects that are outside of the WARNING: The cloud-in-cells method of generating the CMD leads to some difficulties since it disperses objects from high-density zones to low density zones. - Magnitudes are not randomized according to their errors """ if seed is not None: np.random.seed(seed) self._setup_cmd() # Randomize the number of stars per bin according to Poisson distribution nstar_per_bin = numpy.random.poisson(lam=self.bkg_lambda) nstar = nstar_per_bin.sum() logger.info("Simulating %i background stars..."%nstar) if not self.config['simulate'].get('uniform'): logger.info("Generating colors from background CMD.") # Distribute the stars within each CMD bin delta_color = self.bkg_centers_color[1]-self.bkg_centers_color[0] delta_mag = self.bkg_centers_mag[1]-self.bkg_centers_mag[0] # Distribute points within each color-mag bins xx,yy = np.meshgrid(self.bkg_centers_color,self.bkg_centers_mag) color = numpy.repeat(xx.flatten(),repeats=nstar_per_bin.flatten()) color += numpy.random.uniform(-delta_color/2.,delta_color/2.,size=nstar) mag_1 = numpy.repeat(yy.flatten(),repeats=nstar_per_bin.flatten()) mag_1 += numpy.random.uniform(-delta_mag/2.,delta_mag/2.,size=nstar) else: # Uniform color-magnitude distribution logger.info("Generating uniform CMD.") mag_1 = np.random.uniform(self.config['mag']['min'],self.config['mag']['max'],size=nstar) color = np.random.uniform(self.config['color']['min'],self.config['color']['max'],size=nstar) mag_2 = mag_1 - color # Random points drawn from healpix subpixels logger.info("Generating uniform positions...") idx = np.random.randint(0,len(self.subpix)-1,size=nstar) lon,lat = pix2ang(self.nside_subpixel,self.subpix[idx]) nside_pixel = self.nside_pixel pix = ang2pix(nside_pixel, lon, lat) # There is probably a better way to do this step without creating the full HEALPix map mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel)) mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse mag_lim_1 = mask[pix] mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel)) mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse mag_lim_2 = mask[pix] mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1) mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2) mc_source_id = mc_source_id * numpy.ones(len(mag_1)) # ADW: Should magnitudes be randomized by the erros? #mag_1 += (numpy.random.normal(size=len(mag_1)) * mag_err_1) #mag_2 += (numpy.random.normal(size=len(mag_2)) * mag_err_2) select = (mag_lim_1>mag_1)&(mag_lim_2>mag_2) ### # Make sure objects lie within the original cmd (should be done later...) ### select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, color, mag_1, ### self.roi.bins_color, self.roi.bins_mag) > 0) logger.info("Clipping %i simulated background stars..."%(~select).sum()) hdu = ugali.observation.catalog.makeHDU(self.config,mag_1[select],mag_err_1[select], mag_2[select],mag_err_2[select], lon[select],lat[select],mc_source_id[select]) catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data) return catalog
def calculate(self, infile, field=1, simple=False): logger.info("Calculating magnitude limit from %s"%infile) #manglefile = self.config['mangle']['infile_%i'%field] #footfile = self.config['data']['footprint'] #try: # footprint = fitsio.read(footfile)['I'].ravel() #except: # logger.warn("Couldn't open %s; will try again."%footfile) # footprint = footfile mag_column = self.config['catalog']['mag_%i_field'%field] magerr_column = self.config['catalog']['mag_err_%i_field'%field] # For simple maglims release = self.config['data']['release'].lower() band = self.config['catalog']['mag_%i_band'%field] pixel_pix_name = 'PIX%i'%self.nside_pixel # If the data already has a healpix pixel assignment then use it # Otherwise recalculate... try: data = fitsio.read(infile,columns=[pixel_pix_name]) except ValueError as e: logger.info(str(e)) columns=[self.config['catalog']['lon_field'], self.config['catalog']['lat_field']] data = fitsio.read(infile,columns=columns)[columns] pix = ang2pix(self.nside_pixel,data[columns[0]],data[columns[1]]) data = recfuncs.rec_append_fields(data,pixel_pix_name,pix) #mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int') mask_maglims = np.zeros(hp.nside2npix(self.nside_mask)) out_pixels = np.zeros(0,dtype='int') out_maglims = np.zeros(0) # Find the objects in each pixel pixel_pix = data[pixel_pix_name] mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask) count = Counter(mask_pix) pixels = sorted(count.keys()) pix_digi = np.digitize(mask_pix,pixels).argsort() idx = 0 min_num = 500 signal_to_noise = 10. magerr_lim = 1/signal_to_noise for pix in pixels: # Calculate the magnitude limit in each pixel num = count[pix] objs = data[pix_digi[idx:idx+num]] idx += num if simple: # Set constant magnitude limits logger.debug("Simple magnitude limit for %s"%infile) mask_maglims[pix] = MAGLIMS[release][band] elif num < min_num: logger.info('Found <%i objects in pixel %i'%(min_num,pix)) mask_maglims[pix] = 0 else: mag = objs[mag_column] magerr = objs[magerr_column] # Estimate the magnitude limit as suggested by: # https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document # (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc) maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)]) # Alternative method to estimate the magnitude limit by fitting median #mag_min, mag_max = mag.min(),mag.max() #mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086? #x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins) #x,y = x[~np.isnan(y)],y[~np.isnan(y)] #magerr_med = interp1d(x,y) #mag0 = np.median(x) #maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False) # Median from just objects near magerr cut mask_maglims[pix] = maglim logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix])) subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel) maglims = np.zeros(len(subpix)) + mask_maglims[pix] out_pixels = np.append(out_pixels,subpix) out_maglims = np.append(out_maglims,maglims) # Remove empty pixels logger.info("Removing empty pixels") idx = np.nonzero(out_maglims > 0)[0] out_pixels = out_pixels[idx] out_maglims = out_maglims[idx] # Remove pixels outside the footprint if self.footfile: logger.info("Checking footprint against %s"%self.footfile) lon,lat = pix2ang(self.nside_pixel,out_pixels) if self.config['coords']['coordsys'] == 'gal': ra,dec = gal2cel(lon,lat) else: ra,dec = lon,lat footprint = inFootprint(self.footprint,ra,dec) idx = np.nonzero(footprint)[0] out_pixels = out_pixels[idx] out_maglims = out_maglims[idx] logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims))) return out_pixels,out_maglims
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'): if not os.path.exists(catfile): msg = "Couldn't find %s"%catfile raise Exception(msg) #columns = [OBJECT_ID,'RA','DEC'] columns = ['RA','DEC'] spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R'] mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z']) #mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z']) columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z] # Hack to get pixel location hpx = int(catfile.split('_')[-1].split('.')[0]) #hpx = ang2pix(NSIDE, cat['RA'], cat['DEC']) ra,dec = pix2ang(NSIDE, hpx) radius = np.degrees(hp.max_pixrad(NSIDE)) msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius) print(msg) #print "Getting coadd catalog: DES" cat = load_infiles([catfile],columns) # Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5 sel = (np.fabs(cat[spread])<0.002) & \ (cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \ (cat[mag_r]>16) & (cat[mag_r]<20) & \ ((cat[mag_g] - cat[mag_i]) > 0.0) & \ ((cat[mag_g] - cat[mag_i]) < 1.5) & \ (cat[nepochs] > 1) cat = cat[sel] if len(cat) == 0: msg = "WARNING: No objects passing selection in: %s"%catfile print(msg) return np.array([],dtype=int), np.array([]) #msg = "Getting external catalog: %s"%catalog ext = get_gaia_catalog(hpx,version=version) m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC']) # Use a fairly wide matching radius (2 arcsec) cut = 1.0 sel = m[-1]*3600. < cut cat_match = cat[m[0][sel]] ext_match = ext[m[1][sel]] cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z], version=version) # Need to put Gaia flux onto the AB system ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934 diff = cat_G - ext_G pix = ang2pix(nside,cat_match['RA'],cat_match['DEC']) upix = np.unique(pix) stat = nd.median(diff,labels=pix,index=upix) if False: plt.figure() plt.hist(cat_G - ext_G) import pdb; pdb.set_trace() return upix,stat
def submit(self, pixels, queue=None, debug=False, configfile=None): """ Submit the likelihood job for the given pixel(s). """ queue = self.config['batch']['cluster'] if queue is None else queue local = (queue == 'local') # Need to develop some way to take command line arguments... self.batch = ugali.utils.batch.batchFactory(queue,**self.config['batch']['opts']) if numpy.isscalar(pixels): pixels = numpy.array([pixels]) outdir = mkdir(self.config['output']['likedir']) logdir = mkdir(join(outdir,'log')) subdir = mkdir(join(outdir,'sub')) # Save the current configuation settings; avoid writing # file multiple times if configfile passed as argument. if configfile is None: if local: configfile = self.configfile else: configfile = '%s/config_queue.py'%(outdir) self.config.write(configfile) lon,lat = pix2ang(self.nside_likelihood,pixels) commands = [] chunk = self.config['batch']['chunk'] istart = 0 logger.info('=== Submit Likelihood ===') for ii,pix in enumerate(pixels): logger.info(' (%i/%i) pixel=%i nside=%i; (lon, lat) = (%.2f, %.2f)'%(ii+1,len(pixels),pix, self.nside_likelihood,lon[ii],lat[ii])) # Create outfile name outfile = self.config.likefile%(pix,self.config['coords']['coordsys'].lower()) outbase = os.path.basename(outfile) jobname = self.config['batch']['jobname'] # Submission command sub = not os.path.exists(outfile) cmd = self.command(outfile,configfile,pix) commands.append([ii,cmd,lon[ii],lat[ii],sub]) if local or chunk == 0: # Not chunking command = cmd submit = sub logfile = join(logdir,os.path.splitext(outbase)[0]+'.log') elif (len(commands)%chunk==0) or (ii+1 == len(pixels)): # End of chunk, create submission script commands = np.array(commands,dtype=object) istart, iend = commands[0][0], commands[-1][0] subfile = join(subdir,'submit_%08i_%08i.sh'%(istart,iend)) logfile = join(logdir,'submit_%08i_%08i.log'%(istart,iend)) command = "sh %s"%subfile submit = np.any(commands[:,-1]) if submit: self.write_script(subfile,commands) else: # Not end of chunk continue commands=[] # Actual job submission if not submit: logger.info(self.skip) continue else: while True: njobs = self.batch.njobs() if njobs < self.config['batch']['max_jobs']: break else: logger.info('%i jobs already in queue, waiting...'%(njobs)) time.sleep(5*chunk) job = self.batch.submit(command,jobname,logfile) logger.info(" "+job) time.sleep(0.5)
def external_astrometry(catfile,catalog='GAIA',nside=64,band='r',plot=False): #nice = os.nice(0) #os.nice(10-nice) if band=='all': band = 'r' if not os.path.exists(catfile): msg = "Couldn't find %s"%catfile raise Exception(msg) columns = [OBJECT_ID,'RA','DEC'] spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band) columns += [spread,mag,nepochs] # Hack to get pixel location hpx = int(catfile.split('_')[-1].split('.')[0]) #hpx = ang2pix(NSIDE, cat['RA'], cat['DEC']) ra,dec = pix2ang(NSIDE, hpx) radius = np.degrees(healpy.max_pixrad(NSIDE)) msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius) print(msg) #print "Getting coadd catalog: DES" cat = load_infiles([catfile],columns) # Select stars with 16 < mag < 21 sel = (np.fabs(cat[spread])<0.002) & \ (cat[mag]>16) & \ (cat[mag]<21) & \ (cat[nepochs] > 1) cat = cat[sel] if len(cat) == 0: msg = "WARNING: No objects passing selection in: %s"%catfile print(msg) return np.array([],dtype=int), np.array([]) #print "Getting external catalog: %s"%catalog if catalog in list(CATALOGS.keys()): ext = get_vizier_catalog(ra,dec,radius,**CATALOGS[catalog]) else: ext = get_local_catalog(ra,dec,radius,catalog) m = match_query(cat['RA'],cat['DEC'],ext['_RAJ2000'],ext['_DEJ2000']) # Use a fairly wide matching radius (2 arcsec) cut = 2.0 sel = m[-1]*3600. < cut sepdeg = m[-1][sel] sepsec = m[-1][sel] * 3600. sepmas = sepsec * 1000. sep = sepmas pix = ang2pix(nside,cat['RA'][sel],cat['DEC'][sel]) upix = np.unique(pix) try: peak = nd.median(sep,labels=pix,index=upix) except ValueError: import pdb; pdb.set_trace() if plot: plt.figure() draw_angsep(sep,bins=np.linspace(0,cut*1000.,101)) if isinstance(plot,basestring): outfile = plot plt.savefig(outfile,bbox_inches='tight') #return cat,ext,m return upix,peak
def allSkyCoordinates(nside): """ Generate a set of coordinates at the centers of pixels of resolutions nside across the full sky. """ lon, lat = pix2ang(nside, np.arange(0, healpy.nside2npix(nside))) return lon, lat