def read_catalog(self, filename=None):
if not filename:
filename = os.path.join(self.config['simulate']['dirname'],self.config['simulate']['catfile'])
catalog = ugali.observation.catalog.Catalog(self.config,filenames=filename)
catalog.data = mlab.rec_append_fields(catalog.data,
names=['PIX8','PIX4096'],
arrs=np.zeros((2,len(catalog.lon)),dtype='>i8'))
return catalog
def read_catalog(self, filename=None):
if not filename:
filename = os.path.join(self.config['simulate']['dirname'],
self.config['simulate']['catfile'])
catalog = ugali.observation.catalog.Catalog(self.config,
filenames=filename)
catalog.data = mlab.rec_append_fields(catalog.data,
names=['PIX8', 'PIX4096'],
arrs=np.zeros(
(2, len(catalog.lon)),
dtype='>i8'))
return catalog
def supplement(self, coordsys='gal'):
""" Add some supplemental columns """
from ugali.utils.projector import gal2cel, gal2cel_angle
from ugali.utils.projector import cel2gal, cel2gal_angle
coordsys = coordsys.lower()
kwargs = dict(usemask=False, asrecarray=True)
out = copy.deepcopy(self)
if ('lon' in out.names) and ('lat' in out.names):
# Ignore entries that are all zero
zeros = np.all(self.ndarray == 0, axis=1)
if coordsys == 'gal':
ra, dec = gal2cel(out.lon, out.lat)
glon, glat = out.lon, out.lat
else:
ra, dec = out.lon, out.lat
glon, glat = cel2gal(out.lon, out.lat)
ra[zeros] = 0
dec[zeros] = 0
glon[zeros] = 0
glat[zeros] = 0
names = ['ra', 'dec', 'glon', 'glat']
arrs = [ra, dec, glon, glat]
out = mlab.rec_append_fields(out, names, arrs).view(Samples)
#out = recfuncs.append_fields(out,names,arrs,**kwargs).view(Samples)
if 'position_angle' in out.names:
if coordsys == 'gal':
pa_gal = out.position_angle
pa_cel = gal2cel_angle(out.lon, out.lat,
out.position_angle)
pa_cel = pa_cel - 180. * (pa_cel > 180.)
else:
pa_gal = cel2gal_angle(out.lon, out.lat,
out.position_angle)
pa_cel = out.position_angle
pa_gal = pa_gal - 180. * (pa_gal > 180.)
pa_gal[zeros] = 0
pa_cel[zeros] = 0
names = ['position_angle_gal', 'position_angle_cel']
arrs = [pa_gal, pa_cel]
out = recfuncs.append_fields(out, names, arrs,
**kwargs).view(Samples)
return out
def _defineVariables(self):
"""
Helper funtion to define pertinent variables from catalog data.
ADW (20170627): This has largely been replaced by properties.
"""
logger.info('Catalog contains %i objects'%(len(self.data)))
mc_source_id_field = self.config['catalog']['mc_source_id_field']
if mc_source_id_field is not None:
if mc_source_id_field not in self.data.dtype.names:
array = np.zeros(len(self.data),dtype='>i8') # FITS byte-order convention
self.data = mlab.rec_append_fields(self.data,
names=mc_source_id_field,
arrs=array)
logger.info('Found %i simulated objects'%(np.sum(self.mc_source_id>0)))
def _defineVariables(self):
"""
Helper funtion to define pertinent variables from catalog data.
ADW (20170627): This has largely been replaced by properties.
"""
logger.info('Catalog contains %i objects' % (len(self.data)))
mc_source_id_field = self.config['catalog']['mc_source_id_field']
if mc_source_id_field is not None:
if mc_source_id_field not in self.data.dtype.names:
array = np.zeros(len(self.data),
dtype='>i8') # FITS byte-order convention
self.data = mlab.rec_append_fields(self.data,
names=mc_source_id_field,
arrs=array)
logger.info('Found %i simulated objects' %
(np.sum(self.mc_source_id > 0)))
def supplement(self,coordsys='gal'):
""" Add some supplemental columns """
from ugali.utils.projector import gal2cel, gal2cel_angle
from ugali.utils.projector import cel2gal, cel2gal_angle
coordsys = coordsys.lower()
kwargs = dict(usemask=False, asrecarray=True)
out = copy.deepcopy(self)
if ('lon' in out.names) and ('lat' in out.names):
# Ignore entries that are all zero
zeros = np.all(self.ndarray==0,axis=1)
if coordsys == 'gal':
ra,dec = gal2cel(out.lon,out.lat)
glon,glat = out.lon,out.lat
else:
ra,dec = out.lon,out.lat
glon,glat = cel2gal(out.lon,out.lat)
ra[zeros] = 0; dec[zeros] = 0
glon[zeros] = 0; glat[zeros] = 0
names = ['ra','dec','glon','glat']
arrs = [ra,dec,glon,glat]
out = mlab.rec_append_fields(out,names,arrs).view(Samples)
#out = recfuncs.append_fields(out,names,arrs,**kwargs).view(Samples)
if 'position_angle' in out.names:
if coordsys == 'gal':
pa_gal = out.position_angle
pa_cel = gal2cel_angle(out.lon,out.lat,out.position_angle)
pa_cel = pa_cel - 180.*(pa_cel > 180.)
else:
pa_gal = cel2gal_angle(out.lon,out.lat,out.position_angle)
pa_cel = out.position_angle
pa_gal = pa_gal - 180.*(pa_gal > 180.)
pa_gal[zeros] = 0; pa_cel[zeros] = 0
names = ['position_angle_gal','position_angle_cel']
arrs = [pa_gal,pa_cel]
out = recfuncs.append_fields(out,names,arrs,**kwargs).view(Samples)
return out
def createAssociations(self):
objects = self.objects
tol = self.config['search']['proximity']
columns = odict()
names = np.empty(len(objects), dtype=object)
names.fill('')
for i, refs in enumerate(self.config['search']['catalogs']):
i += 1
catalog = SourceCatalog()
for ref in refs:
print(ref)
catalog += catalogFactory(ref)
# String length (should be greater than longest name)
length = len(max(catalog['name'], key=len)) + 1
dtype = 'S%i' % length
fitstype = '%iA' % length
assoc = np.empty(len(objects), dtype=dtype)
assoc.fill('')
angsep = np.zeros(len(objects), dtype=np.float32)
idx1, idx2, sep = catalog.match(objects['GLON'],
objects['GLAT'],
tol=tol)
assoc[idx1] = catalog['name'][idx2].astype(dtype)
angsep[idx1] = sep
columns['ASSOC%i' % i] = assoc
columns['ANGSEP%i' % i] = angsep
if length > objects['NAME'].itemsize:
logger.warning("Association name may not fit.")
names = np.where(names == '', assoc, names)
names = names.astype(objects['NAME'].dtype)
objects['NAME'][:] = np.where(names == '', objects['NAME'], names)
objects['NAME'][:] = np.char.replace(objects['NAME'], '_', ' ')
self.assocs = mlab.rec_append_fields(objects, columns.keys(),
columns.values())
self.assocs = self.assocs[self.assocs['NAME'].argsort()]
def createAssociations(self):
objects = self.objects
tol = self.config['search']['proximity']
columns = odict()
names = np.empty(len(objects),dtype=object)
names.fill('')
for i,refs in enumerate(self.config['search']['catalogs']):
i += 1
catalog = SourceCatalog()
for ref in refs:
print(ref)
catalog += catalogFactory(ref)
# String length (should be greater than longest name)
length = len(max(catalog['name'],key=len)) + 1
dtype = 'S%i'%length; fitstype='%iA'%length
assoc = np.empty(len(objects),dtype=dtype)
assoc.fill('')
angsep = np.zeros(len(objects),dtype=np.float32)
idx1,idx2,sep = catalog.match(objects['GLON'],objects['GLAT'],tol=tol)
assoc[idx1] = catalog['name'][idx2].astype(dtype)
angsep[idx1] = sep
columns['ASSOC%i'%i] = assoc
columns['ANGSEP%i'%i] = angsep
if length > objects['NAME'].itemsize:
logger.warning("Association name may not fit.")
names = np.where(names=='',assoc,names)
names = names.astype(objects['NAME'].dtype)
objects['NAME'][:] = np.where(names=='',objects['NAME'],names)
objects['NAME'][:] = np.char.replace(objects['NAME'],'_',' ')
self.assocs=mlab.rec_append_fields(objects,columns.keys(),columns.values())
self.assocs=self.assocs[self.assocs['NAME'].argsort()]
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i,filename in enumerate(infiles):
logger.info('(%i/%i) %s'%(i+1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found"%len(data))
if not len(data): continue
columns = map(str.upper,data.dtype.names)
names,arrs = [],[]
if (lon_field in columns) and (lat_field in columns):
lon,lat = data[lon_field],data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon,lat = cel2gal(data['RA'],data['DEC'])
names += [lon_field,lat_field]
arrs += [lon,lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from GLON,GLAT"
lon,lat = gal2cel(data['GLON'],data['GLAT'])
names += [lon_field,lat_field]
arrs += [lon,lat]
cat_pix = ang2pix(nside_catalog,lon,lat)
pix_pix = ang2pix(nside_pixel,lon,lat)
cat_pix_name = 'PIX%i'%nside_catalog
pix_pix_name = 'PIX%i'%nside_pixel
try:
names += [cat_pix_name,pix_pix_name]
arrs += [cat_pix,pix_pix]
data=mlab.rec_append_fields(data,names=names,arrs=arrs)
except ValueError as e:
logger.warn(str(e)+'; not adding column.')
#data[cat_pix_name] = cat_pix
#data[pix_pix_name] = pix_pix
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s"%pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s"%outfile)
out=fitsio.FITS(outfile,mode='rw')
out.write(arr)
hdr=healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE','ORDERING','NSIDE','COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',pix,comment='HEALPIX pixel for this file')
else:
out=fitsio.FITS(outfile,mode='rw')
out[1].append(arr)
logger.debug("Writing %s"%outfile)
out.close()
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i, filename in enumerate(infiles):
logger.info('(%i/%i) %s' % (i + 1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found" % len(data))
if not len(data): continue
columns = map(str.upper, data.dtype.names)
names, arrs = [], []
if (lon_field in columns) and (lat_field in columns):
lon, lat = data[lon_field], data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon, lat = cel2gal(data['RA'], data['DEC'])
names += [lon_field, lat_field]
arrs += [lon, lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field)
msg += "\nConverting from GLON,GLAT"
lon, lat = gal2cel(data['GLON'], data['GLAT'])
names += [lon_field, lat_field]
arrs += [lon, lat]
cat_pix = ang2pix(nside_catalog, lon, lat)
pix_pix = ang2pix(nside_pixel, lon, lat)
cat_pix_name = 'PIX%i' % nside_catalog
pix_pix_name = 'PIX%i' % nside_pixel
names += [cat_pix_name, pix_pix_name]
arrs += [cat_pix, pix_pix]
data = mlab.rec_append_fields(data, names=names, arrs=arrs)
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s" % pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s" % outfile)
out = fitsio.FITS(outfile, mode='rw')
out.write(arr)
hdr = healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE', 'ORDERING', 'NSIDE', 'COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',
pix,
comment='HEALPIX pixel for this file')
else:
out = fitsio.FITS(outfile, mode='rw')
out[1].append(arr)
logger.debug("Writing %s" % outfile)
out.close()
