def __init__(self, lon_ref, lat_ref, proj_type='ait'):
self.lon_ref = lon_ref
self.lat_ref = lat_ref
self.proj_type = proj_type
if proj_type.lower() == 'ait':
self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False)
self.sphere_to_image_func = aitoffSphereToImage
self.image_to_sphere_func = aitoffImageToSphere
elif proj_type.lower() == 'tan':
self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=True)
self.sphere_to_image_func = gnomonicSphereToImage
self.image_to_sphere_func = gnomonicImageToSphere
elif proj_type.lower() == 'car':
def rotate(lon, lat, invert=False):
if invert:
return lon + np.array([lon_ref]), lat + np.array([lat_ref])
else:
return lon - np.array([lon_ref]), lat - np.array([lat_ref])
self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False)
# Monkey patch the rotate function
self.rotator.rotate = rotate
self.sphere_to_image_func = cartesianSphereToImage
self.image_to_sphere_func = cartesianImageToSphere
else:
logger.warn('%s not recognized' % (proj_type))
def load_infiles(infiles,columns=None,multiproc=False):
if isinstance(infiles,str):
infiles = [infiles]
logger.debug("Loading %s files..."%len(infiles))
args = list(zip(infiles,len(infiles)*[columns]))
if multiproc:
from multiprocessing import Pool
processes = multiproc if multiproc > 0 else None
p = Pool(processes,maxtasksperchild=1)
out = p.map(load,args)
else:
out = [load(arg) for arg in args]
dtype = out[0].dtype
for i,d in enumerate(out):
if d.dtype != dtype:
# ADW: Not really safe...
logger.warn("Casting input data to same type.")
out[i] = d.astype(dtype)
logger.debug('Concatenating arrays...')
return np.concatenate(out)
def load_files(filenames, multiproc=False, **kwargs):
""" Load a set of FITS files with kwargs. """
filenames = np.atleast_1d(filenames)
logger.debug("Loading %s files..." % len(filenames))
kwargs = [dict(filename=f, **kwargs) for f in filenames]
if multiproc:
from multiprocessing import Pool
processes = multiproc if multiproc > 0 else None
pool = Pool(processes, maxtasksperchild=1)
out = pool.map(load_file, kwargs)
del pool
else:
out = [load_file(kw) for kw in kwargs]
dtype = out[0].dtype
for i, d in enumerate(out):
if d.dtype != dtype:
# ADW: Not really safe...
logger.warn("Casting input data to same type.")
out[i] = d.astype(dtype, copy=False)
logger.debug('Concatenating arrays...')
return np.concatenate(out)
def __init__(self, lon_ref, lat_ref, proj_type = 'ait'):
self.lon_ref = lon_ref
self.lat_ref = lat_ref
self.proj_type = proj_type
if proj_type.lower() == 'ait':
self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False)
self.sphere_to_image_func = aitoffSphereToImage
self.image_to_sphere_func = aitoffImageToSphere
elif proj_type.lower() == 'tan':
self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=True)
self.sphere_to_image_func = gnomonicSphereToImage
self.image_to_sphere_func = gnomonicImageToSphere
elif proj_type.lower() == 'car':
def rotate(lon,lat,invert=False):
if invert:
return lon + np.array([lon_ref]), lat + np.array([lat_ref])
else:
return lon - np.array([lon_ref]), lat - np.array([lat_ref])
self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False)
# Monkey patch the rotate function
self.rotator.rotate = rotate
self.sphere_to_image_func = cartesianSphereToImage
self.image_to_sphere_func = cartesianImageToSphere
else:
logger.warn('%s not recognized'%(proj_type))
def load_infiles(infiles, columns=None, multiproc=False):
if isstring(infiles):
infiles = [infiles]
logger.debug("Loading %s files..." % len(infiles))
args = list(zip(infiles, len(infiles) * [columns]))
if multiproc:
from multiprocessing import Pool
processes = multiproc if multiproc > 0 else None
p = Pool(processes, maxtasksperchild=1)
out = p.map(load, args)
else:
out = [load(arg) for arg in args]
dtype = out[0].dtype
for i, d in enumerate(out):
if d.dtype != dtype:
# ADW: Not really safe...
logger.warn("Casting input data to same type.")
out[i] = d.astype(dtype)
logger.debug('Concatenating arrays...')
return np.concatenate(out)
def _createFilenames(self):
"""
Create a masked records array of all filenames for the given set of
pixels and store the existence of those files in the mask values.
Parameters:
-----------
None
Returns:
--------
recarray : pixels and mask value
"""
nside_catalog = self['coords']['nside_catalog']
npix = hp.nside2npix(nside_catalog)
pixels = np.arange(npix)
catalog_dir = self['catalog']['dirname']
catalog_base = self['catalog']['basename']
catalog_path = os.path.join(catalog_dir, catalog_base)
mask_dir = self['mask']['dirname']
mask_base_1 = self['mask']['basename_1']
mask_base_2 = self['mask']['basename_2']
mask_path_1 = os.path.join(mask_dir, mask_base_1)
mask_path_2 = os.path.join(mask_dir, mask_base_2)
data = np.ma.empty(npix,
dtype=[('pix', int), ('catalog', object),
('mask_1', object), ('mask_2', object)])
mask = np.ma.empty(npix,
dtype=[('pix', bool), ('catalog', bool),
('mask_1', bool), ('mask_2', bool)])
# Build the filenames
data['pix'] = pixels
data['catalog'] = np.char.mod(catalog_path, pixels)
data['mask_1'] = np.char.mod(mask_path_1, pixels)
data['mask_2'] = np.char.mod(mask_path_2, pixels)
# Build the mask of existing files using glob
mask['catalog'] = ~np.in1d(data['catalog'],
glob.glob(catalog_dir + '/*'))
mask['mask_1'] = ~np.in1d(data['mask_1'], glob.glob(mask_dir + '/*'))
mask['mask_2'] = ~np.in1d(data['mask_2'], glob.glob(mask_dir + '/*'))
for name in ['catalog', 'mask_1', 'mask_2']:
if np.all(mask[name]): logger.warn("All '%s' files masked" % name)
# mask 'pix' if all files not present
mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2']
if np.all(mask['pix']): logger.warn("All pixels masked")
return np.ma.MaskedArray(data, mask, fill_value=[-1, '', '', ''])
def __init__(self, type='chabrier'):
"""
Initialize an instance of an initial mass function.
"""
self.type = type
if self.type == 'chabrier':
self.pdf = chabrierIMF
else:
logger.warn('initial mass function type %s not recognized' %
(self.type))
def _createFilenames(self):
"""
Create a masked records array of all filenames for the given set of
pixels and store the existence of those files in the mask values.
Parameters:
-----------
None
Returns:
--------
recarray : pixels and mask value
"""
nside_catalog = self['coords']['nside_catalog']
npix = hp.nside2npix(nside_catalog)
pixels = np.arange(npix)
catalog_dir = self['catalog']['dirname']
catalog_base = self['catalog']['basename']
catalog_path = os.path.join(catalog_dir,catalog_base)
mask_dir = self['mask']['dirname']
mask_base_1 = self['mask']['basename_1']
mask_base_2 = self['mask']['basename_2']
mask_path_1 = os.path.join(mask_dir,mask_base_1)
mask_path_2 = os.path.join(mask_dir,mask_base_2)
data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object),
('mask_1',object), ('mask_2',object)])
mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool),
('mask_1',bool), ('mask_2',bool)])
# Build the filenames
data['pix'] = pixels
data['catalog'] = np.char.mod(catalog_path,pixels)
data['mask_1'] = np.char.mod(mask_path_1,pixels)
data['mask_2'] = np.char.mod(mask_path_2,pixels)
# Build the mask of existing files using glob
mask['catalog'] = ~np.in1d(data['catalog'],glob.glob(catalog_dir+'/*'))
mask['mask_1'] = ~np.in1d(data['mask_1'],glob.glob(mask_dir+'/*'))
mask['mask_2'] = ~np.in1d(data['mask_2'],glob.glob(mask_dir+'/*'))
for name in ['catalog','mask_1','mask_2']:
if np.all(mask[name]): logger.warn("All '%s' files masked"%name)
# mask 'pix' if all files not present
mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2']
if np.all(mask['pix']): logger.warn("All pixels masked")
return np.ma.MaskedArray(data, mask, fill_value=[-1,'','',''])
def _setup(self):
self.nside_catalog = self.config['coords']['nside_catalog']
self.nside_mask = self.config['coords']['nside_mask']
self.nside_pixel = self.config['coords']['nside_pixel']
self.filenames = self.config.getFilenames()
self.footfile = self.config['data']['footprint']
try:
self.footprint = fitsio.read(self.footfile)['I'].ravel()
except:
logger.warn("Couldn't open %s; will pass through."%self.footfile)
self.footprint = self.footfile
def _setup(self):
self.nside_catalog = self.config['coords']['nside_catalog']
self.nside_mask = self.config['coords']['nside_mask']
self.nside_pixel = self.config['coords']['nside_pixel']
self.filenames = self.config.getFilenames()
self.footfile = self.config['data']['footprint']
try:
self.footprint = fitsio.read(self.footfile)['I'].ravel()
except:
logger.warn("Couldn't open %s; will pass through."%self.footfile)
self.footprint = self.footfile
def __init__(self, infiles, roi):
"""
Parameters:
-----------
infiles : list of sparse healpix mask files
roi : roi object
Returns:
--------
mask : MaskBand object
"""
self.roi = roi
self.config = self.roi.config
# ADW: It's overkill to make the full map just to slim it
# down, but we don't have a great way to go from map pixels to
# roi pixels.
nside, pixel, maglim = healpix.read_partial_map(infiles,
column='MAGLIM')
self.nside = nside
# Sparse maps of pixels in various ROI regions
self.mask_roi_sparse = maglim[self.roi.pixels]
# Try to get the detection fraction
self.frac_roi_sparse = (self.mask_roi_sparse > 0)
try:
logger.debug("Reading FRACDET...")
nside, pixel, frac = healpix.read_partial_map(infiles,
column='FRACDET')
# This clipping might gloss over bugs...
fractype = self.config['mask'].get('fractype', 'binary')
fracmin = self.config['mask'].get('fracmin', 0.5)
if fractype == 'binary':
frac = np.where(frac < fracmin, 0.0, 1.0)
elif fractype == 'full':
frac = np.where(frac < fracmin, 0.0, frac)
elif not fractype:
pass
else:
msg = "Unrecognized fractype: %s" % fractype
logger.warn(msg)
self.frac_roi_sparse = np.clip(frac[self.roi.pixels], 0.0, 1.0)
except ValueError as e:
# No detection fraction present
msg = "No 'FRACDET' column found in masks; assuming FRACDET = 1.0"
logger.info(msg)
# Explicitly zero the maglim of pixels with fracdet < fracmin
self.mask_roi_sparse[self.frac_roi_sparse == 0] = 0.0
def __init__(self, infiles, roi):
"""
Parameters:
-----------
infiles : list of sparse healpix mask files
roi : roi object
Returns:
--------
mask : MaskBand object
"""
self.roi = roi
self.config = self.roi.config
# ADW: It's overkill to make the full map just to slim it
# down, but we don't have a great way to go from map pixels to
# roi pixels.
nside,pixel,maglim = healpix.read_partial_map(infiles,column='MAGLIM')
self.nside = nside
# Sparse maps of pixels in various ROI regions
self.mask_roi_sparse = maglim[self.roi.pixels]
# Try to get the detection fraction
self.frac_roi_sparse = (self.mask_roi_sparse > 0)
try:
logger.debug("Reading FRACDET...")
nside,pixel,frac=healpix.read_partial_map(infiles,column='FRACDET')
# This clipping might gloss over bugs...
fractype = self.config['mask'].get('fractype','binary')
fracmin = self.config['mask'].get('fracmin',0.5)
if fractype == 'binary':
frac = np.where(frac < fracmin, 0.0, 1.0)
elif fractype == 'full':
frac = np.where(frac < fracmin, 0.0, frac)
elif not fractype:
pass
else:
msg = "Unrecognized fractype: %s"%fractype
logger.warn(msg)
self.frac_roi_sparse = np.clip(frac[self.roi.pixels],0.0,1.0)
except ValueError as e:
# No detection fraction present
msg = "No 'FRACDET' column found in masks; assuming FRACDET = 1.0"
logger.info(msg)
# Explicitly zero the maglim of pixels with fracdet < fracmin
self.mask_roi_sparse[self.frac_roi_sparse == 0] = 0.0
def download(self,age=None,metallicity=None,outdir=None,force=False):
"""
Check valid parameter range and download isochrones from:
http://stev.oapd.inaf.it/cgi-bin/cmd
"""
try:
from urllib.error import URLError
except ImportError:
from urllib2 import URLError
if age is None: age = float(self.age)
if metallicity is None: metallicity = float(self.metallicity)
if outdir is None: outdir = './'
basename = self.params2filename(age,metallicity)
outfile = os.path.join(outdir,basename)
if os.path.exists(outfile) and not force:
try:
self.verify(outfile,self.survey,age,metallicity)
logger.info("Found %s; skipping..."%(outfile))
return
except Exception as e:
msg = "Overwriting corrupted %s..."%(outfile)
logger.warn(msg)
os.remove(outfile)
mkdir(outdir)
self.print_info(age,metallicity)
self.query_server(outfile,age,metallicity)
if not os.path.exists(outfile):
raise RuntimeError('Download failed')
try:
self.verify(outfile,self.survey,age,metallicity)
except Exception as e:
msg = "Output file is corrupted."
logger.error(msg)
msg = "Removing %s."%outfile
logger.info(msg)
os.remove(outfile)
raise(e)
return outfile
def download(self,age=None,metallicity=None,outdir=None,force=False):
"""
Check valid parameter range and download isochrones from:
http://stev.oapd.inaf.it/cgi-bin/cmd
"""
try:
from urllib.error import URLError
except ImportError:
from urllib2 import URLError
if age is None: age = float(self.age)
if metallicity is None: metallicity = float(self.metallicity)
if outdir is None: outdir = './'
basename = self.params2filename(age,metallicity)
outfile = os.path.join(outdir,basename)
if os.path.exists(outfile) and not force:
try:
self.verify(outfile,self.survey,age,metallicity)
logger.info("Found %s; skipping..."%(outfile))
return
except Exception as e:
msg = "Overwriting corrupted %s..."%(outfile)
logger.warn(msg)
os.remove(outfile)
mkdir(outdir)
self.print_info(age,metallicity)
self.query_server(outfile,age,metallicity)
if not os.path.exists(outfile):
raise RuntimeError('Download failed')
try:
self.verify(outfile,self.survey,age,metallicity)
except Exception as e:
msg = "Output file is corrupted."
logger.error(msg)
msg = "Removing %s."%outfile
logger.info(msg)
os.remove(outfile)
raise(e)
return outfile
def download(self, age, metallicity, outdir=None, force=False):
"""
Check valid parameter range and download isochrones from:
http://stev.oapd.inaf.it/cgi-bin/cmd
"""
if outdir is None: outdir = './'
basename = self.isochrone.params2filename(age, metallicity)
outfile = os.path.join(outdir, basename)
if os.path.exists(outfile) and not force:
try:
self.verify(outfile, self.survey, age, metallicity)
logger.info("Found %s; skipping..." % (outfile))
return
except Exception as e:
msg = "Overwriting corrupted %s..." % (outfile)
logger.warn(msg)
#os.remove(outfile)
mkdir(outdir)
self.print_info(age, metallicity)
try:
self.query_server(outfile, age, metallicity)
except Exception as e:
logger.debug(str(e))
raise RuntimeError('Bad server response')
if not os.path.exists(outfile):
raise RuntimeError('Download failed')
try:
self.verify(outfile, self.survey, age, metallicity)
except Exception as e:
msg = "Output file is corrupted."
logger.error(msg)
#os.remove(outfile)
raise (e)
return outfile
def download(self,age,metallicity,outdir=None,force=False):
"""
Check valid parameter range and download isochrones from:
http://stev.oapd.inaf.it/cgi-bin/cmd
"""
if outdir is None: outdir = './'
basename = self.isochrone.params2filename(age,metallicity)
outfile = os.path.join(outdir,basename)
if os.path.exists(outfile) and not force:
try:
self.verify(outfile,self.survey,age,metallicity)
logger.info("Found %s; skipping..."%(outfile))
return
except Exception as e:
msg = "Overwriting corrupted %s..."%(outfile)
logger.warn(msg)
#os.remove(outfile)
mkdir(outdir)
self.print_info(age,metallicity)
try:
self.query_server(outfile,age,metallicity)
except Exception as e:
logger.debug(str(e))
raise RuntimeError('Bad server response')
if not os.path.exists(outfile):
raise RuntimeError('Download failed')
try:
self.verify(outfile,self.survey,age,metallicity)
except Exception as e:
msg = "Output file is corrupted."
logger.error(msg)
#os.remove(outfile)
raise(e)
return outfile
def run(self):
if 'scan' in self.opts.run:
logger.info("Running 'scan'...")
farm = Farm(self.config,verbose=self.opts.verbose)
farm.submit_all(coords=self.opts.coords,queue=self.opts.queue,debug=self.opts.debug)
if 'merge' in self.opts.run:
logger.info("Running 'merge'...")
mergefile = self.config.mergefile
roifile = self.config.roifile
filenames = self.config.likefile.split('_%')[0]+'_*.fits'
infiles = sorted(glob.glob(filenames))
if exists(mergefile) and not self.opts.force:
logger.warn(" Found %s; skipping..."%mergefile)
else:
logger.info(" Merging likelihood files...")
ugali.utils.healpix.merge_partial_maps(infiles,mergefile)
if exists(roifile) and not self.opts.force:
logger.warn(" Found %s; skipping..."%roifile)
else:
logger.info(" Merging likelihood headers...")
ugali.utils.healpix.merge_likelihood_headers(infiles,roifile)
#ugali.utils.skymap.mergeLikelihoodFiles(infiles,mergefile,roifile)
if 'tar' in self.opts.run:
logger.info("Running 'tar'...")
outdir = mkdir(self.config['output']['likedir'])
logdir = mkdir(join(outdir,'log'))
scanfile = self.config.likefile.split('_%')[0]+'_[0-9]*.fits'
tarfile = join(self.config.likefile.split('_%')[0]+'_pixels.tar.gz')
jobname = 'tar'
logfile = os.path.join(logdir,'scan_tar.log')
cmd = 'tar --remove-files -cvzf %s %s'%(tarfile,scanfile)
if exists(tarfile) and not self.opts.force:
logger.warn(" Found %s; skipping..."%tarfile)
else:
logger.info(" Tarring likelihood files...")
logger.info(cmd)
self.batch.submit(cmd,jobname,logfile)
if 'plot' in self.opts.run:
# WARNING: Loading the full 3D healpix map is memory intensive.
logger.info("Running 'plot'...")
# Should do this in environment variable
import matplotlib
matplotlib.use('Agg')
import pylab as plt
import ugali.utils.plotting as plotting
skymap = ugali.utils.skymap.readSparseHealpixMap(self.config.mergefile,'LOG_LIKELIHOOD')[1]
plotting.plotSkymap(skymap)
outdir = mkdir(self.config['output']['plotdir'])
basename = os.path.basename(self.config.mergefile.replace('.fits','.png'))
outfile = os.path.join(outdir,basename)
plt.savefig(outfile)
def load_files(filenames,multiproc=False,**kwargs):
""" Load a set of FITS files with kwargs. """
filenames = np.atleast_1d(filenames)
logger.debug("Loading %s files..."%len(filenames))
kwargs = [dict(filename=f,**kwargs) for f in filenames]
if multiproc:
from multiprocessing import Pool
processes = multiproc if multiproc > 0 else None
p = Pool(processes,maxtasksperchild=1)
out = p.map(load_file,kwargs)
else:
out = [load_file(kw) for kw in kwargs]
dtype = out[0].dtype
for i,d in enumerate(out):
if d.dtype != dtype:
# ADW: Not really safe...
logger.warn("Casting input data to same type.")
out[i] = d.astype(dtype,copy=False)
logger.debug('Concatenating arrays...')
return np.concatenate(out)
def run(args):
try:
p.download(*args)
except Exception as e:
logger.warn(str(e))
logger.error("Download failed.")
def run(self):
if self.opts.coords is not None:
coords = self.opts.coords
names = vars(self.opts).get('names',len(coords)*[''])
else:
names,coords = self.parser.parse_targets(self.config.candfile)
labels=[n.lower().replace(' ','_').replace('(','').replace(')','') for n in names]
self.outdir=mkdir(self.config['output']['mcmcdir'])
self.logdir=mkdir(join(self.outdir,'log'))
args = list(zip(len(names)*[self.opts.config],names,labels,coords))
if 'mcmc' in self.opts.run:
logger.info("Running 'mcmc'...")
try: shutil.copy(self.opts.config,self.outdir)
except Exception as e: logger.warn(e.message)
for config,name,label,coord in args:
glon,glat,radius = coord
outfile = make_filenames(self.config,label)['samfile']
base = splitext(basename(outfile))[0]
logfile=join(self.logdir,base+'.log')
jobname=base
script = self.config['mcmc']['script']
nthreads = self.config['mcmc']['nthreads']
srcmdl = self.config['mcmc'].get('srcmdl')
if srcmdl is not None:
try: shutil.copy(srcmdl,self.outdir)
except Exception as e: logger.warn(e.message)
logger.info('%s (%s)'%(name,srcmdl))
cmd='%s %s --name %s --srcmdl %s %s' % (
script,self.opts.config,name,srcmdl,outfile)
else:
logger.info('%s (%.4f,%.4f)'%(name,glon,glat))
cmd='%s %s --name %s --gal %.4f %.4f --grid %s'% (
script,self.opts.config,name,glon,glat,outfile)
logger.info(cmd)
self.batch.submit(cmd,jobname,logfile,n=nthreads,a='mpirun')
if 'results' in self.opts.run:
logger.info("Running 'results'...")
if len(args) > 1:
pool = Pool(maxtasksperchild=1)
pool.map(do_results,args)
else:
do_results(*args)
if 'membership' in self.opts.run:
logger.info("Running 'membership'...")
if len(args) > 1:
pool = Pool(maxtasksperchild=1)
pool.map(do_membership,args)
else:
do_membership(*args)
if 'plot' in self.opts.run:
logger.info("Running 'plot'...")
if len(args) > 1:
pool = Pool(maxtasksperchild=1)
pool.map(do_plot,args)
#map(do_plot,args)
else:
do_plot(*args)
if 'collect' in self.opts.run:
logger.info("Running 'collect'...")
results = odict()
srcmdl = odict()
params = odict()
for config,name,label,coord in args:
srcfile = make_filenames(self.config,name)['srcfile']
results[name] = yaml.load(open(srcfile))['results']
srcmdl[name] = yaml.load(open(srcfile))['source']
params[name] = yaml.load(open(srcfile))['params']
for base,output in [('results.yaml',results),('srcmdl.yaml',srcmdl),('params.yaml',params)]:
outfile = join(self.outdir,base)
out = open(outfile,'w')
out.write(yaml.dump(output))
out.close()
if 'scan' in self.opts.run:
logger.info("Running 'scan'...")
for config,name,label,coord in args:
logdir = mkdir('plots/log')
logfile=join(logdir,'%s_lnlscan.log')
cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45'%(self.opts.config,name,'age','metallicity')
self.batch.submit(cmd,logfile=logfile)
cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45'%(self.opts.config,name,'metallicity','distance_modulus')
self.batch.submit(cmd,logfile=logfile)
cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45'%(self.opts.config,name,'age','distance_modulus')
self.batch.submit(cmd,logfile=logfile)
def search(self,
coords=None,
distance_modulus=None,
extension=None,
tolerance=1.e-2):
"""
Organize a grid search over ROI target pixels, distance
moduli, and extensions. If coords, distance_modulus, or
extension is specified, then the nearest value in the
predefined scan grid is used. ***This may be different than
the input value.**
Parameters
----------
coords : (float,float)
coordinate to search (matched to nearest scan value)
distance_modulus : float
distance modulus to search (matched to nearest scan value)
extension : float
extension to search (matched to nearest scan value)
tolerance : float
tolerance on richness maximization
Returns
-------
None
"""
nmoduli = len(self.distance_modulus_array)
npixels = len(self.roi.pixels_target)
self.loglike_array = np.zeros([nmoduli, npixels], dtype='f4')
self.richness_array = np.zeros([nmoduli, npixels], dtype='f4')
self.stellar_mass_array = np.zeros([nmoduli, npixels], dtype='f4')
self.fraction_observable_array = np.zeros([nmoduli, npixels],
dtype='f4')
self.extension_fit_array = np.zeros([nmoduli, npixels], dtype='f4')
# DEPRECATED: ADW 2019-04-27
self.richness_lower_array = np.zeros([nmoduli, npixels], dtype='f4')
self.richness_upper_array = np.zeros([nmoduli, npixels], dtype='f4')
self.richness_ulimit_array = np.zeros([nmoduli, npixels], dtype='f4')
# Specific pixel/distance_modulus
coord_idx, distance_modulus_idx, extension_idx = None, None, None
if coords is not None:
# Match to nearest grid coordinate index
coord_idx = self.roi.indexTarget(coords[0], coords[1])
if distance_modulus is not None:
# Match to nearest distance modulus index
distance_modulus_idx = np.fabs(self.distance_modulus_array -
distance_modulus).argmin()
if extension is not None:
# Match to nearest extension
extension_idx = np.fabs(self.extension_array - extension).argmin()
lon, lat = self.roi.pixels_target.lon, self.roi.pixels_target.lat
logger.info('Looping over distance moduli in grid search ...')
for ii, distance_modulus in enumerate(self.distance_modulus_array):
# Specific distance
if distance_modulus_idx is not None:
if ii != distance_modulus_idx: continue
logger.info(' (%-2i/%i) Distance Modulus=%.1f ...' %
(ii + 1, nmoduli, distance_modulus))
# No objects, continue
if len(self.loglike.catalog) == 0:
logger.warn("No catalog objects")
continue
# Set distance_modulus once to save time
self.loglike.set_params(distance_modulus=distance_modulus)
# Loop over pixels
for jj in range(0, npixels):
# Specific pixel
if coord_idx is not None:
if jj != coord_idx: continue
# Set kernel location
self.loglike.set_params(lon=lon[jj], lat=lat[jj])
loglike = 0
# Loop over extensions
for kk, ext in enumerate(self.extension_array):
# Specific extension
if extension_idx is not None:
if kk != extension_idx: continue
# Set extension
self.loglike.set_params(extension=ext)
# Doesn't re-sync distance_modulus each time
self.loglike.sync_params()
# Maximize the likelihood with respect to richness
loglike, rich, p = self.loglike.fit_richness()
if loglike < self.loglike_array[ii][jj]:
# No loglike increase, continue
continue
self.loglike_array[ii][jj], self.richness_array[ii][
jj], parabola = loglike, rich, p
self.stellar_mass_array[ii][
jj] = self.stellar_mass_conversion * self.richness_array[
ii][jj]
self.fraction_observable_array[ii][jj] = self.loglike.f
self.extension_fit_array[ii][jj] = self.source.extension
# ADW: Careful, we are leaving the extension at the
# last value in the array, not at the maximum...
# Debug output
args = (jj + 1, npixels, lon[jj], lat[jj],
2. * self.loglike_array[ii][jj],
self.stellar_mass_array[ii][jj],
self.fraction_observable_array[ii][jj],
self.extension_fit_array[ii][jj])
msg = ' (%-3i/%i) Candidate at (%.2f, %.2f) ... '
msg += 'TS=%.1f, Mstar=%.2g, ObsFrac=%.2g, Ext=%.2g'
logger.debug(msg % args)
"""
# This is debugging output
if self.config['scan']['full_pdf']:
DeprecationWarning("'full_pdf' is deprecated.")
self.richness_lower_array[ii][jj], self.richness_upper_array[ii][jj] = self.loglike.richness_interval(0.6827)
self.richness_ulimit_array[ii][jj] = parabola.bayesianUpperLimit(0.95)
args = (
2. * self.loglike_array[ii][jj],
self.stellar_mass_conversion*self.richness_array[ii][jj],
self.stellar_mass_conversion*self.richness_lower_array[ii][jj],
self.stellar_mass_conversion*self.richness_upper_array[ii][jj],
self.stellar_mass_conversion*self.richness_ulimit_array[ii][jj]
)
msg = 'TS=%.1f, Stellar Mass=%.1f (%.1f -- %.1f @ 0.68 CL, < %.1f @ 0.95 CL)'%(args)
logger.debug(msg)
"""
jj_max = self.loglike_array[ii].argmax()
args = (jj_max + 1, npixels, lon[jj_max], lat[jj_max],
2. * self.loglike_array[ii][jj_max],
self.stellar_mass_conversion *
self.richness_array[ii][jj_max],
self.extension_fit_array[ii][jj_max])
msg = ' (%-3i/%i) Max at (%.2f, %.2f) : TS=%.1f, Mstar=%.2g, Ext=%.2f' % (
args)
logger.info(msg)
def getFilenames(self,pixels=None):
"""
Create a masked records array of all filenames for the given set of
pixels and store the existence of those files in the mask values.
If pixels is None, default behavior is to try to join grab
dirname + basename with no pixel insertion. If pixels == -1, grab
all pixels of 'nside_catalog'.
Examples:
f = getFilenames([1,2,3])
# All possible catalog files
f['catalog'].data
# All existing catalog files
f['catalog'][~f.mask['catalog']]
# or
f['catalog'].compressed()
# All missing mask_1 files
f['mask_1'][f.mask['mask_1']]
# Pixels where all files exist
f['pix'][~f.mask['pix']]
"""
nside_catalog = self['coords']['nside_catalog']
if nside_catalog is None:
pixels = [None]
elif pixels is not None:
pixels = [pixels] if np.isscalar(pixels) else pixels
else:
pixels = np.arange(healpy.nside2npix(nside_catalog))
npix = len(pixels)
catalog_dir = self['catalog']['dirname']
catalog_base = self['catalog']['basename']
mask_dir = self['mask']['dirname']
mask_base_1 = self['mask']['basename_1']
mask_base_2 = self['mask']['basename_2']
data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object),
('mask_1',object), ('mask_2',object)])
mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool),
('mask_1',bool), ('mask_2',bool)])
for ii,pix in enumerate(pixels):
if pix is None:
catalog = os.path.join(catalog_dir,catalog_base)
mask_1 = os.path.join(mask_dir,mask_base_1)
mask_2 = os.path.join(mask_dir,mask_base_2)
else:
catalog = os.path.join(catalog_dir,catalog_base%pix)
mask_1 = os.path.join(mask_dir,mask_base_1%pix)
mask_2 = os.path.join(mask_dir,mask_base_2%pix)
data[ii]['pix'] = pix if pix is not None else -1
data[ii]['catalog'] = catalog
data[ii]['mask_1'] = mask_1
data[ii]['mask_2'] = mask_2
mask[ii]['catalog'] = not os.path.exists(catalog)
mask[ii]['mask_1'] = not os.path.exists(mask_1)
mask[ii]['mask_2'] = not os.path.exists(mask_2)
for name in ['catalog','mask_1','mask_2']:
if np.all(mask[name]): logger.warn("All '%s' files masked"%name)
# 'pix' is masked if all files not present
mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2']
if np.all(mask['pix']): logger.warn("All pixels masked")
#return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,None,None,None])
return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,'','',''])
def run(self):
if 'scan' in self.opts.run:
logger.info("Running 'scan'...")
farm = Farm(self.config, verbose=self.opts.verbose)
farm.submit_all(coords=self.opts.coords,
queue=self.opts.queue,
debug=self.opts.debug)
if 'merge' in self.opts.run:
logger.info("Running 'merge'...")
mergefile = self.config.mergefile
roifile = self.config.roifile
filenames = self.config.likefile.split('_%')[0] + '_*.fits'
infiles = sorted(glob.glob(filenames))
if exists(mergefile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % mergefile)
else:
logger.info(" Merging likelihood files...")
ugali.utils.healpix.merge_partial_maps(infiles, mergefile)
if exists(roifile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % roifile)
else:
logger.info(" Merging likelihood headers...")
ugali.utils.healpix.merge_likelihood_headers(infiles, roifile)
#ugali.utils.skymap.mergeLikelihoodFiles(infiles,mergefile,roifile)
if 'tar' in self.opts.run:
logger.info("Running 'tar'...")
outdir = mkdir(self.config['output']['likedir'])
logdir = mkdir(join(outdir, 'log'))
scanfile = self.config.likefile.split('_%')[0] + '_[0-9]*.fits'
tarfile = join(self.config.likefile.split('_%')[0] + '_pixels.tar.gz')
jobname = 'tar'
logfile = os.path.join(logdir, 'scan_tar.log')
cmd = 'tar --remove-files -cvzf %s %s' % (tarfile, scanfile)
if exists(tarfile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % tarfile)
else:
logger.info(" Tarring likelihood files...")
logger.info(cmd)
self.batch.submit(cmd, jobname, logfile)
if 'plot' in self.opts.run:
# WARNING: Loading the full 3D healpix map is memory intensive.
logger.info("Running 'plot'...")
# Should do this in environment variable
import matplotlib
matplotlib.use('Agg')
import pylab as plt
import ugali.utils.plotting as plotting
skymap = ugali.utils.skymap.readSparseHealpixMap(
self.config.mergefile, 'LOG_LIKELIHOOD')[1]
plotting.plotSkymap(skymap)
outdir = mkdir(self.config['output']['plotdir'])
basename = os.path.basename(
self.config.mergefile.replace('.fits', '.png'))
outfile = os.path.join(outdir, basename)
plt.savefig(outfile)
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i,filename in enumerate(infiles):
logger.info('(%i/%i) %s'%(i+1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found"%len(data))
if not len(data): continue
columns = map(str.upper,data.dtype.names)
names,arrs = [],[]
if (lon_field in columns) and (lat_field in columns):
lon,lat = data[lon_field],data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon,lat = cel2gal(data['RA'],data['DEC'])
names += [lon_field,lat_field]
arrs += [lon,lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from GLON,GLAT"
lon,lat = gal2cel(data['GLON'],data['GLAT'])
names += [lon_field,lat_field]
arrs += [lon,lat]
cat_pix = ang2pix(nside_catalog,lon,lat)
pix_pix = ang2pix(nside_pixel,lon,lat)
cat_pix_name = 'PIX%i'%nside_catalog
pix_pix_name = 'PIX%i'%nside_pixel
try:
names += [cat_pix_name,pix_pix_name]
arrs += [cat_pix,pix_pix]
data=mlab.rec_append_fields(data,names=names,arrs=arrs)
except ValueError as e:
logger.warn(str(e)+'; not adding column.')
#data[cat_pix_name] = cat_pix
#data[pix_pix_name] = pix_pix
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s"%pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s"%outfile)
out=fitsio.FITS(outfile,mode='rw')
out.write(arr)
hdr=healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE','ORDERING','NSIDE','COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',pix,comment='HEALPIX pixel for this file')
else:
out=fitsio.FITS(outfile,mode='rw')
out[1].append(arr)
logger.debug("Writing %s"%outfile)
out.close()
def get_results(self,**kwargs):
kwargs.setdefault('alpha',self.alpha)
kwargs.setdefault('burn',self.nburn*self.nwalkers)
# Calculate best-fit parameters from MCMC chain
logger.debug('Estimating parameters...')
estimate = self.estimate_params(**kwargs)
params = {k:v[0] for k,v in estimate.items()}
results = dict(estimate)
# Extra parameters from the MCMC chain
logger.debug('Estimating auxiliary parameters...')
try:
results['ra'] = self.estimate('ra',**kwargs)
results['dec'] = self.estimate('dec',**kwargs)
except KeyError:
logger.warn("Didn't find 'ra' or 'dec'")
ra,dec = gal2cel(results['lon'][0],results['lat'][0])
results['ra'] = ugali.utils.stats.interval(ra)
results['dec'] = ugali.utils.stats.interval(dec)
ra,dec = results['ra'][0],results['dec'][0]
glon,glat = lon,lat = results['lon'][0],results['lat'][0]
results.update(gal=[float(glon),float(glat)])
results.update(cel=[float(ra),float(dec)])
try:
results['position_angle_cel'] = self.estimate('position_angle_cel',**kwargs)
except KeyError:
results['position_angle_cel'] = ugali.utils.stats.interval(np.nan)
# Update the loglike to the best-fit parameters from the chain
logger.debug('Calculating TS...')
ts = 2*self.loglike.value(**params)
results['ts'] = ugali.utils.stats.interval(ts,np.nan,np.nan)
#lon,lat = estimate['lon'][0],estimate['lat'][0]
#
#results.update(gal=[float(lon),float(lat)])
#ra,dec = gal2cel(lon,lat)
#results.update(cel=[float(ra),float(dec)])
#results['ra'] = ugali.utils.stats.interval(ra,np.nan,np.nan)
#results['dec'] = ugali.utils.stats.interval(dec,np.nan,np.nan)
# Celestial position angle
# Break ambiguity in direction with '% 180.'
pa,pa_err = results['position_angle']
pa_cel = gal2cel_angle(lon,lat,pa) % 180.
pa_cel_err = np.array(pa_err) - pa + pa_cel
results['position_angle_cel'] = ugali.utils.stats.interval(pa_cel,pa_cel_err[0],pa_cel_err[1])
mod,mod_err = estimate['distance_modulus']
dist = mod2dist(mod)
dist_lo,dist_hi = [mod2dist(mod_err[0]),mod2dist(mod_err[1])]
results['distance'] = ugali.utils.stats.interval(dist,dist_lo,dist_hi)
dist,dist_err = results['distance']
ext,ext_err = estimate['extension']
ext_sigma = np.nan_to_num(np.array(ext_err) - ext)
results['extension_arcmin'] = ugali.utils.stats.interval(60*ext,60*ext_err[0],60*ext_err[1])
# Radially symmetric extension (correct for ellipticity).
ell,ell_err = estimate['ellipticity']
rext,rext_err = ext*np.sqrt(1-ell),np.array(ext_err)*np.sqrt(1-ell)
rext_sigma = np.nan_to_num(np.array(rext_err) - rext)
results['extension_radial'] = ugali.utils.stats.interval(rext,rext_err[0],rext_err[1])
results['extension_radial_arcmin'] = ugali.utils.stats.interval(60*rext,60*rext_err[0],60*rext_err[1])
# Bayes factor for ellipticity
results['ellipticity_bayes_factor'] = self.bayes_factor('ellipticity',burn=kwargs['burn'])
# Physical Size (should do this with the posteriors)
# Radially symmetric
dist_sigma = np.nan_to_num(np.array(dist_err) - dist)
size = np.arctan(np.radians(ext)) * dist
size_sigma = size * np.sqrt((ext_sigma/ext)**2 + (dist_sigma/dist)**2)
size_err = [size-size_sigma[0],size+size_sigma[1]]
results['physical_size'] = ugali.utils.stats.interval(size,size_err[0],size_err[1])
rsize = np.arctan(np.radians(rext)) * dist
rsize_sigma = rsize * np.sqrt((rext_sigma/rext)**2 + (dist_sigma/dist)**2)
rsize_err = [rsize-rsize_sigma[0],rsize+rsize_sigma[1]]
results['physical_size_radial'] = ugali.utils.stats.interval(rsize,rsize_err[0],rsize_err[1])
# Richness
rich,rich_err = estimate['richness']
# Number of observed stars (sum of p-values)
nobs = self.loglike.p.sum()
nobs_lo,nobs_hi = nobs + np.sqrt(nobs)*np.array([-1,1])
results['nobs'] = ugali.utils.stats.interval(nobs,nobs_lo,nobs_hi)
# Number of predicted stars (pixelization effects?)
npred = self.loglike.f*rich
npred_lo,npred_hi = rich_err[0]*self.loglike.f,rich_err[1]*self.loglike.f
results['npred'] = ugali.utils.stats.interval(npred,npred_lo,npred_hi)
# Careful, depends on the isochrone...
stellar_mass = self.source.stellar_mass()
mass = rich*stellar_mass
mass_lo,mass_hi = rich_err[0]*stellar_mass,rich_err[1]*stellar_mass
results['mass'] = ugali.utils.stats.interval(mass,mass_lo,mass_hi)
stellar_luminosity = self.source.stellar_luminosity()
lum = rich*stellar_luminosity
lum_lo,lum_hi = rich_err[0]*stellar_luminosity,rich_err[1]*stellar_luminosity
results['luminosity'] = ugali.utils.stats.interval(lum,lum_lo,lum_hi)
# Absolute magnitude only calculated for DES isochrones with g,r
try:
Mv = self.source.absolute_magnitude(rich)
Mv_lo = self.source.absolute_magnitude(rich_err[0])
Mv_hi = self.source.absolute_magnitude(rich_err[1])
results['Mv'] = ugali.utils.stats.interval(Mv,Mv_lo,Mv_hi)
except ValueError as e:
logger.warning("Skipping absolute magnitude")
logger.warn(str(e))
results['Mv'] = np.nan
# ADW: WARNING this is very fragile.
# Also, this is not quite right, should cut on the CMD available space
kwargs = dict(richness=rich,mag_bright=16., mag_faint=23.,
n_trials=5000,alpha=self.alpha, seed=0)
martin = self.config['results'].get('martin')
if martin:
logger.info("Calculating Martin magnitude...")
if martin > 1: kwargs['n_trials'] = martin
Mv_martin = self.source.isochrone.absolute_magnitude_martin(**kwargs)
results['Mv_martin'] = Mv_martin
else:
logger.warning("Skipping Martin magnitude")
results['Mv_martin'] = np.nan
mu = surfaceBrightness(Mv, size, dist)
results['surface_brightness'] = ugali.utils.stats.interval(mu,np.nan,np.nan)
try:
results['constellation'] = ang2const(lon,lat,self.coordsys)[1]
except:
pass
results['iau'] = ugali.utils.projector.ang2iau(lon,lat)
coord = SkyCoord(ra*u.deg,dec*u.deg,distance=dist*u.kpc)
results['ra_sex'] = str(coord.ra.to_string())
results['dec_sex'] = str(coord.dec.to_string())
# Calculate some separations from GC, LMC, SMC
#NED coordinates with de Grisj distance
LMC = SkyCoord(80.8939*u.deg,-69.7561*u.deg,distance=49.89*u.kpc)
#NED coordinates with de Grisj distance
SMC = SkyCoord(13.1866*u.deg,-72.8286*u.deg,distance=61.94*u.kpc)
# GC from astropy?
GC = SkyCoord(266.4168262*u.deg,-29.0077969*u.deg,distance=8.0*u.kpc)
results['d_gc'] = coord.separation_3d(GC).value
results['d_lmc'] = coord.separation_3d(LMC).value
results['d_smc'] = coord.separation_3d(SMC).value
try:
results['feh'] = float(self.source.isochrone.feh)
except:
results['feh'] = np.nan
output = dict()
output['params'] = params
output['results'] = results
return output
def run(args):
try:
p.download(*args)
except Exception as e:
logger.warn(str(e))
logger.error("Download failed.")
def runone(self, i):
""" Run one simulation.
Parameters:
-----------
i : index of the simulation to run
Returns:
--------
results : result array
"""
results = self.results
results[i]['FLAG'] = FLAG_PROC
params = dict(list(zip(results[i].dtype.names, results[i])))
size = len(results)
lon, lat = params['RA'], params['DEC']
distance_modulus = params['DISTANCE_MODULUS']
mc_source_id = params['MC_SOURCE_ID']
extension = np.degrees(
np.arctan(params['R_PHYSICAL'] / params['DISTANCE']))
logger.info(
'\n(%i/%i); (id, lon, lat, mod, ext) = (%i, %.2f, %.2f, %.1f, %.3f)'
%
(i + 1, size, mc_source_id, lon, lat, distance_modulus, extension))
if params['EBV'] > 0.2:
msg = "High reddening region; skipping..."
logger.warn(msg)
results[i]['FLAG'] |= FLAG_EBV
#raise Exception(msg)
#results[i]['TS'] = np.nan
#return
# This links to the parameters in the data scan
section = 'scan'
# This links to the parameters in the simulate section
#section = 'simulate'
source = ugali.analysis.loglike.createSource(self.config,
section=section,
lon=lon,
lat=lat)
logger.info("Reading data catalog...")
obs = ugali.analysis.loglike.createObservation(self.config,
lon=lon,
lat=lat)
# Select just the simulated target of interest
logger.info("Merging simulated catalog...")
data = self.catalog.data[self.catalog.mc_source_id ==
mc_source_id].copy()
data = np.array(data[list(obs.catalog.data.dtype.names)],
dtype=obs.catalog.data.dtype)
obs.catalog = ugali.observation.catalog.Catalog(
self.config, data=np.concatenate([obs.catalog.data, data]))
loglike = ugali.analysis.loglike.LogLikelihood(self.config, obs,
source)
# Mitigate memory overflow issues by cutting objects with
# too many catalog stars
if len(loglike.catalog) > 5e5: # 1e5
msg = "Large catalog (N_CATALOG = %i)." % len(loglike.catalog)
logger.warn(msg)
results[i]['FLAG'] |= FLAG_NOBJ
grid = ugali.analysis.scan.GridSearch(self.config, loglike)
self.grid = grid
self.loglike = self.grid.loglike
pix = self.loglike.roi.indexTarget(lon, lat)
# ADW: Should fit_distance be free in order to model search procedure?
if self.config['simulate'].get('fit_distance', False):
fit_distance = None
else:
idx = np.fabs(grid.distance_modulus_array -
distance_modulus).argmin()
fit_distance = grid.distance_modulus_array[idx]
try:
grid.search(coords=(lon, lat), distance_modulus=fit_distance)
results[i]['FLAG'] &= ~FLAG_FIT
except ValueError as e:
logger.error(str(e))
results[i]['FLAG'] |= FLAG_FIT
mle = grid.mle()
distance_idx = np.fabs(grid.distance_modulus_array -
mle['distance_modulus']).argmin()
ts = 2 * grid.loglike_array[distance_idx][pix]
results[i]['FIT_KERNEL'] = grid.loglike.kernel.name
results[i]['TS'] = ts
results[i]['FIT_MASS'] = grid.stellar_mass_conversion * mle['richness']
results[i]['FIT_DISTANCE'] = mle['distance_modulus']
results[i]['FIT_EXTENSION'] = grid.loglike.kernel.extension
err = grid.err()
richness_err = (err['richness'][1] - err['richness'][0]) / 2.
results[i][
'FIT_MASS_ERR'] = grid.stellar_mass_conversion * richness_err
distance_modulus_err = (err['distance_modulus'][1] -
err['distance_modulus'][0]) / 2.
results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err
"""
fit_extension = [0.02, 0.07, 0.15] # half light radii (deg)
# ADW: This won't work since we add extension inside the search loop
for ext in fit_extension:
grid.loglike.set_params(extension = ext)
# Fit failures are often due to fracdet = 0
try:
grid.search(coords=(lon,lat),distance_modulus=fit_distance)
results[i]['FLAG'] &= ~FLAG_FIT
except ValueError as e:
logger.error(str(e))
results[i]['FLAG'] |= FLAG_FIT
continue
mle = grid.mle()
ts = 2*grid.loglike_array[distance_idx][pix]
if ts <= results[i]['TS']:
logger.info("No TS increase; continuing...")
continue
results[i]['FIT_KERNEL'] = grid.loglike.kernel.name
results[i]['TS'] = ts
results[i]['FIT_MASS'] = grid.stellar_mass_conversion*mle['richness']
results[i]['FIT_DISTANCE'] = fit_distance #mle['distance_modulus']
results[i]['FIT_EXTENSION'] = ext
err = grid.err()
richness_err = (err['richness'][1]-err['richness'][0])/2.
results[i]['FIT_MASS_ERR'] = grid.stellar_mass_conversion*richness_err
distance_modulus_err = (err['distance_modulus'][1]-err['distance_modulus'][0])/2.
results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err
"""
return results
def run(self, population=None, catalog=None, outfile=None, mc_source_id=None):
if not population: population = self.population
if not catalog: catalog = self.catalog
if mc_source_id is not None:
sel = np.in1d(population['MC_SOURCE_ID'],mc_source_id)
if not sel.sum():
msg = "Requested MC_SOURCE_ID not found: %i"%mc_source_id
logger.warn(msg)
return
else:
# Select only systems that are in the catalog
sel = np.in1d(population['MC_SOURCE_ID'],catalog.mc_source_id)
population = population[sel]
size = len(population)
dtype=[('KERNEL','S18'),('TS','>f4'),('FIT_KERNEL','S18'),
('FIT_EXTENSION','>f4'),('FIT_MASS','>f4'),('FIT_MASS_ERR','>f4'),
('FIT_DISTANCE','>f4'),('FIT_DISTANCE_ERR','>f4'),('FLAG','>i8')]
results = np.array(np.nan*np.ones(size),dtype=dtype)
results = recfuncs.merge_arrays([population,results],flatten=True,asrecarray=False,usemask=False)
results['TS'] = -np.inf
results['FLAG'] = 0
self.results = results
if outfile:
ugali.utils.fileio.write(outfile,results,clobber=True)
for i,d in enumerate(results):
params = dict(list(zip(results.dtype.names,d)))
lon,lat = params['RA'],params['DEC']
distance_modulus = params['DISTANCE_MODULUS']
mc_source_id = params['MC_SOURCE_ID']
extension=np.degrees(np.arctan(params['R_PHYSICAL']/params['DISTANCE']))
logger.info('\n(%i/%i); (id, lon, lat, mod, ext) = (%i, %.2f, %.2f, %.1f, %.3f)'%(i+1,size,mc_source_id,lon,lat,distance_modulus,extension))
if params['EBV'] > 0.2:
msg = "High reddening region E(B-V) > 0.2; skipping..."
logger.info(msg)
results[i]['FLAG'] |= FLAG_EBV
results[i]['TS'] = np.nan
continue
source = ugali.analysis.loglike.createSource(self.config,section='scan',lon=lon,lat=lat)
logger.info("Reading data catalog...")
obs = ugali.analysis.loglike.createObservation(self.config,lon=lon,lat=lat)
# Select just the simulated target of interest
logger.info("Merging simulated catalog...")
data = catalog.data[catalog.mc_source_id == mc_source_id].copy()
data = np.array(data[list(obs.catalog.data.dtype.names)],dtype=obs.catalog.data.dtype)
obs.catalog = ugali.observation.catalog.Catalog(self.config, data=np.concatenate([obs.catalog.data,data]))
# Index of closest distance modulus
loglike = ugali.analysis.loglike.LogLikelihood(self.config,obs,source)
grid = ugali.analysis.scan.GridSearch(self.config,loglike)
#if len(loglike.catalog) > 1.0e5:
if len(loglike.catalog) > 5.0e5:
msg = "High stellar density (N_CATALOG = %i); skipping..."%len(loglike.catalog)
logger.warn(msg)
results[i]['FLAG'] |= FLAG_DENSITY
results[i]['TS'] = np.nan
continue
self.grid = grid
self.loglike = self.grid.loglike
pix = self.loglike.roi.indexTarget(lon,lat)
# ADW: Should allow fit_distance to float in order to model search procedure?
# Currently we are evaluating at the true distance.
distance_idx = np.fabs(grid.distance_modulus_array-distance_modulus).argmin()
fit_distance = grid.distance_modulus_array[distance_idx]
fit_extension = [0.02, 0.07, 0.15] # half light radii (deg)
for ext in fit_extension:
grid.loglike.set_params(extension = ext)
try:
grid.search(coords=(lon,lat),distance_modulus=fit_distance)
results[i]['FLAG'] &= ~FLAG_FRACDET
except ValueError as e:
logger.error(str(e))
results[i]['FLAG'] |= FLAG_FRACDET
continue
mle = grid.mle()
ts = 2*grid.log_likelihood_sparse_array[distance_idx][pix]
if ts <= results[i]['TS']:
logger.info("No TS increase; continuing...")
continue
results[i]['FIT_KERNEL'] = grid.loglike.kernel.name
results[i]['TS'] = ts
results[i]['FIT_MASS'] = grid.stellar_mass_conversion*mle['richness']
results[i]['FIT_DISTANCE'] = fit_distance #mle['distance_modulus']
results[i]['FIT_EXTENSION'] = ext
err = grid.err()
richness_err = (err['richness'][1]-err['richness'][0])/2.
results[i]['FIT_MASS_ERR'] = grid.stellar_mass_conversion*richness_err
distance_modulus_err = (err['distance_modulus'][1]-err['distance_modulus'][0])/2.
results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err
logger.info("Fit parameter values:")
for d in dtype:
logger.info('\t%s: %s'%(d[0], results[i][d[0]]))
if (i%self.config['simulate']['save'])==0 and outfile:
ugali.utils.fileio.write(outfile,results)
if outfile: ugali.utils.fileio.write(outfile,results,clobber=True)
return results
def get_results(self, **kwargs):
kwargs.setdefault('alpha', self.alpha)
kwargs.setdefault('burn', self.nburn * self.nwalkers)
# Calculate best-fit parameters from MCMC chain
logger.debug('Estimating parameters...')
estimate = self.estimate_params(**kwargs)
params = {k: v[0] for k, v in estimate.items()}
results = dict(estimate)
# Extra parameters from the MCMC chain
logger.debug('Estimating auxiliary parameters...')
try:
results['ra'] = self.estimate('ra', **kwargs)
results['dec'] = self.estimate('dec', **kwargs)
except KeyError:
logger.warn("Didn't find 'ra' or 'dec'")
ra, dec = gal2cel(results['lon'][0], results['lat'][0])
results['ra'] = ugali.utils.stats.interval(ra)
results['dec'] = ugali.utils.stats.interval(dec)
ra, dec = results['ra'][0], results['dec'][0]
glon, glat = lon, lat = results['lon'][0], results['lat'][0]
results.update(gal=[float(glon), float(glat)])
results.update(cel=[float(ra), float(dec)])
try:
results['position_angle_cel'] = self.estimate(
'position_angle_cel', **kwargs)
except KeyError:
results['position_angle_cel'] = ugali.utils.stats.interval(np.nan)
# Update the loglike to the best-fit parameters from the chain
logger.debug('Calculating TS...')
ts = 2 * self.loglike.value(**params)
results['ts'] = ugali.utils.stats.interval(ts, np.nan, np.nan)
#lon,lat = estimate['lon'][0],estimate['lat'][0]
#
#results.update(gal=[float(lon),float(lat)])
#ra,dec = gal2cel(lon,lat)
#results.update(cel=[float(ra),float(dec)])
#results['ra'] = ugali.utils.stats.interval(ra,np.nan,np.nan)
#results['dec'] = ugali.utils.stats.interval(dec,np.nan,np.nan)
# Celestial position angle
# Break ambiguity in direction with '% 180.'
pa, pa_err = results['position_angle']
pa_cel = gal2cel_angle(lon, lat, pa) % 180.
pa_cel_err = np.array(pa_err) - pa + pa_cel
results['position_angle_cel'] = ugali.utils.stats.interval(
pa_cel, pa_cel_err[0], pa_cel_err[1])
mod, mod_err = estimate['distance_modulus']
dist = mod2dist(mod)
dist_lo, dist_hi = [mod2dist(mod_err[0]), mod2dist(mod_err[1])]
results['distance'] = ugali.utils.stats.interval(
dist, dist_lo, dist_hi)
dist, dist_err = results['distance']
ext, ext_err = estimate['extension']
ext_sigma = np.nan_to_num(np.array(ext_err) - ext)
results['extension_arcmin'] = ugali.utils.stats.interval(
60 * ext, 60 * ext_err[0], 60 * ext_err[1])
# Radially symmetric extension (correct for ellipticity).
ell, ell_err = estimate['ellipticity']
rext, rext_err = ext * np.sqrt(1 -
ell), np.array(ext_err) * np.sqrt(1 -
ell)
rext_sigma = np.nan_to_num(np.array(rext_err) - rext)
results['extension_radial'] = ugali.utils.stats.interval(
rext, rext_err[0], rext_err[1])
results['extension_radial_arcmin'] = ugali.utils.stats.interval(
60 * rext, 60 * rext_err[0], 60 * rext_err[1])
# Bayes factor for ellipticity
results['ellipticity_bayes_factor'] = self.bayes_factor(
'ellipticity', burn=kwargs['burn'])
# Physical Size (should do this with the posteriors)
# Radially symmetric
dist_sigma = np.nan_to_num(np.array(dist_err) - dist)
size = np.arctan(np.radians(ext)) * dist
size_sigma = size * np.sqrt((ext_sigma / ext)**2 +
(dist_sigma / dist)**2)
size_err = [size - size_sigma[0], size + size_sigma[1]]
results['physical_size'] = ugali.utils.stats.interval(
size, size_err[0], size_err[1])
rsize = np.arctan(np.radians(rext)) * dist
rsize_sigma = rsize * np.sqrt((rext_sigma / rext)**2 +
(dist_sigma / dist)**2)
rsize_err = [rsize - rsize_sigma[0], rsize + rsize_sigma[1]]
results['physical_size_radial'] = ugali.utils.stats.interval(
rsize, rsize_err[0], rsize_err[1])
# Richness
rich, rich_err = estimate['richness']
# Number of observed stars (sum of p-values)
nobs = self.loglike.p.sum()
nobs_lo, nobs_hi = nobs + np.sqrt(nobs) * np.array([-1, 1])
results['nobs'] = ugali.utils.stats.interval(nobs, nobs_lo, nobs_hi)
# Number of predicted stars (pixelization effects?)
npred = self.loglike.f * rich
npred_lo, npred_hi = rich_err[0] * self.loglike.f, rich_err[
1] * self.loglike.f
results['npred'] = ugali.utils.stats.interval(npred, npred_lo,
npred_hi)
# Careful, depends on the isochrone...
stellar_mass = self.source.stellar_mass()
mass = rich * stellar_mass
mass_lo, mass_hi = rich_err[0] * stellar_mass, rich_err[
1] * stellar_mass
results['mass'] = ugali.utils.stats.interval(mass, mass_lo, mass_hi)
stellar_luminosity = self.source.stellar_luminosity()
lum = rich * stellar_luminosity
lum_lo, lum_hi = rich_err[0] * stellar_luminosity, rich_err[
1] * stellar_luminosity
results['luminosity'] = ugali.utils.stats.interval(lum, lum_lo, lum_hi)
Mv = self.source.absolute_magnitude(rich)
Mv_lo = self.source.absolute_magnitude(rich_err[0])
Mv_hi = self.source.absolute_magnitude(rich_err[1])
results['Mv'] = ugali.utils.stats.interval(Mv, Mv_lo, Mv_hi)
# ADW: WARNING this is very fragile.
# Also, this is not quite right, should cut on the CMD available space
kwargs = dict(richness=rich,
mag_bright=16.,
mag_faint=23.,
n_trials=5000,
alpha=self.alpha,
seed=0)
martin = self.config['results'].get('martin')
if martin:
logger.info("Calculating Martin magnitude...")
if martin > 1: kwargs['n_trials'] = martin
Mv_martin = self.source.isochrone.absolute_magnitude_martin(
**kwargs)
results['Mv_martin'] = Mv_martin
else:
logger.warning("Skipping Martin magnitude")
results['Mv_martin'] = np.nan
mu = surfaceBrightness(Mv, size, dist)
results['surface_brightness'] = ugali.utils.stats.interval(
mu, np.nan, np.nan)
try:
results['constellation'] = ugali.utils.projector.ang2const(
lon, lat)[1]
except:
pass
results['iau'] = ugali.utils.projector.ang2iau(lon, lat)
coord = SkyCoord(ra * u.deg, dec * u.deg, distance=dist * u.kpc)
results['ra_sex'] = str(coord.ra.to_string())
results['dec_sex'] = str(coord.dec.to_string())
# Calculate some separations from GC, LMC, SMC
#NED coordinates with de Grisj distance
LMC = SkyCoord(80.8939 * u.deg,
-69.7561 * u.deg,
distance=49.89 * u.kpc)
#NED coordinates with de Grisj distance
SMC = SkyCoord(13.1866 * u.deg,
-72.8286 * u.deg,
distance=61.94 * u.kpc)
# GC from astropy?
GC = SkyCoord(266.4168262 * u.deg,
-29.0077969 * u.deg,
distance=8.0 * u.kpc)
results['d_gc'] = coord.separation_3d(GC).value
results['d_lmc'] = coord.separation_3d(LMC).value
results['d_smc'] = coord.separation_3d(SMC).value
try:
results['feh'] = float(self.source.isochrone.feh)
except:
results['feh'] = np.nan
output = dict()
output['params'] = params
output['results'] = results
return output
def runall(self, outfile=None, mc_source_id=None, rerun=False):
"""Run all sources in population.
Parameters:
-----------
outfile : file to write output to
mc_source_id : list of sources to process (None is all sources)
Returns:
--------
results : processing results
"""
if mc_source_id is None:
mc_source_id = np.unique(self.catalog.mc_source_id)
# Select only systems that are in the catalog
sel = np.in1d(self.population['MC_SOURCE_ID'], mc_source_id)
if not sel.sum():
msg = "Requested MC_SOURCE_IDs not found in population."
raise ValueError(msg)
if os.path.exists(outfile) and rerun:
# read the results from the existing outfile
self.results = self.read_results(outfile)
else:
# create the results
self.results = self.create_results(population=self.population[sel])
if not np.in1d(mc_source_id, self.results['MC_SOURCE_ID']).all():
msg = "Requested MC_SOURCE_IDs not found in results."
raise ValueError(msg)
if outfile:
logger.info("Writing %s..." % outfile)
self.write_results(outfile, clobber=True)
for i, r in enumerate(self.results):
# Skip if not in mc_source_id list
if self.results[i]['MC_SOURCE_ID'] not in mc_source_id:
msg = "%i skipped." % self.results[i]['MC_SOURCE_ID']
logger.info(msg)
continue
# Rerun in NOPROC or MEM flagged
if (self.results[i]['FLAG'] & (FLAG_NOPROC | FLAG_MEM)) == 0:
msg = "%i already processed." % self.results[i]['MC_SOURCE_ID']
logger.info(msg)
continue
start_time = time.time()
try:
self.runone(i)
except MemoryError as e:
msg = "Memory usage exceeded %.3f GB" % (self.mlimit / GB)
logger.warn(msg)
self.results[i]['FLAG'] |= FLAG_MEM
except Exception as e:
logger.error(str(e))
self.results[i]['FLAG'] |= FLAG_FIT
runtime = time.time() - start_time
self.results[i]['MEMORY'] = self.get_memory_usage()
self.results[i]['RUNTIME'] = runtime
logger.info("Fit parameter values:")
for d in DTYPES:
logger.info('\t%s: %s' % (d[0], self.results[i][d[0]]))
logger.info("Memory usage: %.3f GB" %
(self.get_memory_usage() / GB))
if (i % self.config['simulate']['save']) == 0 and outfile:
logger.info("Writing %s..." % outfile)
self.write_results(outfile, clobber=True)
if outfile:
logger.info("Writing %s..." % outfile)
self.write_results(outfile, clobber=True)
return self.results
def run(self,
population=None,
catalog=None,
outfile=None,
mc_source_id=None):
if not population: population = self.population
if not catalog: catalog = self.catalog
if mc_source_id is not None:
sel = np.in1d(population['MC_SOURCE_ID'], mc_source_id)
if not sel.sum():
msg = "Requested MC_SOURCE_ID not found: %i" % mc_source_id
logger.warn(msg)
return
else:
# Select only systems that are in the catalog
sel = np.in1d(population['MC_SOURCE_ID'], catalog.mc_source_id)
population = population[sel]
size = len(population)
dtype = [('KERNEL', 'S18'), ('TS', '>f4'), ('FIT_KERNEL', 'S18'),
('FIT_EXTENSION', '>f4'), ('FIT_MASS', '>f4'),
('FIT_MASS_ERR', '>f4'), ('FIT_DISTANCE', '>f4'),
('FIT_DISTANCE_ERR', '>f4'), ('FLAG', '>i8')]
results = np.array(np.nan * np.ones(size), dtype=dtype)
results = recfuncs.merge_arrays([population, results],
flatten=True,
asrecarray=False,
usemask=False)
results['TS'] = -np.inf
results['FLAG'] = 0
self.results = results
if outfile:
ugali.utils.fileio.write(outfile, results, clobber=True)
for i, d in enumerate(results):
params = dict(list(zip(results.dtype.names, d)))
lon, lat = params['RA'], params['DEC']
distance_modulus = params['DISTANCE_MODULUS']
mc_source_id = params['MC_SOURCE_ID']
extension = np.degrees(
np.arctan(params['R_PHYSICAL'] / params['DISTANCE']))
logger.info(
'\n(%i/%i); (id, lon, lat, mod, ext) = (%i, %.2f, %.2f, %.1f, %.3f)'
% (i + 1, size, mc_source_id, lon, lat, distance_modulus,
extension))
if params['EBV'] > 0.2:
msg = "High reddening region E(B-V) > 0.2; skipping..."
logger.info(msg)
results[i]['FLAG'] |= FLAG_EBV
results[i]['TS'] = np.nan
continue
source = ugali.analysis.loglike.createSource(self.config,
section='scan',
lon=lon,
lat=lat)
logger.info("Reading data catalog...")
obs = ugali.analysis.loglike.createObservation(self.config,
lon=lon,
lat=lat)
# Select just the simulated target of interest
logger.info("Merging simulated catalog...")
data = catalog.data[catalog.mc_source_id == mc_source_id].copy()
data = np.array(data[list(obs.catalog.data.dtype.names)],
dtype=obs.catalog.data.dtype)
obs.catalog = ugali.observation.catalog.Catalog(
self.config, data=np.concatenate([obs.catalog.data, data]))
# Index of closest distance modulus
loglike = ugali.analysis.loglike.LogLikelihood(
self.config, obs, source)
grid = ugali.analysis.scan.GridSearch(self.config, loglike)
#if len(loglike.catalog) > 1.0e5:
if len(loglike.catalog) > 5.0e5:
msg = "High stellar density (N_CATALOG = %i); skipping..." % len(
loglike.catalog)
logger.warn(msg)
results[i]['FLAG'] |= FLAG_DENSITY
results[i]['TS'] = np.nan
continue
self.grid = grid
self.loglike = self.grid.loglike
pix = self.loglike.roi.indexTarget(lon, lat)
# ADW: Should allow fit_distance to float in order to model search procedure?
# Currently we are evaluating at the true distance.
distance_idx = np.fabs(grid.distance_modulus_array -
distance_modulus).argmin()
fit_distance = grid.distance_modulus_array[distance_idx]
fit_extension = [0.02, 0.07, 0.15] # half light radii (deg)
for ext in fit_extension:
grid.loglike.set_params(extension=ext)
try:
grid.search(coords=(lon, lat),
distance_modulus=fit_distance)
results[i]['FLAG'] &= ~FLAG_FRACDET
except ValueError as e:
logger.error(str(e))
results[i]['FLAG'] |= FLAG_FRACDET
continue
mle = grid.mle()
ts = 2 * grid.log_likelihood_sparse_array[distance_idx][pix]
if ts <= results[i]['TS']:
logger.info("No TS increase; continuing...")
continue
results[i]['FIT_KERNEL'] = grid.loglike.kernel.name
results[i]['TS'] = ts
results[i]['FIT_MASS'] = grid.stellar_mass_conversion * mle[
'richness']
results[i][
'FIT_DISTANCE'] = fit_distance #mle['distance_modulus']
results[i]['FIT_EXTENSION'] = ext
err = grid.err()
richness_err = (err['richness'][1] - err['richness'][0]) / 2.
results[i][
'FIT_MASS_ERR'] = grid.stellar_mass_conversion * richness_err
distance_modulus_err = (err['distance_modulus'][1] -
err['distance_modulus'][0]) / 2.
results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err
logger.info("Fit parameter values:")
for d in dtype:
logger.info('\t%s: %s' % (d[0], results[i][d[0]]))
if (i % self.config['simulate']['save']) == 0 and outfile:
ugali.utils.fileio.write(outfile, results)
if outfile: ugali.utils.fileio.write(outfile, results, clobber=True)
return results
def run(self):
if 'scan' in self.opts.run:
logger.info("Running 'scan'...")
farm = Farm(self.config, verbose=self.opts.verbose)
farm.submit_all(coords=self.opts.coords,
queue=self.opts.queue,
debug=self.opts.debug)
if 'merge' in self.opts.run:
logger.info("Running 'merge'...")
mergefile = self.config.mergefile
roifile = self.config.roifile
filenames = self.config.likefile.split('_%')[0] + '_*.fits'
infiles = np.array(sorted(glob.glob(filenames)))
if 'mergedir' in self.config['output']:
mkdir(self.config['output']['mergedir'])
pixels = np.char.rpartition(
np.char.rpartition(infiles, '_')[:, 0], '_')[:, -1]
pixels = pixels.astype(int)
superpixel = healpix.superpixel(
pixels, self.config['coords']['nside_likelihood'],
self.config['coords']['nside_merge'])
for pix in np.unique(superpixel):
outfile = mergefile % pix
if exists(outfile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % outfile)
else:
healpix.merge_partial_maps(infiles[superpixel == pix],
outfile,
multiproc=8)
if exists(roifile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % roifile)
else:
logger.info(" Merging likelihood headers...")
healpix.merge_likelihood_headers(infiles, roifile)
if 'tar' in self.opts.run:
logger.info("Running 'tar'...")
outdir = mkdir(self.config['output']['likedir'])
logdir = mkdir(join(outdir, 'log'))
scanfile = self.config.likefile.split('_%')[0] + '_[0-9]*.fits'
tarfile = join(self.config.likefile.split('_%')[0] + '_pixels.tar.gz')
jobname = 'tar'
logfile = os.path.join(logdir, 'scan_tar.log')
cmd = 'tar --remove-files -cvzf %s %s' % (tarfile, scanfile)
if exists(tarfile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % tarfile)
else:
logger.info(" Tarring likelihood files...")
logger.info(cmd)
self.batch.submit(cmd, jobname, logfile)
if 'plot' in self.opts.run:
# WARNING: Loading the full 3D healpix map is memory intensive.
logger.info("Running 'plot'...")
# Should do this in environment variable
import matplotlib
matplotlib.use('Agg')
import pylab as plt
import ugali.utils.plotting as plotting
skymap = ugali.utils.skymap.readSparseHealpixMap(
self.config.mergefile, 'LOG_LIKELIHOOD')[1]
plotting.plotSkymap(skymap)
outdir = mkdir(self.config['output']['plotdir'])
basename = os.path.basename(
self.config.mergefile.replace('.fits', '.png'))
outfile = os.path.join(outdir, basename)
plt.savefig(outfile)
if 'check' in self.opts.run:
# Check the completion fraction
logger.info("Running 'check'...")
import fitsio
import numpy as np
import healpy as hp
from ugali.utils.skymap import inFootprint
# Load the ROI file
roi = fitsio.read(self.config.roifile)
done = roi['PIXEL']
# Get all target pixels
nside = self.config['coords']['nside_likelihood']
pixels = np.arange(hp.nside2npix(nside))
pixarea = hp.nside2pixarea(nside, degrees=True)
foot = pixels[inFootprint(self.config, pixels)]
# And find the pixels that haven't been processed
undone = ~np.in1d(foot, done)
hpxmap = np.zeros(len(pixels))
hpxmap[foot[undone]] = True
logger.info("Found %i incomplete pixels with an area of %.1f deg^2." %
(hpxmap.sum(), hpxmap.sum() * pixarea))
hp.write_map('check.fits.gz', hpxmap)
def _createFilenames(self,pixels=None):
"""
Create a masked records array of all filenames for the given set of
pixels and store the existence of those files in the mask values.
Examples:
f = getFilenames([1,2,3])
# All possible catalog files
f['catalog'].data
# All existing catalog files
f['catalog'][~f.mask['catalog']]
# or
f['catalog'].compressed()
# All missing mask_1 files
f['mask_1'][f.mask['mask_1']]
# Pixels where all files exist
f['pix'][~f.mask['pix']]
Parameters:
-----------
pixels : If pixels is None, grab all pixels of 'nside_catalog'.
Returns:
--------
recarray : pixels and mask value
"""
nside_catalog = self['coords']['nside_catalog']
# Deprecated: ADW 2018-06-17
#if nside_catalog is None:
# pixels = [None]
if pixels is not None:
pixels = [pixels] if np.isscalar(pixels) else pixels
else:
pixels = np.arange(hp.nside2npix(nside_catalog))
npix = len(pixels)
catalog_dir = self['catalog']['dirname']
catalog_base = self['catalog']['basename']
mask_dir = self['mask']['dirname']
mask_base_1 = self['mask']['basename_1']
mask_base_2 = self['mask']['basename_2']
data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object),
('mask_1',object), ('mask_2',object)])
mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool),
('mask_1',bool), ('mask_2',bool)])
for ii,pix in enumerate(pixels):
if pix is None:
# DEPRECTATED: ADW 2018-06-17
# This is not really being used anymore
catalog = os.path.join(catalog_dir,catalog_base)
mask_1 = os.path.join(mask_dir,mask_base_1)
mask_2 = os.path.join(mask_dir,mask_base_2)
else:
catalog = os.path.join(catalog_dir,catalog_base%pix)
mask_1 = os.path.join(mask_dir,mask_base_1%pix)
mask_2 = os.path.join(mask_dir,mask_base_2%pix)
data[ii]['pix'] = pix if pix is not None else -1
data[ii]['catalog'] = catalog
data[ii]['mask_1'] = mask_1
data[ii]['mask_2'] = mask_2
mask[ii]['catalog'] = not os.path.exists(catalog)
mask[ii]['mask_1'] = not os.path.exists(mask_1)
mask[ii]['mask_2'] = not os.path.exists(mask_2)
for name in ['catalog','mask_1','mask_2']:
if np.all(mask[name]): logger.warn("All '%s' files masked"%name)
# mask 'pix' if all files not present
mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2']
if np.all(mask['pix']): logger.warn("All pixels masked")
#return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,None,None,None])
#return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,'','',''])
return np.ma.MaskedArray(data, mask, fill_value=[-1,'','',''])
def run(self):
if self.opts.coords is not None:
coords = self.opts.coords
names = vars(self.opts).get('names', len(coords) * [''])
else:
names, coords = self.parser.parse_targets(self.config.candfile)
labels = [
n.lower().replace(' ', '_').replace('(', '').replace(')', '')
for n in names
]
self.outdir = mkdir(self.config['output']['mcmcdir'])
self.logdir = mkdir(join(self.outdir, 'log'))
args = list(zip(len(names) * [self.opts.config], names, labels, coords))
if 'mcmc' in self.opts.run:
logger.info("Running 'mcmc'...")
try:
shutil.copy(self.opts.config, self.outdir)
except Exception as e:
logger.warn(e.message)
for config, name, label, coord in args:
glon, glat, radius = coord
outfile = make_filenames(self.config, label)['samfile']
base = splitext(basename(outfile))[0]
logfile = join(self.logdir, base + '.log')
jobname = base
script = self.config['mcmc']['script']
nthreads = self.config['mcmc']['nthreads']
srcmdl = self.config['mcmc'].get('srcmdl')
if srcmdl is not None:
try:
shutil.copy(srcmdl, self.outdir)
except Exception as e:
logger.warn(e.message)
logger.info('%s (%s)' % (name, srcmdl))
cmd = '%s %s --name %s --srcmdl %s %s' % (
script, self.opts.config, name, srcmdl, outfile)
else:
logger.info('%s (%.4f,%.4f)' % (name, glon, glat))
cmd = '%s %s --name %s --gal %.4f %.4f --grid %s' % (
script, self.opts.config, name, glon, glat, outfile)
logger.info(cmd)
self.batch.submit(cmd, jobname, logfile, n=nthreads, a='mpirun')
if 'results' in self.opts.run:
logger.info("Running 'results'...")
if len(args) > 1:
pool = Pool(maxtasksperchild=1)
pool.map(do_results, args)
else:
do_results(*args)
if 'membership' in self.opts.run:
logger.info("Running 'membership'...")
if len(args) > 1:
pool = Pool(maxtasksperchild=1)
pool.map(do_membership, args)
else:
do_membership(*args)
if 'plot' in self.opts.run:
logger.info("Running 'plot'...")
if len(args) > 1:
pool = Pool(maxtasksperchild=1)
pool.map(do_plot, args)
#map(do_plot,args)
else:
do_plot(*args)
if 'collect' in self.opts.run:
logger.info("Running 'collect'...")
results = odict()
srcmdl = odict()
params = odict()
for config, name, label, coord in args:
srcfile = make_filenames(self.config, name)['srcfile']
results[name] = yaml.load(open(srcfile))['results']
srcmdl[name] = yaml.load(open(srcfile))['source']
params[name] = yaml.load(open(srcfile))['params']
for base, output in [('results.yaml', results),
('srcmdl.yaml', srcmdl), ('params.yaml', params)]:
outfile = join(self.outdir, base)
out = open(outfile, 'w')
out.write(yaml.dump(output))
out.close()
if 'scan' in self.opts.run:
logger.info("Running 'scan'...")
for config, name, label, coord in args:
logdir = mkdir('plots/log')
logfile = join(logdir, '%s_lnlscan.log')
cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45' % (
self.opts.config, name, 'age', 'metallicity')
self.batch.submit(cmd, logfile=logfile)
cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45' % (
self.opts.config, name, 'metallicity', 'distance_modulus')
self.batch.submit(cmd, logfile=logfile)
cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45' % (
self.opts.config, name, 'age', 'distance_modulus')
self.batch.submit(cmd, logfile=logfile)
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i, filename in enumerate(infiles):
logger.info('(%i/%i) %s' % (i + 1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found" % len(data))
if not len(data): continue
columns = map(str.upper, data.dtype.names)
names, arrs = [], []
if (lon_field in columns) and (lat_field in columns):
lon, lat = data[lon_field], data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon, lat = cel2gal(data['RA'], data['DEC'])
names += [lon_field, lat_field]
arrs += [lon, lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field)
msg += "\nConverting from GLON,GLAT"
lon, lat = gal2cel(data['GLON'], data['GLAT'])
names += [lon_field, lat_field]
arrs += [lon, lat]
cat_pix = ang2pix(nside_catalog, lon, lat)
pix_pix = ang2pix(nside_pixel, lon, lat)
cat_pix_name = 'PIX%i' % nside_catalog
pix_pix_name = 'PIX%i' % nside_pixel
try:
names += [cat_pix_name, pix_pix_name]
arrs += [cat_pix, pix_pix]
data = mlab.rec_append_fields(data, names=names, arrs=arrs)
except ValueError as e:
logger.warn(str(e) + '; not adding column.')
#data[cat_pix_name] = cat_pix
#data[pix_pix_name] = pix_pix
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s" % pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s" % outfile)
out = fitsio.FITS(outfile, mode='rw')
out.write(arr)
hdr = healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE', 'ORDERING', 'NSIDE', 'COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',
pix,
comment='HEALPIX pixel for this file')
else:
out = fitsio.FITS(outfile, mode='rw')
out[1].append(arr)
logger.debug("Writing %s" % outfile)
out.close()
