def sample_lonlat(self, n):
"""
Sample 2D distribution of points in lon, lat
"""
# From http://en.wikipedia.org/wiki/Ellipse#General_parametric_form
# However, Martin et al. (2009) use PA theta "from North to East"
# Definition of phi (position angle) is offset by pi/4
# Definition of t (eccentric anamoly) remains the same (x,y-frame usual)
# In the end, everything is trouble because we use glon, glat...
radius = self.sample_radius(n)
a = radius; b = self.jacobian * radius
t = 2. * np.pi * numpy.random.rand(n)
cost,sint = np.cos(t),np.sin(t)
phi = np.pi/2. - np.deg2rad(self.theta)
cosphi,sinphi = np.cos(phi),np.sin(phi)
x = a*cost*cosphi - b*sint*sinphi
y = a*cost*sinphi + b*sint*cosphi
if self.projector is None:
logger.debug("Creating AITOFF projector for sampling")
projector = Projector(self.lon,self.lat,'ait')
else:
projector = self.projector
lon, lat = projector.imageToSphere(x, y)
return lon, lat
def load_infiles(infiles,columns=None,multiproc=False):
if isinstance(infiles,str):
infiles = [infiles]
logger.debug("Loading %s files..."%len(infiles))
args = list(zip(infiles,len(infiles)*[columns]))
if multiproc:
from multiprocessing import Pool
processes = multiproc if multiproc > 0 else None
p = Pool(processes,maxtasksperchild=1)
out = p.map(load,args)
else:
out = [load(arg) for arg in args]
dtype = out[0].dtype
for i,d in enumerate(out):
if d.dtype != dtype:
# ADW: Not really safe...
logger.warn("Casting input data to same type.")
out[i] = d.astype(dtype)
logger.debug('Concatenating arrays...')
return np.concatenate(out)
def simple_maglims(config,dirname='simple',force=False):
"""
Create simple, uniform magnitude limits based on nominal
survey depth.
"""
filenames = config.getFilenames()
release = config['data']['release'].lower()
#band_1 = config['isochrone']['mag_1_field']
#band_2 = config['isochrone']['mag_2_field']
band_1 = config['catalog']['mag_1_field']
band_2 = config['catalog']['mag_2_field']
mask_1 = filenames['mask_1'].compressed()
mask_2 = filenames['mask_2'].compressed()
basedir,basename = os.path.split(config['mask']['dirname'])
if basename == dirname:
raise Exception("Input and output directory are the same.")
outdir = mkdir(os.path.join(basedir,dirname))
for band, infiles in [(band_1,mask_1),(band_2,mask_2)]:
maglim = MAGLIMS[release][band]
for infile in infiles:
basename = os.path.basename(infile)
outfile = join(outdir,basename)
logger.debug('Reading %s...'%infile)
f = pyfits.open(infile)
f[1].data['MAGLIM'][:] = maglim
logger.debug('Writing %s...'%outfile)
f.writeto(outfile,clobber=True)
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = np.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels,nside,nside_catalog)
catalog_pix = np.intersect1d(catalog_pix,catalog_pixels)
for fnames in filenames[catalog_pix]:
logger.debug("Loading %s"%filenames['mask_1'])
#subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_1'],'MAGLIM',construct_map=False)
_nside,subpix_1,val_1 = ugali.utils.healpix.read_partial_map(fnames['mask_1'],'MAGLIM',fullsky=False)
logger.debug("Loading %s"%fnames['mask_2'])
#subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_2'],'MAGLIM',construct_map=False)
_nside,subpix_2,val_2 = ugali.utils.healpix.read_partial_map(fnames['mask_2'],'MAGLIM',fullsky=False)
subpix = np.intersect1d(subpix_1,subpix_2)
superpix = np.unique(superpixel(subpix,nside_pixel,nside))
inside |= np.in1d(pixels, superpix)
return inside
def absolute_magnitude_martin(self, richness=1, steps=1e4, n_trials=1000, mag_bright=None, mag_faint=23., alpha=0.32, seed=None):
"""
Calculate the absolute magnitude (Mv) of the isochrone using
the prescription of Martin et al. 2008.
ADW: Seems like the faint and bright limits should depend on the survey maglim?
Parameters:
-----------
richness : Isochrone nomalization factor
steps : Number of steps for sampling the isochrone.
n_trials : Number of bootstrap samples
mag_bright : Bright magnitude limit [SDSS g-band] for luminosity calculation
mag_faint : Faint magnitude limit [SDSS g-band] for luminosity calculation
alpha : Output confidence interval (1-alpha)
seed : Random seed
Returns:
--------
med,lo,hi : Total absolute magnitude interval
"""
# ADW: This function is not quite right. It should restrict
# the catalog to the obsevable space using the mask in each
# pixel. This becomes even more complicated when we transform
# the isochrone into SDSS g,r...
if seed is not None: np.random.seed(seed)
# Create a copy of the isochrone in the SDSS system
params = {k:v.value for k,v in self._params.items()}
params.update(band_1='g',band_2='r',survey='sdss')
iso = self.__class__(**params)
# Analytic part (below detection threshold)
# g, r are absolute magnitudes
mass_init, mass_pdf, mass_act, sdss_g, sdss_r = iso.sample(mass_steps = steps)
V = jester_mag_v(sdss_g, sdss_r)
cut = ( (sdss_g + iso.distance_modulus) > mag_faint)
mag_unobs = sum_mags(V[cut], weights = richness * mass_pdf[cut])
# Stochastic part (above detection threshold)
abs_mag_v = np.zeros(n_trials)
for i in range(n_trials):
if i%100==0: logger.debug('%i absolute magnitude trials'%i)
# g,r are apparent magnitudes
sdss_g, sdss_r = iso.simulate(richness * iso.stellar_mass())
cut = (sdss_g < mag_faint)
# V is absolute magnitude
V = jester_mag_v(sdss_g[cut]-iso.distance_modulus,
sdss_r[cut]-iso.distance_modulus)
mag_obs = sum_mags(V)
abs_mag_v[i] = sum_mags([mag_obs,mag_unobs])
# ADW: Careful, fainter abs mag is larger (less negative) number
q = [100*alpha/2., 50, 100*(1-alpha/2.)]
hi,med,lo = np.percentile(abs_mag_v,q)
return ugali.utils.stats.interval(med,lo,hi)
def load_file(kwargs):
""" Load a FITS file with kwargs.
Parameters:
kwargs : keyword arguments passed to fitsio.read
Returns:
ndarray : fits catalog data
"""
logger.debug("Loading %s..."%kwargs['filename'])
return fitsio.read(**kwargs)
def labelHealpix(pixels, values, nside, threshold=0, xsize=1000):
"""
Label contiguous regions of a (sparse) HEALPix map. Works by mapping
HEALPix array to a Mollweide projection and applying scipy.ndimage.label
Assumes non-nested HEALPix map.
Parameters:
pixels : Pixel values associated to (sparse) HEALPix array
values : (Sparse) HEALPix array of data values
nside : HEALPix dimensionality
threshold : Threshold value for object detection
xsize : Size of Mollweide projection
Returns:
labels, nlabels
"""
proj = healpy.projector.MollweideProj(xsize=xsize)
vec = healpy.pix2vec(nside,pixels)
xy = proj.vec2xy(vec)
ij = proj.xy2ij(xy)
xx,yy = proj.ij2xy()
# Convert to Mollweide
searchims = []
if values.ndim < 2: iterate = [values]
else: iterate = values.T
for i,value in enumerate(iterate):
logger.debug("Labeling slice %i...")
searchim = numpy.zeros(xx.shape,dtype=bool)
select = (value > threshold)
yidx = ij[0][select]; xidx = ij[1][select]
searchim[yidx,xidx] |= True
searchims.append( searchim )
searchims = numpy.array(searchims)
# Full binary structure
s = ndimage.generate_binary_structure(searchims.ndim,searchims.ndim)
### # Dilate in the z-direction
logger.info(" Dilating image...")
searchims = ndimage.binary_dilation(searchims,s,1)
# Do the labeling
logger.info(" Labeling image...")
labels,nlabels = ndimage.label(searchims,structure=s)
# Convert back to healpix
pix_labels = labels[:,ij[0],ij[1]].T
pix_labels = pix_labels.reshape(values.shape)
pix_labels *= (values > threshold) # re-trim
return pix_labels, nlabels
def createLabels2D(self):
""" 2D labeling at zmax """
logger.debug(" Creating 2D labels...")
self.zmax = np.argmax(self.values,axis=1)
self.vmax = self.values[np.arange(len(self.pixels),dtype=int),self.zmax]
kwargs=dict(pixels=self.pixels,values=self.vmax,nside=self.nside,
threshold=self.threshold,xsize=self.xsize)
labels,nlabels = CandidateSearch.labelHealpix(**kwargs)
self.nlabels = nlabels
self.labels = np.repeat(labels,len(self.distances)).reshape(len(labels),len(self.distances))
return self.labels, self.nlabels
def stellarDensity(infile, nside=256, lon_field='RA', lat_field='DEC'):
area = hp.nside2pixarea(nside,degrees=True)
logger.debug("Reading %s"%infile)
data = fitsio.read(infile,columns=[lon_field,lat_field])
lon,lat = data[lon_field],data[lat_field]
pix = ang2pix(nside,lon,lat)
counts = collections.Counter(pix)
pixels, number = np.array(sorted(counts.items())).T
density = number/area
return pixels, density
def stellarDensity(infile, nside=2**8):
area = healpy.nside2pixarea(nside,degrees=True)
f = pyfits.open(infile)
data = f[1].data
logger.debug("Reading %s"%infile)
glon,glat = data['GLON'],data['GLAT']
pix = ang2pix(nside,glon,glat,coord='GAL')
counts = collections.Counter(pix)
pixels, number = numpy.array(sorted(counts.items())).T
density = number/area
f.close()
return pixels, density
def __init__(self, infiles, roi):
"""
Parameters:
-----------
infiles : list of sparse healpix mask files
roi : roi object
Returns:
--------
mask : MaskBand object
"""
self.roi = roi
self.config = self.roi.config
# ADW: It's overkill to make the full map just to slim it
# down, but we don't have a great way to go from map pixels to
# roi pixels.
nside,pixel,maglim = healpix.read_partial_map(infiles,column='MAGLIM')
self.nside = nside
# Sparse maps of pixels in various ROI regions
self.mask_roi_sparse = maglim[self.roi.pixels]
# Try to get the detection fraction
self.frac_roi_sparse = (self.mask_roi_sparse > 0)
try:
logger.debug("Reading FRACDET...")
nside,pixel,frac=healpix.read_partial_map(infiles,column='FRACDET')
# This clipping might gloss over bugs...
fractype = self.config['mask'].get('fractype','binary')
fracmin = self.config['mask'].get('fracmin',0.5)
if fractype == 'binary':
frac = np.where(frac < fracmin, 0.0, 1.0)
elif fractype == 'full':
frac = np.where(frac < fracmin, 0.0, frac)
elif not fractype:
pass
else:
msg = "Unrecognized fractype: %s"%fractype
logger.warn(msg)
self.frac_roi_sparse = np.clip(frac[self.roi.pixels],0.0,1.0)
except ValueError as e:
# No detection fraction present
msg = "No 'FRACDET' column found in masks; assuming FRACDET = 1.0"
logger.info(msg)
# Explicitly zero the maglim of pixels with fracdet < fracmin
self.mask_roi_sparse[self.frac_roi_sparse == 0] = 0.0
def get_ugali_dir():
"""Get the path to the ugali data directory from the environment"""
ugalidir = os.getenv('UGALIDIR')
# Get the HOME directory
if not ugalidir:
ugalidir=os.path.join(os.getenv('HOME'),'.ugali')
if not os.path.exists(ugalidir):
from ugali.utils.logger import logger
msg = "Creating UGALIDIR:\n%s"%ugalidir
logger.debug(msg)
return mkdir(ugalidir)
def createObjects(self):
logger.debug(" Creating objects...")
hist,edges,rev = reverseHistogram(self.labels,bins=numpy.arange(self.nlabels+2))
self.rev = rev
# Make some cut on the minimum size of a labelled object
good, = numpy.where( (hist >= self.minpix) )
# Get rid of zero label (below threshold)
self.good, = numpy.nonzero(good)
kwargs=dict(pixels=self.pixels,values=self.values,nside=self.nside,
zvalues=self.distances, rev=self.rev, good=self.good)
objects = self.findObjects(**kwargs)
self.objects = self.finalizeObjects(objects)
return self.objects
def add_column(filename,column,formula,force=False):
""" Add a column to a FITS file.
ADW: Could this be replaced by a ftool?
"""
columns = parse_formula(formula)
logger.info("Running file: %s"%filename)
logger.debug(" Reading columns: %s"%columns)
data = fitsio.read(filename,columns=columns)
logger.debug(' Evaluating formula: %s'%formula)
col = eval(formula)
col = np.asarray(col,dtype=[(column,col.dtype)])
insert_columns(filename,col,force=force)
return True
def getFilenames(self,pixels=None):
"""
Return the requested filenames.
Parameters:
-----------
pixels : requeseted pixels
Returns:
--------
filenames : recarray
"""
logger.debug("Getting filenames...")
if pixels is None:
return self.filenames
else:
return self.filenames[np.in1d(self.filenames['pix'],pixels)]
def mergeLikelihoodFiles(infiles, lkhdfile, roifile):
mergeSparseHealpixMaps(infiles,lkhdfile)
ext='PIX_DATA'
keys=['STELLAR','NINSIDE','NANNULUS']
nside = pyfits.open(infiles[0])[ext].header['LKDNSIDE']
pix_array = []
data_dict = dict([(k,[]) for k in keys])
for ii in range(0, len(infiles)):
logger.debug('(%i/%i) %s'%(ii+1, len(infiles), infiles[ii]))
reader = pyfits.open(infiles[ii])
pix_array.append(reader[ext].header['LKDPIX'])
for key in data_dict.keys():
data_dict[key].append(reader[ext].header[key])
pix_array = numpy.array(pix_array)
for key in data_dict.keys():
data_dict[key] = numpy.array(data_dict[key])
writeSparseHealpixMap(pix_array, data_dict, nside, roifile)
def download(self,age,metallicity,outdir=None,force=False):
"""
Check valid parameter range and download isochrones from:
http://stev.oapd.inaf.it/cgi-bin/cmd
"""
if outdir is None: outdir = './'
basename = self.isochrone.params2filename(age,metallicity)
outfile = os.path.join(outdir,basename)
if os.path.exists(outfile) and not force:
try:
self.verify(outfile,self.survey,age,metallicity)
logger.info("Found %s; skipping..."%(outfile))
return
except Exception as e:
msg = "Overwriting corrupted %s..."%(outfile)
logger.warn(msg)
#os.remove(outfile)
mkdir(outdir)
self.print_info(age,metallicity)
try:
self.query_server(outfile,age,metallicity)
except Exception as e:
logger.debug(str(e))
raise RuntimeError('Bad server response')
if not os.path.exists(outfile):
raise RuntimeError('Download failed')
try:
self.verify(outfile,self.survey,age,metallicity)
except Exception as e:
msg = "Output file is corrupted."
logger.error(msg)
#os.remove(outfile)
raise(e)
return outfile
def lnprob(self,theta):
""" Logarithm of the probability """
global niter
params,priors,loglike = self.params,self.priors,self.loglike
# Avoid extra likelihood calls with bad priors
_lnprior = self.lnprior(theta)
if np.isfinite(_lnprior):
_lnlike = self.lnlike(theta)
else:
_lnprior = -np.inf
_lnlike = -np.inf
_lnprob = _lnprior + _lnlike
if (niter%100==0):
msg = "%i function calls ...\n"%niter
msg+= ', '.join('%s: %.3f'%(k,v) for k,v in zip(params,theta))
msg+= '\nlog(like): %.3f, log(prior): %.3f'%(_lnprior,_lnlike)
logger.debug(msg)
niter+=1
return _lnprob
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog up into a set of healpix files.
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
for ii,infile in enumerate(infiles):
logger.info('(%i/%i) %s'%(ii+1, len(infiles), infile))
f = pyfits.open(infile)
data = f[1].data
header = f[1].header
logger.info("%i objects found"%len(data))
if not len(data): continue
glon,glat = cel2gal(data['RA'],data['DEC'])
catalog_pix = ang2pix(nside_catalog,glon,glat,coord='GAL')
pixel_pix = ang2pix(nside_pixel,glon,glat,coord='GAL')
names = [n.upper() for n in data.columns.names]
ra_idx = names.index('RA'); dec_idx = names.index('DEC')
idx = ra_idx if ra_idx > dec_idx else dec_idx
catalog_pix_name = 'PIX%i'%nside_catalog
pixel_pix_name = 'PIX%i'%nside_pixel
coldefs = pyfits.ColDefs(
[pyfits.Column(name='GLON',format='1D',array=glon),
pyfits.Column(name='GLAT',format='1D',array=glat),
pyfits.Column(name=catalog_pix_name,format='1J',array=catalog_pix),
pyfits.Column(name=pixel_pix_name ,format='1J',array=pixel_pix)]
)
hdu = pyfits.new_table(data.columns[:idx+1]+coldefs+data.columns[idx+1:])
table = hdu.data
for pix in numpy.unique(catalog_pix):
logger.debug("Processing pixel %s"%pix)
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s"%outfile)
names = [n.upper() for n in table.columns.names]
formats = table.columns.formats
columns = [pyfits.Column(n,f) for n,f in zip(names,formats)]
out = pyfits.HDUList([pyfits.PrimaryHDU(),pyfits.new_table(columns)])
out[1].header['NSIDE'] = nside_catalog
out[1].header['PIX'] = pix
out.writeto(outfile)
hdulist = pyfits.open(outfile,mode='update')
t1 = hdulist[1].data
# Could we speed up with sorting and indexing?
t2 = table[ table[catalog_pix_name] == pix ]
nrows1 = t1.shape[0]
nrows2 = t2.shape[0]
nrows = nrows1 + nrows2
out = pyfits.new_table(t1.columns, nrows=nrows)
for name in t1.columns.names:
out.data.field(name)[nrows1:]=t2.field(name)
hdulist[1] = out
logger.debug("Writing %s"%outfile)
hdulist.flush()
hdulist.close()
def query_server(self, outfile, age, metallicity):
z = metallicity
feh = self.z2feh(z)
params = copy.deepcopy(self.download_defaults)
params['age']=age
params['feh']='%.6f'%feh
params['clr']=dict_clr[self.survey]
server = self.download_url
url = server + '/models/isolf_new.php'
# First check that the server is alive
logger.debug("Accessing %s..."%url)
urlopen(url,timeout=2)
query = url + '?' + urlencode(params)
logger.debug(query)
response = urlopen(query)
page_source = str(response.read())
try:
file_id = int(page_source.split('tmp/tmp')[-1].split('.iso')[0])
except Exception as e:
logger.debug(str(e))
msg = 'Output filename not found'
raise RuntimeError(msg)
infile = 'http://stellar.dartmouth.edu/models/tmp/tmp%s.iso'%(file_id)
command = 'wget -q %s -O %s'%(infile, outfile)
subprocess.call(command,shell=True)
def readSparseHealpixMaps(infiles, field, extension='PIX_DATA', default_value=healpy.UNSEEN, construct_map=True):
"""
Read multiple sparse healpix maps and output the results
identically to a single file read.
"""
if isinstance(infiles,basestring): infiles = [infiles]
pix_array = []
value_array = []
# Create a map based on the first file in the list
map = readSparseHealpixMap(infiles[0], field, extension=extension, default_value=healpy.UNSEEN, construct_map=True)
for ii in range(0, len(infiles)):
logger.debug('(%i/%i) %s'%(ii+1, len(infiles), infiles[ii]))
pix_array_current, value_array_current = readSparseHealpixMap(infiles[ii], field,
extension=extension,
construct_map=False)
pix_array.append(pix_array_current)
value_array.append(value_array_current)
map[pix_array[ii]] = value_array[ii]
# Check to see whether there are any conflicts
pix_master = numpy.concatenate(pix_array)
value_master = numpy.concatenate(value_array)
n_conflicting_pixels = len(pix_master) - len(numpy.unique(pix_master))
if n_conflicting_pixels != 0:
logger.warning('%i conflicting pixels during merge.'%(n_conflicting_pixels))
if construct_map:
return map
else:
if n_conflicting_pixels == 0:
pix_master = numpy.sort(pix_master)
return pix_master, map[pix_master]
else:
pix_valid = numpy.nonzero(map != default_value)[0]
return pix_valid, map[pix_valid]
def load_files(filenames,multiproc=False,**kwargs):
""" Load a set of FITS files with kwargs. """
filenames = np.atleast_1d(filenames)
logger.debug("Loading %s files..."%len(filenames))
kwargs = [dict(filename=f,**kwargs) for f in filenames]
if multiproc:
from multiprocessing import Pool
processes = multiproc if multiproc > 0 else None
p = Pool(processes,maxtasksperchild=1)
out = p.map(load_file,kwargs)
else:
out = [load_file(kw) for kw in kwargs]
dtype = out[0].dtype
for i,d in enumerate(out):
if d.dtype != dtype:
# ADW: Not really safe...
logger.warn("Casting input data to same type.")
out[i] = d.astype(dtype,copy=False)
logger.debug('Concatenating arrays...')
return np.concatenate(out)
def inMangle(polyfile,ra,dec):
coords = tempfile.NamedTemporaryFile(suffix='.txt',delete=False)
logger.debug("Writing coordinates to %s"%coords.name)
np.savetxt(coords, np.array( [ra,dec] ).T, fmt='%.6g' )
coords.close()
weights = tempfile.NamedTemporaryFile(suffix='.txt',delete=False)
cmd = "polyid -W %s %s %s"%(polyfile,coords.name,weights.name)
logger.debug(cmd)
subprocess.call(cmd,shell=True)
tmp = tempfile.NamedTemporaryFile(suffix='.txt',delete=False)
cmd = """awk '{if($3==""){$3=0} print $1, $2, $3}' %s > %s"""%(weights.name,tmp.name)
logger.debug(cmd)
subprocess.call(cmd,shell=True)
data = np.loadtxt(tmp.name,unpack=True,skiprows=1)[-1]
for f in [coords,weights,tmp]:
logger.debug("Removing %s"%f.name)
os.remove(f.name)
return data > 0
def query_server(self,outfile,age,metallicity):
epsilon = 1e-4
lage = np.log10(age*1e9)
lage_min,lage_max = self.defaults['isoc_lage0'],self.defaults['isoc_lage1']
if not (lage_min-epsilon < lage <lage_max+epsilon):
msg = 'Age outside of valid range: %g [%g < log(age) < %g]'%(lage,lage_min,lage_max)
raise RuntimeError(msg)
z_min,z_max = self.defaults['isoc_z0'],self.defaults['isoc_z1']
if not (z_min <= metallicity <= z_max):
msg = 'Metallicity outside of valid range: %g [%g < z < %g]'%(metallicity,z_min,z_max)
raise RuntimeError(msg)
d = dict(self.defaults)
d['photsys_file'] = photsys_dict[self.survey]
d['isoc_age'] = age * 1e9
d['isoc_zeta'] = metallicity
server = 'http://stev.oapd.inaf.it'
url = server + '/cgi-bin/cmd_%s'%d['cmd_version']
logger.debug("Accessing %s..."%url)
q = urlencode(d)
logger.debug(url+'?'+q)
c = urlopen(url, q).read()
aa = re.compile('output\d+')
fname = aa.findall(c)
if len(fname) == 0:
msg = "Output filename not found"
raise RuntimeError(msg)
out = '{0}/~lgirardi/tmp/{1}.dat'.format(server, fname[0])
cmd = 'wget %s -O %s'%(out,outfile)
logger.debug(cmd)
stdout = subprocess.check_output(cmd,shell=True,stderr=subprocess.STDOUT)
logger.debug(stdout)
return outfile
def clip_catalog(self):
# ROI-specific catalog
logger.debug("Clipping full catalog...")
cut_observable = self.mask.restrictCatalogToObservableSpace(self.catalog_full)
# All objects within disk ROI
logger.debug("Creating roi catalog...")
self.catalog_roi = self.catalog_full.applyCut(cut_observable)
self.catalog_roi.project(self.roi.projector)
self.catalog_roi.spatialBin(self.roi)
# All objects interior to the background annulus
logger.debug("Creating interior catalog...")
cut_interior = numpy.in1d(ang2pix(self.config['coords']['nside_pixel'], self.catalog_roi.lon, self.catalog_roi.lat),
self.roi.pixels_interior)
#cut_interior = self.roi.inInterior(self.catalog_roi.lon,self.catalog_roi.lat)
self.catalog_interior = self.catalog_roi.applyCut(cut_interior)
self.catalog_interior.project(self.roi.projector)
self.catalog_interior.spatialBin(self.roi)
# Set the default catalog
#logger.info("Using interior ROI for likelihood calculation")
self.catalog = self.catalog_interior
def run(self):
outdir = self.config['output']['simdir']
logdir = join(outdir, 'log')
if 'simulate' in self.opts.run:
logger.info("Running 'simulate'...")
mkdir(outdir)
mkdir(logdir)
if self.opts.num is None:
self.opts.num = self.config['simulate']['njobs']
for i in range(self.opts.num):
outfile = join(outdir, self.config['output']['simfile'] % i)
base = splitext(os.path.basename(outfile))[0]
logfile = join(logdir, base + '.log')
jobname = base
script = self.config['simulate']['script']
cmd = '%s %s %s --seed %i' % (script, self.opts.config, outfile, i)
#cmd='%s %s %s'%(script,self.opts.config,outfile)
self.batch.submit(cmd, jobname, logfile)
time.sleep(0.1)
if 'sensitivity' in self.opts.run:
logger.info("Running 'sensitivity'...")
if 'merge' in self.opts.run:
logger.info("Running 'merge'...")
filenames = join(
outdir, self.config['output']['simfile']).split('_%')[0] + '_*'
infiles = sorted(glob.glob(filenames))
f = pyfits.open(infiles[0])
table = np.empty(0, dtype=data.dtype)
for filename in infiles:
logger.debug("Reading %s..." % filename)
f = pyfits.open(filename)
t = f[1].data[~np.isnan(f[1].data['ts'])]
table = recfuncs.stack_arrays([table, t],
usemask=False,
asrecarray=True)
logger.info("Found %i simulations." % len(table))
outfile = join(outdir, "merged_sims.fits")
hdu = pyfits.new_table(table)
logger.info("Writing %s..." % outfile)
hdu.writeto(outfile, clobber=True)
if 'plot' in self.opts.run:
logger.info("Running 'plot'...")
import ugali.utils.plotting
import pylab as plt
plotdir = mkdir(self.config['output']['plotdir'])
data = pyfits.open(join(outdir, "merged_sims.fits"))[1].data
data = data[~np.isnan(data['ts'])]
bigfig, bigax = plt.subplots()
for dist in np.unique(data['fit_distance']):
logger.info(' Plotting distance: %s' % dist)
ts = data['ts'][data['fit_distance'] == dist]
ugali.utils.plotting.drawChernoff(bigax,
ts,
bands='none',
color='gray')
fig, ax = plt.subplots(1, 2, figsize=(10, 5))
ugali.utils.plotting.drawChernoff(ax[0],
ts,
bands='none',
pdf=True)
ugali.utils.plotting.drawChernoff(ax[1], ts)
fig.suptitle(r'Chernoff ($\mu = %g$)' % dist)
ax[0].annotate(r"$N=%i$" % len(ts),
xy=(0.15, 0.85),
xycoords='axes fraction',
bbox={
'boxstyle': "round",
'fc': '1'
})
basename = 'chernoff_u%g.png' % dist
outfile = os.path.join(plotdir, basename)
plt.savefig(outfile)
bigfig.suptitle('Chernoff!')
basename = 'chernoff_all.png'
outfile = os.path.join(plotdir, basename)
plt.savefig(outfile)
#idx=np.random.randint(len(data['ts'])-1,size=400)
#idx=slice(400)
#ugali.utils.plotting.plotChernoff(data['ts'][idx])
#ugali.utils.plotting.plotChernoff(data['fit_ts'])
plt.ion()
"""
def absolute_magnitude_martin(self, richness=1, steps=1e4, n_trials=1000, mag_bright=16., mag_faint=23., alpha=0.32, seed=None):
"""
Calculate the absolute magnitude (Mv) of the isochrone using
the prescription of Martin et al. 2008.
Parameters:
-----------
richness : Isochrone nomalization factor
steps : Number of steps for sampling the isochrone.
n_trials : Number of bootstrap samples
mag_bright : Bright magnitude limit for calculating luminosity.
mag_faint : Faint magnitude limit for calculating luminosity.
alpha : Output confidence interval (1-alpha)
seed : Random seed
Returns:
--------
med,lo,hi : Absolute magnitude interval
"""
# ADW: This function is not quite right. You should be restricting
# the catalog to the obsevable space (using the function named as such)
# Also, this needs to be applied in each pixel individually
# Using the SDSS g,r -> V from Jester 2005 [arXiv:0506022]
# for stars with R-I < 1.15
# V = g_sdss - 0.59(g_sdss-r_sdss) - 0.01
# g_des = g_sdss - 0.104(g_sdss - r_sdss) + 0.01
# r_des = r_sdss - 0.102(g_sdss - r_sdss) + 0.02
np.random.seed(seed)
if self.survey.lower() != 'des':
raise Exception('Only valid for DES')
if 'g' not in [self.band_1,self.band_2]:
msg = "Need g-band for absolute magnitude"
raise Exception(msg)
if 'r' not in [self.band_1,self.band_2]:
msg = "Need r-band for absolute magnitude"
raise Exception(msg)
def visual(g, r, pdf=None):
v = g - 0.487 * (g - r) - 0.0249
if pdf is None:
flux = np.sum(10**(-v / 2.5))
else:
flux = np.sum(pdf * 10**(-v / 2.5))
abs_mag_v = -2.5 * np.log10(flux)
return abs_mag_v
def sumMag(mag_1, mag_2):
flux_1 = 10**(-mag_1 / 2.5)
flux_2 = 10**(-mag_2 / 2.5)
return -2.5 * np.log10(flux_1 + flux_2)
# Analytic part
mass_init, mass_pdf, mass_act, mag_1, mag_2 = self.sample(mass_steps = steps)
g,r = (mag_1,mag_2) if self.band_1 == 'g' else (mag_2,mag_1)
#cut = np.logical_not((g > mag_bright) & (g < mag_faint) & (r > mag_bright) & (r < mag_faint))
cut = ((g + self.distance_modulus) > mag_faint) if self.band_1 == 'g' else ((r + self.distance_modulus) > mag_faint)
mag_unobs = visual(g[cut], r[cut], richness * mass_pdf[cut])
# Stochastic part
abs_mag_obs_array = np.zeros(n_trials)
for ii in range(0, n_trials):
if ii%100==0: logger.debug('%i absolute magnitude trials'%ii)
g, r = self.simulate(richness * self.stellar_mass())
#cut = (g > 16.) & (g < 23.) & (r > 16.) & (r < 23.)
cut = (g < mag_faint) if self.band_1 == 'g' else (r < mag_faint)
mag_obs = visual(g[cut] - self.distance_modulus, r[cut] - self.distance_modulus)
abs_mag_obs_array[ii] = sumMag(mag_obs, mag_unobs)
# ADW: This shouldn't be necessary
#abs_mag_obs_array = np.sort(abs_mag_obs_array)[::-1]
# ADW: Careful, fainter abs mag is larger (less negative) number
q = [100*alpha/2., 50, 100*(1-alpha/2.)]
hi,med,lo = np.percentile(abs_mag_obs_array,q)
return ugali.utils.stats.interval(med,lo,hi)
def writeColorLUT2(config,
outfile=None,
isochrone=None,
distance_modulus_array=None,
delta_mag=None,
mag_err_array=None,
mass_steps=10000,
plot=False):
"""
Precompute a 4-dimensional signal color probability look-up table to speed up the likelihood evaluation.
Inputs are a Config object (or file name), an Isochrone object,
an array of distance moduli at which to evaluate the signal color probability,
and an array of magnitude uncertainties which set the bin edges of those dimensions (zero implicity included).
Finally there is an outfile name.
"""
if plot: import ugali.utils.plotting
if type(config) == str:
config = ugali.utils.config.Config(config)
if outfile is None:
outfile = config.params['color_lut']['filename']
if isochrone is None:
isochrones = []
for ii, name in enumerate(config.params['isochrone']['infiles']):
isochrones.append(ugali.isochrone.Isochrone(config, name))
isochrone = ugali.isochrone.CompositeIsochrone(
isochrones, config.params['isochrone']['weights'])
if distance_modulus_array is None:
distance_modulus_array = config.params['color_lut'][
'distance_modulus_array']
if delta_mag is None:
delta_mag = config.params['color_lut']['delta_mag']
if mag_err_array is None:
mag_err_array = config.params['color_lut']['mag_err_array']
mag_buffer = 0.5 # Safety buffer in magnitudes around the color-magnitude space defined by the ROI
epsilon = 1.e-10
if config.params['catalog']['band_1_detection']:
bins_mag_1 = numpy.arange(
config.params['mag']['min'] - mag_buffer,
config.params['mag']['max'] + mag_buffer + epsilon, delta_mag)
bins_mag_2 = numpy.arange(
config.params['mag']['min'] - config.params['color']['max'] -
mag_buffer, config.params['mag']['max'] -
config.params['color']['min'] + mag_buffer + epsilon, delta_mag)
else:
bins_mag_1 = numpy.arange(
config.params['mag']['min'] + config.params['color']['min'] -
mag_buffer, config.params['mag']['max'] +
config.params['color']['max'] + mag_buffer + epsilon, delta_mag)
bins_mag_2 = numpy.arange(
config.params['mag']['min'] - mag_buffer,
config.params['mag']['max'] + mag_buffer + epsilon, delta_mag)
# Output binning configuration
#print config.params['catalog']['band_1_detection']
#print config.params['mag']['min'], config.params['mag']['max']
#print config.params['color']['min'], config.params['color']['max']
#print bins_mag_1[0], bins_mag_1[-1], len(bins_mag_1)
#print bins_mag_2[0], bins_mag_2[-1], len(bins_mag_2)
isochrone_mass_init, isochrone_mass_pdf, isochrone_mass_act, isochrone_mag_1, isochrone_mag_2 = isochrone.sample(
mass_steps=mass_steps)
hdul = pyfits.HDUList()
for index_distance_modulus, distance_modulus in enumerate(
distance_modulus_array):
logger.debug('(%i/%i)' %
(index_distance_modulus, len(distance_modulus_array)))
columns_array = []
time_start = time.time()
histo_isochrone_pdf = numpy.histogram2d(
distance_modulus + isochrone_mag_1,
distance_modulus + isochrone_mag_2,
bins=[bins_mag_1, bins_mag_2],
weights=isochrone_mass_pdf)[0]
if plot:
# Checked that axis are plotted correctly
ugali.utils.plotting.twoDimensionalHistogram(
'Isochrone',
'mag_1',
'mag_2',
numpy.log10(histo_isochrone_pdf + epsilon).transpose(),
bins_mag_1,
bins_mag_2,
lim_x=None,
lim_y=None,
vmin=None,
vmax=None)
for index_mag_err_1, mag_err_1 in enumerate(mag_err_array):
for index_mag_err_2, mag_err_2 in enumerate(mag_err_array):
logger.debug(
' Distance modulus = %.2f mag_err_1 = %.2f mag_err_2 = %.2f'
% (distance_modulus, mag_err_1, mag_err_2))
mag_1_sigma_step = delta_mag / mag_err_1
n = int(numpy.ceil(4. / mag_1_sigma_step))
mag_1_sigma = numpy.arange(-1. * (n + 0.5) * mag_1_sigma_step,
((n + 0.5) * mag_1_sigma_step) +
epsilon, mag_1_sigma_step)
mag_1_pdf_array = scipy.stats.norm.cdf(
mag_1_sigma[1:]) - scipy.stats.norm.cdf(mag_1_sigma[0:-1])
mag_2_sigma_step = delta_mag / mag_err_2
n = int(numpy.ceil(4. / mag_2_sigma_step))
mag_2_sigma = numpy.arange(-1. * (n + 0.5) * mag_2_sigma_step,
((n + 0.5) * mag_2_sigma_step) +
epsilon, mag_2_sigma_step)
mag_2_pdf_array = scipy.stats.norm.cdf(
mag_2_sigma[1:]) - scipy.stats.norm.cdf(mag_2_sigma[0:-1])
mag_1_pdf, mag_2_pdf = numpy.meshgrid(mag_2_pdf_array,
mag_1_pdf_array)
pdf = mag_1_pdf * mag_2_pdf
histo_isochrone_pdf_convolve = scipy.signal.convolve2d(
histo_isochrone_pdf, pdf, mode='same')
if plot:
# Checked that axis are plotted correctly
ugali.utils.plotting.twoDimensionalHistogram(
'Convolved Isochrone',
'mag_1',
'mag_2',
numpy.log10(histo_isochrone_pdf_convolve +
epsilon).transpose(),
bins_mag_1,
bins_mag_2,
lim_x=None,
lim_y=None,
vmin=None,
vmax=None)
columns_array.append(
pyfits.Column(
name='%i%i' % (index_mag_err_1, index_mag_err_2),
format='%iE' % (histo_isochrone_pdf_convolve.shape[1]),
array=histo_isochrone_pdf_convolve))
hdu = pyfits.new_table(columns_array)
hdu.header.update('DIST_MOD', distance_modulus)
hdu.name = '%.2f' % (distance_modulus)
hdul.append(hdu)
time_end = time.time()
logger.debug('%.2f s' % (time_end - time_start))
# Store distance modulus info
columns_array = [
pyfits.Column(name='DISTANCE_MODULUS',
format='E',
array=distance_modulus_array)
]
hdu = pyfits.new_table(columns_array)
hdu.name = 'DISTANCE_MODULUS'
hdul.append(hdu)
# Store magnitude error info
columns_array = [
pyfits.Column(name='BINS_MAG_ERR',
format='E',
array=numpy.insert(mag_err_array, 0, 0.))
]
hdu = pyfits.new_table(columns_array)
hdu.name = 'BINS_MAG_ERR'
hdul.append(hdu)
# Store magnitude 1 info
columns_array = [
pyfits.Column(name='BINS_MAG_1', format='E', array=bins_mag_1)
]
hdu = pyfits.new_table(columns_array)
hdu.name = 'BINS_MAG_1'
hdul.append(hdu)
# Store magnitude 2 info
columns_array = [
pyfits.Column(name='BINS_MAG_2', format='E', array=bins_mag_2)
]
hdu = pyfits.new_table(columns_array)
hdu.name = 'BINS_MAG_2'
hdul.append(hdu)
logger.info('Writing look-up table to %s' % (outfile))
hdul.writeto(outfile, clobber=True)
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i,filename in enumerate(infiles):
logger.info('(%i/%i) %s'%(i+1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found"%len(data))
if not len(data): continue
columns = map(str.upper,data.dtype.names)
names,arrs = [],[]
if (lon_field in columns) and (lat_field in columns):
lon,lat = data[lon_field],data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon,lat = cel2gal(data['RA'],data['DEC'])
names += [lon_field,lat_field]
arrs += [lon,lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from GLON,GLAT"
lon,lat = gal2cel(data['GLON'],data['GLAT'])
names += [lon_field,lat_field]
arrs += [lon,lat]
cat_pix = ang2pix(nside_catalog,lon,lat)
pix_pix = ang2pix(nside_pixel,lon,lat)
cat_pix_name = 'PIX%i'%nside_catalog
pix_pix_name = 'PIX%i'%nside_pixel
try:
names += [cat_pix_name,pix_pix_name]
arrs += [cat_pix,pix_pix]
data=mlab.rec_append_fields(data,names=names,arrs=arrs)
except ValueError as e:
logger.warn(str(e)+'; not adding column.')
#data[cat_pix_name] = cat_pix
#data[pix_pix_name] = pix_pix
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s"%pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s"%outfile)
out=fitsio.FITS(outfile,mode='rw')
out.write(arr)
hdr=healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE','ORDERING','NSIDE','COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',pix,comment='HEALPIX pixel for this file')
else:
out=fitsio.FITS(outfile,mode='rw')
out[1].append(arr)
logger.debug("Writing %s"%outfile)
out.close()
def query_server(self, outfile, age, metallicity):
""" Server query for the isochrone file.
Parameters:
-----------
outfile : name of output isochrone file
age : isochrone age
metallicity : isochrone metallicity
Returns:
--------
outfile : name of output isochrone file
"""
params = copy.deepcopy(self.download_defaults)
epsilon = 1e-4
lage = np.log10(age * 1e9)
lage_min = params.get('isoc_lage0', 6.602)
lage_max = params.get('isoc_lage1', 10.1303)
if not (lage_min - epsilon < lage < lage_max + epsilon):
msg = 'Age outside of valid range: %g [%g < log(age) < %g]' % (
lage, lage_min, lage_max)
raise RuntimeError(msg)
z_min = params.get('isoc_z0', 0.0001)
z_max = params.get('isoc_z1', 0.03)
if not (z_min <= metallicity <= z_max):
msg = 'Metallicity outside of valid range: %g [%g < z < %g]' % (
metallicity, z_min, z_max)
raise RuntimeError(msg)
params['photsys_file'] = photsys_dict[self.survey]
if params['cmd_version'] < 3.3:
params['isoc_age'] = age * 1e9
params['isoc_zeta'] = metallicity
else:
params['isoc_agelow'] = age * 1e9
params['isoc_zlow'] = metallicity
server = self.download_url
url = server + '/cgi-bin/cmd_%s' % params['cmd_version']
# First check that the server is alive
logger.debug("Accessing %s..." % url)
urlopen(url, timeout=2)
q = urlencode(params).encode('utf-8')
logger.debug(url + '?' + q)
c = str(urlopen(url, q).read())
aa = re.compile('output\d+')
fname = aa.findall(c)
if len(fname) == 0:
msg = "Output filename not found"
raise RuntimeError(msg)
out = '{0}/tmp/{1}.dat'.format(server, fname[0])
cmd = 'wget --progress dot:binary %s -O %s' % (out, outfile)
logger.debug(cmd)
stdout = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
logger.debug(stdout)
return outfile
def search(self, coords=None, distance_modulus=None, tolerance=1.e-2):
"""
Organize a grid search over ROI target pixels and distance moduli in distance_modulus_array
coords: (lon,lat)
distance_modulus: scalar
"""
nmoduli = len(self.distance_modulus_array)
npixels = len(self.roi.pixels_target)
self.log_likelihood_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_lower_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_upper_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_upper_limit_sparse_array = numpy.zeros([nmoduli, npixels])
self.stellar_mass_sparse_array = numpy.zeros([nmoduli, npixels])
self.fraction_observable_sparse_array = numpy.zeros([nmoduli, npixels])
# Specific pixel/distance_modulus
coord_idx, distance_modulus_idx = None, None
if coords is not None:
# Match to nearest grid coordinate index
coord_idx = self.roi.indexTarget(coords[0],coords[1])
if distance_modulus is not None:
# Match to nearest distance modulus index
distance_modulus_idx=np.fabs(self.distance_modulus_array-distance_modulus).argmin()
lon, lat = self.roi.pixels_target.lon, self.roi.pixels_target.lat
logger.info('Looping over distance moduli in grid search ...')
for ii, distance_modulus in enumerate(self.distance_modulus_array):
# Specific pixel
if distance_modulus_idx is not None:
if ii != distance_modulus_idx: continue
logger.info(' (%-2i/%i) Distance Modulus=%.1f ...'%(ii+1,nmoduli,distance_modulus))
# Set distance_modulus once to save time
self.loglike.set_params(distance_modulus=distance_modulus)
for jj in range(0, npixels):
# Specific pixel
if coord_idx is not None:
if jj != coord_idx: continue
# Set kernel location
self.loglike.set_params(lon=lon[jj],lat=lat[jj])
# Doesn't re-sync distance_modulus each time
self.loglike.sync_params()
args = (jj+1, npixels, self.loglike.source.lon, self.loglike.source.lat)
message = ' (%-3i/%i) Candidate at (%.2f, %.2f) ... '%(args)
self.log_likelihood_sparse_array[ii][jj], self.richness_sparse_array[ii][jj], parabola = self.loglike.fit_richness()
self.stellar_mass_sparse_array[ii][jj] = self.stellar_mass_conversion * self.richness_sparse_array[ii][jj]
self.fraction_observable_sparse_array[ii][jj] = self.loglike.f
if self.config['scan']['full_pdf']:
#n_pdf_points = 100
#richness_range = parabola.profileUpperLimit(delta=25.) - self.richness_sparse_array[ii][jj]
#richness = numpy.linspace(max(0., self.richness_sparse_array[ii][jj] - richness_range),
# self.richness_sparse_array[ii][jj] + richness_range,
# n_pdf_points)
#if richness[0] > 0.:
# richness = numpy.insert(richness, 0, 0.)
# n_pdf_points += 1
#
#log_likelihood = numpy.zeros(n_pdf_points)
#for kk in range(0, n_pdf_points):
# log_likelihood[kk] = self.loglike.value(richness=richness[kk])
#parabola = ugali.utils.parabola.Parabola(richness, 2.*log_likelihood)
#self.richness_lower_sparse_array[ii][jj], self.richness_upper_sparse_array[ii][jj] = parabola.confidenceInterval(0.6827)
self.richness_lower_sparse_array[ii][jj], self.richness_upper_sparse_array[ii][jj] = self.loglike.richness_interval(0.6827)
self.richness_upper_limit_sparse_array[ii][jj] = parabola.bayesianUpperLimit(0.95)
args = (
2. * self.log_likelihood_sparse_array[ii][jj],
self.stellar_mass_conversion*self.richness_sparse_array[ii][jj],
self.stellar_mass_conversion*self.richness_lower_sparse_array[ii][jj],
self.stellar_mass_conversion*self.richness_upper_sparse_array[ii][jj],
self.stellar_mass_conversion*self.richness_upper_limit_sparse_array[ii][jj]
)
message += 'TS=%.1f, Stellar Mass=%.1f (%.1f -- %.1f @ 0.68 CL, < %.1f @ 0.95 CL)'%(args)
else:
args = (
2. * self.log_likelihood_sparse_array[ii][jj],
self.stellar_mass_conversion * self.richness_sparse_array[ii][jj],
self.fraction_observable_sparse_array[ii][jj]
)
message += 'TS=%.1f, Stellar Mass=%.1f, Fraction=%.2g'%(args)
logger.debug( message )
#if coords is not None and distance_modulus is not None:
# results = [self.richness_sparse_array[ii][jj],
# self.log_likelihood_sparse_array[ii][jj],
# self.richness_lower_sparse_array[ii][jj],
# self.richness_upper_sparse_array[ii][jj],
# self.richness_upper_limit_sparse_array[ii][jj],
# richness, log_likelihood, self.loglike.p, self.loglike.f]
# return results
jj_max = self.log_likelihood_sparse_array[ii].argmax()
args = (
jj_max+1, npixels, lon[jj_max], lat[jj_max],
2. * self.log_likelihood_sparse_array[ii][jj_max],
self.stellar_mass_conversion * self.richness_sparse_array[ii][jj_max]
)
message = ' (%-3i/%i) Maximum at (%.2f, %.2f) ... TS=%.1f, Stellar Mass=%.1f'%(args)
logger.info( message )
def load(args):
#logger = logging.getLogger()
infile, columns = args
logger.debug("Loading %s..." % infile)
return fitsio.read(infile, columns=columns)
def writeColorLUT(config,
outfile=None, isochrone=None, distance_modulus_array=None,
delta_mag=None, mag_err_array=None,
mass_steps=1000000, plot=False):
"""
Precompute a 4-dimensional signal color probability look-up table to speed up the likelihood evaluation.
Inputs are a Config object (or file name), an Isochrone object,
an array of distance moduli at which to evaluate the signal color probability,
and an array of magnitude uncertainties which set the bin edges of those dimensions (zero implicity included).
Finally there is an outfile name.
"""
if plot: import ugali.utils.plotting
if type(config) == str:
config = ugali.utils.config.Config(config)
if outfile is None:
outfile = config.params['color_lut']['filename']
if isochrone is None:
isochrones = []
for ii, name in enumerate(config.params['isochrone']['infiles']):
isochrones.append(ugali.analysis.isochrone.Isochrone(config, name))
isochrone = ugali.analysis.isochrone.CompositeIsochrone(isochrones, config.params['isochrone']['weights'])
if distance_modulus_array is None:
distance_modulus_array = config.params['color_lut']['distance_modulus_array']
if delta_mag is None:
delta_mag = config.params['color_lut']['delta_mag']
if mag_err_array is None:
mag_err_array = config.params['color_lut']['mag_err_array']
mag_buffer = 0.5 # Safety buffer in magnitudes around the color-magnitude space defined by the ROI
epsilon = 1.e-10
if config.params['catalog']['band_1_detection']:
bins_mag_1 = numpy.arange(config.params['mag']['min'] - mag_buffer,
config.params['mag']['max'] + mag_buffer + epsilon,
delta_mag)
bins_mag_2 = numpy.arange(config.params['mag']['min'] - config.params['color']['max'] - mag_buffer,
config.params['mag']['max'] - config.params['color']['min'] + mag_buffer + epsilon,
delta_mag)
else:
bins_mag_1 = numpy.arange(config.params['mag']['min'] + config.params['color']['min'] - mag_buffer,
config.params['mag']['max'] + config.params['color']['max'] + mag_buffer + epsilon,
delta_mag)
bins_mag_2 = numpy.arange(config.params['mag']['min'] - mag_buffer,
config.params['mag']['max'] + mag_buffer + epsilon,
delta_mag)
# Output binning configuration
#print config.params['catalog']['band_1_detection']
#print config.params['mag']['min'], config.params['mag']['max']
#print config.params['color']['min'], config.params['color']['max']
#print bins_mag_1[0], bins_mag_1[-1], len(bins_mag_1)
#print bins_mag_2[0], bins_mag_2[-1], len(bins_mag_2)
isochrone_mass_init, isochrone_mass_pdf, isochrone_mass_act, isochrone_mag_1, isochrone_mag_2 = isochrone.sample(mass_steps=mass_steps)
# make randoms
randoms_1 = numpy.random.normal(0., 1., len(isochrone_mass_pdf))
randoms_2 = numpy.random.normal(0., 1., len(isochrone_mass_pdf))
hdul = pyfits.HDUList()
for index_distance_modulus, distance_modulus in enumerate(distance_modulus_array):
logger.debug('(%i/%i)'%(index_distance_modulus, len(distance_modulus_array)))
columns_array = []
time_start = time.time()
for index_mag_err_1, mag_err_1 in enumerate(mag_err_array):
for index_mag_err_2, mag_err_2 in enumerate(mag_err_array):
logger.debug(' (%i/%i) Distance modulus = %.2f mag_err_1 = %.3f mag_err_2 = %.3f'%(index_mag_err_1 * len(mag_err_array) + index_mag_err_2,
len(mag_err_array)**2,
distance_modulus,
mag_err_1,
mag_err_2))
# Add randoms
histo_isochrone_pdf = numpy.histogram2d(distance_modulus + isochrone_mag_1 + randoms_1 * mag_err_1,
distance_modulus + isochrone_mag_2 + randoms_2 * mag_err_2,
bins=[bins_mag_1, bins_mag_2],
weights=isochrone_mass_pdf)[0]
if plot:
# Checked that axis are plotted correctly
ugali.utils.plotting.twoDimensionalHistogram('Convolved Isochrone', 'mag_1', 'mag_2',
numpy.log10(histo_isochrone_pdf + epsilon).transpose(),
bins_mag_1, bins_mag_2,
lim_x=None, lim_y=None,
vmin=None, vmax=None)
raw_input('WAIT')
columns_array.append(pyfits.Column(name = '%i%i'%(index_mag_err_1, index_mag_err_2),
format = '%iE'%(histo_isochrone_pdf.shape[1]),
array = histo_isochrone_pdf))
hdu = pyfits.new_table(columns_array)
hdu.header.update('DIST_MOD', distance_modulus)
hdu.name = '%.2f'%(distance_modulus)
hdul.append(hdu)
time_end = time.time()
logger.debug('%.2f s'%(time_end - time_start))
# Store distance modulus info
columns_array = [pyfits.Column(name = 'DISTANCE_MODULUS',
format = 'E',
array = distance_modulus_array)]
hdu = pyfits.new_table(columns_array)
hdu.name = 'DISTANCE_MODULUS'
hdul.append(hdu)
# Store magnitude error info
columns_array = [pyfits.Column(name = 'BINS_MAG_ERR',
format = 'E',
array = numpy.insert(mag_err_array, 0, 0.))]
hdu = pyfits.new_table(columns_array)
hdu.name = 'BINS_MAG_ERR'
hdul.append(hdu)
# Store magnitude 1 info
columns_array = [pyfits.Column(name = 'BINS_MAG_1',
format = 'E',
array = bins_mag_1)]
hdu = pyfits.new_table(columns_array)
hdu.name = 'BINS_MAG_1'
hdul.append(hdu)
# Store magnitude 2 info
columns_array = [pyfits.Column(name = 'BINS_MAG_2',
format = 'E',
array = bins_mag_2)]
hdu = pyfits.new_table(columns_array)
hdu.name = 'BINS_MAG_2'
hdul.append(hdu)
logger.info('Writing look-up table to %s'%(outfile))
hdul.writeto(outfile, clobber = True)
def download(self,age,metallicity,outdir=None,force=False):
"""
Check valid parameter range and download isochrones from:
http://stev.oapd.inaf.it/cgi-bin/cmd
"""
epsilon = 1e-4
lage = np.log10(age*1e9)
lage_min,lage_max = self.defaults['isoc_lage0'],self.defaults['isoc_lage1']
if not (lage_min-epsilon < lage <lage_max+epsilon):
msg = 'Age outside of valid range: %g [%g < log(age) < %g]'%(lage,lage_min,lage_max)
raise RuntimeError(msg)
z_min,z_max = self.defaults['isoc_z0'],self.defaults['isoc_z1']
if not (z_min <= metallicity <= z_max):
msg = 'Metallicity outside of valid range: %g [%g < z < %g]'%(metallicity,z_min,z_max)
raise RuntimeError(msg)
survey=self.survey.lower()
if survey=='des':
photsys_file='tab_mag_odfnew/tab_mag_decam.dat'
elif survey=='sdss':
photsys_file='tab_mag_odfnew/tab_mag_sloan.dat'
else:
msg = 'Unrecognized survey: %s'%survey
raise RuntimeError(msg)
if outdir is None: outdir = './'
mkdir(outdir)
basename = self.params2filename(age,metallicity)
outfile = os.path.join(outdir,basename)
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%(outfile))
return
logger.info("Downloading isochrone: %s (age=%.2fGyr, metallicity=%g)"%(basename,age,metallicity))
d = dict(self.defaults)
d['photsys_file'] = photsys_file
d['isoc_age'] = age * 1e9
d['isoc_zeta'] = metallicity
server = 'http://stev.oapd.inaf.it'
url = server + '/cgi-bin/cmd_%s'%d['cmd_version']
logger.info("Accessing %s..."%url)
q = urlencode(d)
logger.debug(url+'?'+q)
c = urlopen(url, q).read()
aa = re.compile('output\d+')
fname = aa.findall(c)
if len(fname) > 0:
out = '{0}/~lgirardi/tmp/{1}.dat'.format(server, fname[0])
cmd = 'wget %s -O %s'%(out,outfile)
logger.debug(cmd)
stdout = subprocess.check_output(cmd,shell=True,stderr=subprocess.STDOUT)
logger.debug(stdout)
else:
#print(c)
raise RuntimeError('Server Response is incorrect')
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i, filename in enumerate(infiles):
logger.info('(%i/%i) %s' % (i + 1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found" % len(data))
if not len(data): continue
columns = map(str.upper, data.dtype.names)
names, arrs = [], []
if (lon_field in columns) and (lat_field in columns):
lon, lat = data[lon_field], data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon, lat = cel2gal(data['RA'], data['DEC'])
names += [lon_field, lat_field]
arrs += [lon, lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field)
msg += "\nConverting from GLON,GLAT"
lon, lat = gal2cel(data['GLON'], data['GLAT'])
names += [lon_field, lat_field]
arrs += [lon, lat]
cat_pix = ang2pix(nside_catalog, lon, lat)
pix_pix = ang2pix(nside_pixel, lon, lat)
cat_pix_name = 'PIX%i' % nside_catalog
pix_pix_name = 'PIX%i' % nside_pixel
names += [cat_pix_name, pix_pix_name]
arrs += [cat_pix, pix_pix]
data = mlab.rec_append_fields(data, names=names, arrs=arrs)
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s" % pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s" % outfile)
out = fitsio.FITS(outfile, mode='rw')
out.write(arr)
hdr = healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE', 'ORDERING', 'NSIDE', 'COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',
pix,
comment='HEALPIX pixel for this file')
else:
out = fitsio.FITS(outfile, mode='rw')
out[1].append(arr)
logger.debug("Writing %s" % outfile)
out.close()
def query_server(self, outfile, age, metallicity):
z = metallicity
feh = self.z2feh(z)
params = dict(self.download_defaults)
params['output'] = dict_output[self.survey]
params['FeH_value'] = feh
params['age_value'] = age * 1e9
if params['age_scale'] == 'log10':
params['age_value'] = np.log10(params['age_value'])
server = self.download_url
url = server + '/iso_form.php'
# First check that the server is alive
logger.debug("Accessing %s..." % url)
urlopen(url, timeout=2)
#response = requests.post(url,data=params)
q = urlencode(params).encode('utf-8')
request = Request(url, data=q)
response = urlopen(request)
try:
fname = os.path.basename(str(response.read()).split('"')[1])
except Exception as e:
logger.debug(str(e))
msg = 'Output filename not found'
raise RuntimeError(msg)
tmpdir = os.path.dirname(tempfile.NamedTemporaryFile().name)
tmpfile = os.path.join(tmpdir, fname)
out = '{0}/tmp/{1}'.format(server, fname)
cmd = 'wget --progress dot:binary %s -P %s' % (out, tmpdir)
logger.debug(cmd)
stdout = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
logger.debug(stdout)
cmd = 'unzip %s -d %s' % (tmpfile, tmpdir)
logger.debug(cmd)
stdout = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
logger.debug(stdout)
logger.debug("Creating %s..." % outfile)
shutil.move(tmpfile.replace('.zip', '.cmd'), outfile)
os.remove(tmpfile)
return outfile
def load(args):
infile, columns = args
logger.debug("Loading %s..." % infile)
return fitsio.read(infile, columns=columns)
def search(self, coords=None, distance_modulus=None, tolerance=1.e-2):
"""
Organize a grid search over ROI target pixels and distance moduli in distance_modulus_array
coords: (lon,lat)
distance_modulus: scalar
"""
nmoduli = len(self.distance_modulus_array)
npixels = len(self.roi.pixels_target)
self.log_likelihood_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_lower_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_upper_sparse_array = numpy.zeros([nmoduli, npixels])
self.richness_upper_limit_sparse_array = numpy.zeros(
[nmoduli, npixels])
self.stellar_mass_sparse_array = numpy.zeros([nmoduli, npixels])
self.fraction_observable_sparse_array = numpy.zeros([nmoduli, npixels])
# Specific pixel/distance_modulus
coord_idx, distance_modulus_idx = None, None
if coords is not None:
# Match to nearest grid coordinate index
coord_idx = self.roi.indexTarget(coords[0], coords[1])
if distance_modulus is not None:
# Match to nearest distance modulus index
distance_modulus_idx = np.fabs(self.distance_modulus_array -
distance_modulus).argmin()
lon, lat = self.roi.pixels_target.lon, self.roi.pixels_target.lat
logger.info('Looping over distance moduli in grid search ...')
for ii, distance_modulus in enumerate(self.distance_modulus_array):
# Specific pixel
if distance_modulus_idx is not None:
if ii != distance_modulus_idx: continue
logger.info(' (%-2i/%i) Distance Modulus=%.1f ...' %
(ii + 1, nmoduli, distance_modulus))
# Set distance_modulus once to save time
self.loglike.set_params(distance_modulus=distance_modulus)
for jj in range(0, npixels):
# Specific pixel
if coord_idx is not None:
if jj != coord_idx: continue
# Set kernel location
self.loglike.set_params(lon=lon[jj], lat=lat[jj])
# Doesn't re-sync distance_modulus each time
self.loglike.sync_params()
args = (jj + 1, npixels, self.loglike.source.lon,
self.loglike.source.lat)
message = ' (%-3i/%i) Candidate at (%.2f, %.2f) ... ' % (
args)
self.log_likelihood_sparse_array[ii][
jj], self.richness_sparse_array[ii][
jj], parabola = self.loglike.fit_richness()
self.stellar_mass_sparse_array[ii][
jj] = self.stellar_mass_conversion * self.richness_sparse_array[
ii][jj]
self.fraction_observable_sparse_array[ii][jj] = self.loglike.f
if self.config['scan']['full_pdf']:
#n_pdf_points = 100
#richness_range = parabola.profileUpperLimit(delta=25.) - self.richness_sparse_array[ii][jj]
#richness = numpy.linspace(max(0., self.richness_sparse_array[ii][jj] - richness_range),
# self.richness_sparse_array[ii][jj] + richness_range,
# n_pdf_points)
#if richness[0] > 0.:
# richness = numpy.insert(richness, 0, 0.)
# n_pdf_points += 1
#
#log_likelihood = numpy.zeros(n_pdf_points)
#for kk in range(0, n_pdf_points):
# log_likelihood[kk] = self.loglike.value(richness=richness[kk])
#parabola = ugali.utils.parabola.Parabola(richness, 2.*log_likelihood)
#self.richness_lower_sparse_array[ii][jj], self.richness_upper_sparse_array[ii][jj] = parabola.confidenceInterval(0.6827)
self.richness_lower_sparse_array[ii][
jj], self.richness_upper_sparse_array[ii][
jj] = self.loglike.richness_interval(0.6827)
self.richness_upper_limit_sparse_array[ii][
jj] = parabola.bayesianUpperLimit(0.95)
args = (2. * self.log_likelihood_sparse_array[ii][jj],
self.stellar_mass_conversion *
self.richness_sparse_array[ii][jj],
self.stellar_mass_conversion *
self.richness_lower_sparse_array[ii][jj],
self.stellar_mass_conversion *
self.richness_upper_sparse_array[ii][jj],
self.stellar_mass_conversion *
self.richness_upper_limit_sparse_array[ii][jj])
message += 'TS=%.1f, Stellar Mass=%.1f (%.1f -- %.1f @ 0.68 CL, < %.1f @ 0.95 CL)' % (
args)
else:
args = (2. * self.log_likelihood_sparse_array[ii][jj],
self.stellar_mass_conversion *
self.richness_sparse_array[ii][jj],
self.fraction_observable_sparse_array[ii][jj])
message += 'TS=%.1f, Stellar Mass=%.1f, Fraction=%.2g' % (
args)
logger.debug(message)
#if coords is not None and distance_modulus is not None:
# results = [self.richness_sparse_array[ii][jj],
# self.log_likelihood_sparse_array[ii][jj],
# self.richness_lower_sparse_array[ii][jj],
# self.richness_upper_sparse_array[ii][jj],
# self.richness_upper_limit_sparse_array[ii][jj],
# richness, log_likelihood, self.loglike.p, self.loglike.f]
# return results
jj_max = self.log_likelihood_sparse_array[ii].argmax()
args = (jj_max + 1, npixels, lon[jj_max], lat[jj_max],
2. * self.log_likelihood_sparse_array[ii][jj_max],
self.stellar_mass_conversion *
self.richness_sparse_array[ii][jj_max])
message = ' (%-3i/%i) Maximum at (%.2f, %.2f) ... TS=%.1f, Stellar Mass=%.1f' % (
args)
logger.info(message)
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s" % infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field' % field]
magerr_column = self.config['catalog']['mag_err_%i_field' % field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band' % field]
pixel_pix_name = 'PIX%i' % self.nside_pixel
# If the data already has a healpix pixel assignment then use it
# Otherwise recalculate...
try:
data = fitsio.read(infile, columns=[pixel_pix_name])
except ValueError as e:
logger.info(str(e))
columns = [
self.config['catalog']['lon_field'],
self.config['catalog']['lat_field']
]
data = fitsio.read(infile, columns=columns)[columns]
pix = ang2pix(self.nside_pixel, data[columns[0]], data[columns[1]])
data = recfuncs.rec_append_fields(data, pixel_pix_name, pix)
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0, dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix, self.nside_pixel,
self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix, pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1 / signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx + num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s" % infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i' % (min_num, pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr > 0.9 * magerr_lim)
& (magerr < 1.1 * magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g" %
(pix, num, mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask,
self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels, subpix)
out_maglims = np.append(out_maglims, maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s" % self.footfile)
lon, lat = pix2ang(self.nside_pixel, out_pixels)
if self.config['coords']['coordsys'] == 'gal':
ra, dec = gal2cel(lon, lat)
else:
ra, dec = lon, lat
footprint = inFootprint(self.footprint, ra, dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f" %
(np.mean(out_maglims), np.std(out_maglims)))
return out_pixels, out_maglims
def findObjects(pixels, values, nside, zvalues, rev, good):
"""
Characterize labelled candidates in a multi-dimensional HEALPix map.
Parameters:
values : (Sparse) HEALPix array of data values
nside : HEALPix dimensionality
pixels : Pixel values associated to (sparse) HEALPix array
zvalues : Values of the z-dimension (usually distance modulus)
rev : Reverse indices for pixels in each "island"
good : Array containg labels for each "island"
Returns:
objs : numpy.recarray of object characteristics
"""
ngood = len(good)
objs = numpy.recarray((ngood, ),
dtype=[
('LABEL', 'i4'),
('NPIX', 'i4'),
('VAL_MAX', 'f4'),
('IDX_MAX', 'i4'),
('ZIDX_MAX', 'i4'),
('PIX_MAX', 'i4'),
('X_MAX', 'f4'),
('Y_MAX', 'f4'),
('Z_MAX', 'f4'),
('X_CENT', 'f4'),
('Y_CENT', 'f4'),
('Z_CENT', 'f4'),
('X_BARY', 'f4'),
('Y_BARY', 'f4'),
('Z_BARY', 'f4'),
('CUT', 'i2'),
])
objs['CUT'][:] = 0
shape = values.shape
ncol = shape[1]
for i in range(0, ngood):
logger.debug("i=%i", i)
# This code could use some cleanup...
indices = rev[rev[good[i]]:rev[good[i] + 1]]
npix = len(indices)
idx = indices // ncol # This is the spatial index
zidx = indices % ncol # This is the distance index
pix = pixels[idx] # This is the healpix pixel
xval, yval = pix2ang(nside, pix)
zval = zvalues[zidx]
objs[i]['LABEL'] = good[i]
objs[i]['NPIX'] = npix
logger.debug("LABEL=%i" % objs[i]['LABEL'])
logger.debug("NPIX=%i" % objs[i]['NPIX'])
island = values[idx, zidx]
idxmax = island.argmax()
xval_max, yval_max, zval_max = xval[idxmax], yval[idxmax], zval[
idxmax]
objs[i]['VAL_MAX'] = island[idxmax]
objs[i]['IDX_MAX'] = idx[idxmax]
objs[i]['ZIDX_MAX'] = zidx[idxmax]
objs[i]['PIX_MAX'] = pix[idxmax]
objs[i]['X_MAX'] = xval_max
objs[i]['Y_MAX'] = yval_max
objs[i]['Z_MAX'] = zval_max
proj = Projector(xval_max, yval_max)
xpix, ypix = proj.sphereToImage(xval, yval)
# Projected centroid
x_cent, y_cent, zval_cent = numpy.average([xpix, ypix, zval],
axis=1)
xval_cent, yval_cent = proj.imageToSphere(x_cent, y_cent)
objs[i]['X_CENT'] = xval_cent
objs[i]['Y_CENT'] = yval_cent
objs[i]['Z_CENT'] = zval_cent
# Projected barycenter
weights = [island, island, island]
x_bary, y_bary, zval_bary = numpy.average([xpix, ypix, zval],
weights=weights,
axis=1)
xval_bary, yval_bary = proj.imageToSphere(x_bary, y_bary)
objs[i]['X_BARY'] = xval_bary
objs[i]['Y_BARY'] = yval_bary
objs[i]['Z_BARY'] = zval_bary
return objs
def wrapper(*args, **kwargs):
logger.debug("Running %r..."%(func.__name__))
t0=time.time()
ret = func(*args,**kwargs)
logger.debug('%4.2fs'%(time.time()-t0))
return ret
def call(self, command):
logger.debug(command)
return sub.call(command,shell=True)
ebvval = ebv(ra, dec, args.ebv)
values = [ebvval]
dtypes = [('EBV', 'f4')]
logger.info("Calculating extinction %s..." % args.ebv)
for b in args.bands:
if b in DESDM:
band = np.repeat(b, len(ra))
extname = args.ext + '_%s' % b.upper()
else:
band = data[b]
extname = args.ext
extval = extinction(ebvval, band)
values.append(extval)
dtypes.append((extname, 'f4'))
out = np.rec.fromarrays(values, dtype=dtypes)
# Writing...
if not args.outfile:
args.outfile = args.infile
else:
logger.debug("Copying %s to %s..." % (args.infile, args.outfile))
shutil.copy(args.infile, args.outfile)
logger.info("Writing %s..." % args.outfile)
writefile(args.outfile, out, force=args.force)
logger.info("Done.")
def call(self, command):
logger.debug(command)
return sub.call(command, shell=True)
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
footfile = self.config['data']['footprint']
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
f = pyfits.open(infile)
header = f[1].header
data = f[1].data
#mask_pixels = numpy.arange( healpy.nside2npix(self.nside_mask), dtype='int')
mask_maglims = numpy.zeros( healpy.nside2npix(self.nside_mask) )
out_pixels = numpy.zeros(0,dtype='int')
out_maglims = numpy.zeros(0)
# Find the objects in each pixel
pixel_pix = data['PIX%i'%self.nside_pixel]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = numpy.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = numpy.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = numpy.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~numpy.isnan(y)],y[~numpy.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = numpy.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = numpy.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = numpy.append(out_pixels,subpix)
out_maglims = numpy.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = numpy.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
logger.info("Checking footprint against %s"%footfile)
glon,glat = pix2ang(self.nside_pixel,out_pixels)
ra,dec = gal2cel(glon,glat)
footprint = inFootprint(footfile,ra,dec)
idx = numpy.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(numpy.mean(out_maglims),numpy.std(out_maglims)))
return out_pixels,out_maglims
def search(self,
coords=None,
distance_modulus=None,
extension=None,
tolerance=1.e-2):
"""
Organize a grid search over ROI target pixels, distance
moduli, and extensions. If coords, distance_modulus, or
extension is specified, then the nearest value in the
predefined scan grid is used. ***This may be different than
the input value.**
Parameters
----------
coords : (float,float)
coordinate to search (matched to nearest scan value)
distance_modulus : float
distance modulus to search (matched to nearest scan value)
extension : float
extension to search (matched to nearest scan value)
tolerance : float
tolerance on richness maximization
Returns
-------
None
"""
nmoduli = len(self.distance_modulus_array)
npixels = len(self.roi.pixels_target)
self.loglike_array = np.zeros([nmoduli, npixels], dtype='f4')
self.richness_array = np.zeros([nmoduli, npixels], dtype='f4')
self.stellar_mass_array = np.zeros([nmoduli, npixels], dtype='f4')
self.fraction_observable_array = np.zeros([nmoduli, npixels],
dtype='f4')
self.extension_fit_array = np.zeros([nmoduli, npixels], dtype='f4')
# DEPRECATED: ADW 2019-04-27
self.richness_lower_array = np.zeros([nmoduli, npixels], dtype='f4')
self.richness_upper_array = np.zeros([nmoduli, npixels], dtype='f4')
self.richness_ulimit_array = np.zeros([nmoduli, npixels], dtype='f4')
# Specific pixel/distance_modulus
coord_idx, distance_modulus_idx, extension_idx = None, None, None
if coords is not None:
# Match to nearest grid coordinate index
coord_idx = self.roi.indexTarget(coords[0], coords[1])
if distance_modulus is not None:
# Match to nearest distance modulus index
distance_modulus_idx = np.fabs(self.distance_modulus_array -
distance_modulus).argmin()
if extension is not None:
# Match to nearest extension
extension_idx = np.fabs(self.extension_array - extension).argmin()
lon, lat = self.roi.pixels_target.lon, self.roi.pixels_target.lat
logger.info('Looping over distance moduli in grid search ...')
for ii, distance_modulus in enumerate(self.distance_modulus_array):
# Specific distance
if distance_modulus_idx is not None:
if ii != distance_modulus_idx: continue
logger.info(' (%-2i/%i) Distance Modulus=%.1f ...' %
(ii + 1, nmoduli, distance_modulus))
# No objects, continue
if len(self.loglike.catalog) == 0:
logger.warn("No catalog objects")
continue
# Set distance_modulus once to save time
self.loglike.set_params(distance_modulus=distance_modulus)
# Loop over pixels
for jj in range(0, npixels):
# Specific pixel
if coord_idx is not None:
if jj != coord_idx: continue
# Set kernel location
self.loglike.set_params(lon=lon[jj], lat=lat[jj])
loglike = 0
# Loop over extensions
for kk, ext in enumerate(self.extension_array):
# Specific extension
if extension_idx is not None:
if kk != extension_idx: continue
# Set extension
self.loglike.set_params(extension=ext)
# Doesn't re-sync distance_modulus each time
self.loglike.sync_params()
# Maximize the likelihood with respect to richness
loglike, rich, p = self.loglike.fit_richness()
if loglike < self.loglike_array[ii][jj]:
# No loglike increase, continue
continue
self.loglike_array[ii][jj], self.richness_array[ii][
jj], parabola = loglike, rich, p
self.stellar_mass_array[ii][
jj] = self.stellar_mass_conversion * self.richness_array[
ii][jj]
self.fraction_observable_array[ii][jj] = self.loglike.f
self.extension_fit_array[ii][jj] = self.source.extension
# ADW: Careful, we are leaving the extension at the
# last value in the array, not at the maximum...
# Debug output
args = (jj + 1, npixels, lon[jj], lat[jj],
2. * self.loglike_array[ii][jj],
self.stellar_mass_array[ii][jj],
self.fraction_observable_array[ii][jj],
self.extension_fit_array[ii][jj])
msg = ' (%-3i/%i) Candidate at (%.2f, %.2f) ... '
msg += 'TS=%.1f, Mstar=%.2g, ObsFrac=%.2g, Ext=%.2g'
logger.debug(msg % args)
"""
# This is debugging output
if self.config['scan']['full_pdf']:
DeprecationWarning("'full_pdf' is deprecated.")
self.richness_lower_array[ii][jj], self.richness_upper_array[ii][jj] = self.loglike.richness_interval(0.6827)
self.richness_ulimit_array[ii][jj] = parabola.bayesianUpperLimit(0.95)
args = (
2. * self.loglike_array[ii][jj],
self.stellar_mass_conversion*self.richness_array[ii][jj],
self.stellar_mass_conversion*self.richness_lower_array[ii][jj],
self.stellar_mass_conversion*self.richness_upper_array[ii][jj],
self.stellar_mass_conversion*self.richness_ulimit_array[ii][jj]
)
msg = 'TS=%.1f, Stellar Mass=%.1f (%.1f -- %.1f @ 0.68 CL, < %.1f @ 0.95 CL)'%(args)
logger.debug(msg)
"""
jj_max = self.loglike_array[ii].argmax()
args = (jj_max + 1, npixels, lon[jj_max], lat[jj_max],
2. * self.loglike_array[ii][jj_max],
self.stellar_mass_conversion *
self.richness_array[ii][jj_max],
self.extension_fit_array[ii][jj_max])
msg = ' (%-3i/%i) Max at (%.2f, %.2f) : TS=%.1f, Mstar=%.2g, Ext=%.2f' % (
args)
logger.info(msg)
def sample(self, mode='data', mass_steps=1000, mass_min=0.1, full_data_range=False):
"""Sample the isochrone in steps of mass interpolating between the
originally defined isochrone points.
Parameters:
-----------
mode :
mass_steps :
mass_min : Minimum mass [Msun]
full_data_range :
Returns:
--------
mass_init : Initial mass of each point
mass_pdf : PDF of number of stars in each point
mass_act : Actual (current mass) of each stellar point
mag_1 : Array of magnitudes in first band (distance modulus applied)
mag_2 : Array of magnitudes in second band (distance modulus applied)
"""
if full_data_range:
# ADW: Might be depricated 02/10/2015
# Generate points over full isochrone data range
select = slice(None)
else:
# Not generating points for the post-AGB stars,
# but still count those stars towards the normalization
select = slice(self.index)
mass_steps = int(mass_steps)
mass_init = self.mass_init[select]
mass_act = self.mass_act[select]
mag_1 = self.mag_1[select]
mag_2 = self.mag_2[select]
# ADW: Assume that the isochrones are pre-sorted by mass_init
# This avoids some numerical instability from points that have the same
# mass_init value (discontinuities in the isochrone).
# ADW: Might consider using np.interp for speed
mass_act_interpolation = scipy.interpolate.interp1d(mass_init, mass_act,assume_sorted=True)
mag_1_interpolation = scipy.interpolate.interp1d(mass_init, mag_1,assume_sorted=True)
mag_2_interpolation = scipy.interpolate.interp1d(mass_init, mag_2,assume_sorted=True)
# ADW: Any other modes possible?
if mode=='data':
# Mass interpolation with uniform coverage between data points from isochrone file
mass_interpolation = scipy.interpolate.interp1d(np.arange(len(mass_init)), mass_init)
mass_array = mass_interpolation(np.linspace(0, len(mass_init)-1, mass_steps+1))
d_mass = mass_array[1:] - mass_array[:-1]
mass_init_array = np.sqrt(mass_array[1:] * mass_array[:-1])
mass_pdf_array = d_mass * self.imf.pdf(mass_init_array, log_mode=False)
mass_act_array = mass_act_interpolation(mass_init_array)
mag_1_array = mag_1_interpolation(mass_init_array)
mag_2_array = mag_2_interpolation(mass_init_array)
# Horizontal branch dispersion
if self.hb_spread and (self.stage==self.hb_stage).any():
logger.debug("Performing dispersion of horizontal branch...")
mass_init_min = self.mass_init[self.stage==self.hb_stage].min()
mass_init_max = self.mass_init[self.stage==self.hb_stage].max()
cut = (mass_init_array>mass_init_min)&(mass_init_array<mass_init_max)
if isinstance(self.hb_spread,collections.Iterable):
# Explicit dispersion spacing
dispersion_array = self.hb_spread
n = len(dispersion_array)
else:
# Default dispersion spacing
dispersion = self.hb_spread
spacing = 0.025
n = int(round(2.0*self.hb_spread/spacing))
if n % 2 != 1: n += 1
dispersion_array = np.linspace(-dispersion, dispersion, n)
# Reset original values
mass_pdf_array[cut] = mass_pdf_array[cut] / float(n)
# Isochrone values for points on the HB
mass_init_hb = mass_init_array[cut]
mass_pdf_hb = mass_pdf_array[cut]
mass_act_hb = mass_act_array[cut]
mag_1_hb = mag_1_array[cut]
mag_2_hb = mag_2_array[cut]
# Add dispersed values
for dispersion in dispersion_array:
if dispersion == 0.: continue
msg = 'Dispersion=%-.4g, HB Points=%i, Iso Points=%i'%(dispersion,cut.sum(),len(mass_init_array))
logger.debug(msg)
mass_init_array = np.append(mass_init_array, mass_init_hb)
mass_pdf_array = np.append(mass_pdf_array, mass_pdf_hb)
mass_act_array = np.append(mass_act_array, mass_act_hb)
mag_1_array = np.append(mag_1_array, mag_1_hb + dispersion)
mag_2_array = np.append(mag_2_array, mag_2_hb + dispersion)
# Note that the mass_pdf_array is not generally normalized to unity
# since the isochrone data range typically covers a different range
# of initial masses
#mass_pdf_array /= np.sum(mass_pdf_array) # ORIGINAL
# Normalize to the number of stars in the satellite with mass > mass_min
mass_pdf_array /= self.imf.integrate(mass_min, self.mass_init_upper_bound)
out = np.vstack([mass_init_array,mass_pdf_array,mass_act_array,mag_1_array,mag_2_array])
return out
def validateSatellite(config, isochrone, kernel, stellar_mass, distance_modulus, trials=1, debug=False, seed=0):
"""
Tool for simple MC validation studies -- specifically to create multiple realizations of
a satellite given an CompositeIsochrone object, Kernel object, stellar mass (M_sol) for normalization,
and distance_modulus.
"""
logger.info('=== Validate Satellite ===')
config.params['kernel']['params'] = [kernel.r_h] # TODO: Need better solution to update size??
logger.debug('Using Plummer profile spatial model with half-light radius %.2f deg'%(config.params['kernel']['params'][0]))
roi = ugali.observation.roi.ROI(config, kernel.lon, kernel.lat)
simulator = ugali.simulation.simulator.Simulator(config, roi=roi)
catalog_base = ugali.observation.catalog.Catalog(config,roi=roi)
mask = ugali.observation.mask.Mask(config, roi)
coords = (kernel.lon, kernel.lat)
results = {'mc_lon': [],
'mc_lat': [],
'mc_distance_modulus': [],
'mc_stellar_mass': [],
'mc_radius': [],
'richness': [],
'log_likelihood': [],
'richness_lower': [],
'richness_upper': [],
'richness_limit': [],
'f': [],
'stellar_mass': []}
numpy.random.seed(seed)
for ii in range(0, trials):
logger.info('=== Running Satellite %i ==='%ii)
# Simulate
catalog_satellite = simulator.satellite(isochrone, kernel, stellar_mass, distance_modulus, mc_source_id=1)
#catalog_bootstrap = catalog_base.bootstrap()
#catalog_merge = ugali.observation.catalog.mergeCatalogs([catalog_bootstrap, catalog_satellite])
catalog_background = simulator.satellite(mc_source_id=1)
catalog_merge = ugali.observation.catalog.mergeCatalogs([catalog_background, catalog_satellite])
# Analyze
likelihood = ugali.analysis.likelihood.Likelihood(config, roi, mask, catalog_merge, isochrone, kernel)
likelihood.precomputeGridSearch([distance_modulus])
richness, log_likelihood, richness_lower, richness_upper, richness_upper_limit, richness_raw, log_likelihood_raw, p, f = likelihood.gridSearch(coords=coords, distance_modulus_index=0)
results['mc_lon'].append(kernel.lon)
results['mc_lat'].append(kernel.lat)
results['mc_distance_modulus'].append(distance_modulus)
results['mc_stellar_mass'].append(stellar_mass)
results['mc_radius'].append(kernel.r_h)
results['richness'].append(richness)
results['log_likelihood'].append(log_likelihood)
results['richness_lower'].append(richness_lower)
results['richness_upper'].append(richness_upper)
results['richness_limit'].append(richness_upper_limit)
results['f'].append(f)
results['stellar_mass'].append(richness * isochrone.stellarMass())
logger.info('MC Stellar Mass = %.2f, Measured Stellar Mass = %.2f'%(stellar_mass,richness * isochrone.stellarMass()))
if debug:
return likelihood, richness, log_likelihood, richness_lower, richness_upper, richness_upper_limit, richness_raw, log_likelihood_raw, p, f
return results
def get_results(self, **kwargs):
kwargs.setdefault('alpha', self.alpha)
kwargs.setdefault('burn', self.nburn * self.nwalkers)
# Calculate best-fit parameters from MCMC chain
logger.debug('Estimating parameters...')
estimate = self.estimate_params(**kwargs)
params = {k: v[0] for k, v in estimate.items()}
results = dict(estimate)
# Extra parameters from the MCMC chain
logger.debug('Estimating auxiliary parameters...')
try:
results['ra'] = self.estimate('ra', **kwargs)
results['dec'] = self.estimate('dec', **kwargs)
except KeyError:
logger.warn("Didn't find 'ra' or 'dec'")
ra, dec = gal2cel(results['lon'][0], results['lat'][0])
results['ra'] = ugali.utils.stats.interval(ra)
results['dec'] = ugali.utils.stats.interval(dec)
ra, dec = results['ra'][0], results['dec'][0]
glon, glat = lon, lat = results['lon'][0], results['lat'][0]
results.update(gal=[float(glon), float(glat)])
results.update(cel=[float(ra), float(dec)])
try:
results['position_angle_cel'] = self.estimate(
'position_angle_cel', **kwargs)
except KeyError:
results['position_angle_cel'] = ugali.utils.stats.interval(np.nan)
# Update the loglike to the best-fit parameters from the chain
logger.debug('Calculating TS...')
ts = 2 * self.loglike.value(**params)
results['ts'] = ugali.utils.stats.interval(ts, np.nan, np.nan)
#lon,lat = estimate['lon'][0],estimate['lat'][0]
#
#results.update(gal=[float(lon),float(lat)])
#ra,dec = gal2cel(lon,lat)
#results.update(cel=[float(ra),float(dec)])
#results['ra'] = ugali.utils.stats.interval(ra,np.nan,np.nan)
#results['dec'] = ugali.utils.stats.interval(dec,np.nan,np.nan)
# Celestial position angle
# Break ambiguity in direction with '% 180.'
pa, pa_err = results['position_angle']
pa_cel = gal2cel_angle(lon, lat, pa) % 180.
pa_cel_err = np.array(pa_err) - pa + pa_cel
results['position_angle_cel'] = ugali.utils.stats.interval(
pa_cel, pa_cel_err[0], pa_cel_err[1])
mod, mod_err = estimate['distance_modulus']
dist = mod2dist(mod)
dist_lo, dist_hi = [mod2dist(mod_err[0]), mod2dist(mod_err[1])]
results['distance'] = ugali.utils.stats.interval(
dist, dist_lo, dist_hi)
dist, dist_err = results['distance']
ext, ext_err = estimate['extension']
ext_sigma = np.nan_to_num(np.array(ext_err) - ext)
results['extension_arcmin'] = ugali.utils.stats.interval(
60 * ext, 60 * ext_err[0], 60 * ext_err[1])
# Radially symmetric extension (correct for ellipticity).
ell, ell_err = estimate['ellipticity']
rext, rext_err = ext * np.sqrt(1 -
ell), np.array(ext_err) * np.sqrt(1 -
ell)
rext_sigma = np.nan_to_num(np.array(rext_err) - rext)
results['extension_radial'] = ugali.utils.stats.interval(
rext, rext_err[0], rext_err[1])
results['extension_radial_arcmin'] = ugali.utils.stats.interval(
60 * rext, 60 * rext_err[0], 60 * rext_err[1])
# Bayes factor for ellipticity
results['ellipticity_bayes_factor'] = self.bayes_factor(
'ellipticity', burn=kwargs['burn'])
# Physical Size (should do this with the posteriors)
# Radially symmetric
dist_sigma = np.nan_to_num(np.array(dist_err) - dist)
size = np.arctan(np.radians(ext)) * dist
size_sigma = size * np.sqrt((ext_sigma / ext)**2 +
(dist_sigma / dist)**2)
size_err = [size - size_sigma[0], size + size_sigma[1]]
results['physical_size'] = ugali.utils.stats.interval(
size, size_err[0], size_err[1])
rsize = np.arctan(np.radians(rext)) * dist
rsize_sigma = rsize * np.sqrt((rext_sigma / rext)**2 +
(dist_sigma / dist)**2)
rsize_err = [rsize - rsize_sigma[0], rsize + rsize_sigma[1]]
results['physical_size_radial'] = ugali.utils.stats.interval(
rsize, rsize_err[0], rsize_err[1])
# Richness
rich, rich_err = estimate['richness']
# Number of observed stars (sum of p-values)
nobs = self.loglike.p.sum()
nobs_lo, nobs_hi = nobs + np.sqrt(nobs) * np.array([-1, 1])
results['nobs'] = ugali.utils.stats.interval(nobs, nobs_lo, nobs_hi)
# Number of predicted stars (pixelization effects?)
npred = self.loglike.f * rich
npred_lo, npred_hi = rich_err[0] * self.loglike.f, rich_err[
1] * self.loglike.f
results['npred'] = ugali.utils.stats.interval(npred, npred_lo,
npred_hi)
# Careful, depends on the isochrone...
stellar_mass = self.source.stellar_mass()
mass = rich * stellar_mass
mass_lo, mass_hi = rich_err[0] * stellar_mass, rich_err[
1] * stellar_mass
results['mass'] = ugali.utils.stats.interval(mass, mass_lo, mass_hi)
stellar_luminosity = self.source.stellar_luminosity()
lum = rich * stellar_luminosity
lum_lo, lum_hi = rich_err[0] * stellar_luminosity, rich_err[
1] * stellar_luminosity
results['luminosity'] = ugali.utils.stats.interval(lum, lum_lo, lum_hi)
Mv = self.source.absolute_magnitude(rich)
Mv_lo = self.source.absolute_magnitude(rich_err[0])
Mv_hi = self.source.absolute_magnitude(rich_err[1])
results['Mv'] = ugali.utils.stats.interval(Mv, Mv_lo, Mv_hi)
# ADW: WARNING this is very fragile.
# Also, this is not quite right, should cut on the CMD available space
kwargs = dict(richness=rich,
mag_bright=16.,
mag_faint=23.,
n_trials=5000,
alpha=self.alpha,
seed=0)
martin = self.config['results'].get('martin')
if martin:
logger.info("Calculating Martin magnitude...")
if martin > 1: kwargs['n_trials'] = martin
Mv_martin = self.source.isochrone.absolute_magnitude_martin(
**kwargs)
results['Mv_martin'] = Mv_martin
else:
logger.warning("Skipping Martin magnitude")
results['Mv_martin'] = np.nan
mu = surfaceBrightness(Mv, size, dist)
results['surface_brightness'] = ugali.utils.stats.interval(
mu, np.nan, np.nan)
try:
results['constellation'] = ugali.utils.projector.ang2const(
lon, lat)[1]
except:
pass
results['iau'] = ugali.utils.projector.ang2iau(lon, lat)
coord = SkyCoord(ra * u.deg, dec * u.deg, distance=dist * u.kpc)
results['ra_sex'] = str(coord.ra.to_string())
results['dec_sex'] = str(coord.dec.to_string())
# Calculate some separations from GC, LMC, SMC
#NED coordinates with de Grisj distance
LMC = SkyCoord(80.8939 * u.deg,
-69.7561 * u.deg,
distance=49.89 * u.kpc)
#NED coordinates with de Grisj distance
SMC = SkyCoord(13.1866 * u.deg,
-72.8286 * u.deg,
distance=61.94 * u.kpc)
# GC from astropy?
GC = SkyCoord(266.4168262 * u.deg,
-29.0077969 * u.deg,
distance=8.0 * u.kpc)
results['d_gc'] = coord.separation_3d(GC).value
results['d_lmc'] = coord.separation_3d(LMC).value
results['d_smc'] = coord.separation_3d(SMC).value
try:
results['feh'] = float(self.source.isochrone.feh)
except:
results['feh'] = np.nan
output = dict()
output['params'] = params
output['results'] = results
return output
def run(self):
outdir=mkdir(self.config['output']['simdir'])
logdir=mkdir(join(outdir,'log'))
if 'simulate' in self.opts.run:
logger.info("Running 'simulate'...")
if self.opts.num is None: self.opts.num = self.config['simulator']['njobs']
for i in range(self.opts.num):
outfile=join(outdir,self.config['output']['simfile']%i)
base = splitext(os.path.basename(outfile))[0]
logfile=join(logdir,base+'.log')
jobname=base
script = self.config['simulator']['script']
cmd='%s %s %s --seed %i'%(script,self.opts.config,outfile,i)
#cmd='%s %s %s'%(script,self.opts.config,outfile)
self.batch.submit(cmd,jobname,logfile)
time.sleep(0.1)
if 'analyze' in self.opts.run:
logger.info("Running 'analyze'...")
dirname = self.config['simulate']['dirname']
catfiles = sorted(glob.glob(join(dirname,self.config['simulate']['catfile'])))
popfile = join(dirname,self.config['simulate']['popfile'])
batch = self.config['simulate']['batch']
for i,catfile in enumerate(catfiles):
basename = os.path.basename(catfile)
outfile = join(outdir,basename)
if exists(outfile) and not self.opts.force:
logger.info("Found %s; skipping..."%outfile)
continue
base = splitext(os.path.basename(outfile))[0]
logfile=join(logdir,base+'.log')
jobname=base
script = self.config['simulate']['script']
cmd='%s %s -p %s -c %s -o %s'%(script,self.opts.config,popfile,catfile,outfile)
self.batch.max_jobs = batch.get('max_jobs',200)
opts = batch.get(self.opts.queue,dict())
self.batch.submit(cmd,jobname,logfile,**opts)
time.sleep(0.1)
if 'sensitivity' in self.opts.run:
logger.info("Running 'sensitivity'...")
if 'merge' in self.opts.run:
logger.info("Running 'merge'...")
filenames=join(outdir,self.config['output']['simfile']).split('_%')[0]+'_*'
infiles=sorted(glob.glob(filenames))
f = fitsio.read(infiles[0])
table = np.empty(0,dtype=data.dtype)
for filename in infiles:
logger.debug("Reading %s..."%filename)
d = fitsio.read(filename)
t = d[~np.isnan(d['ts'])]
table = recfuncs.stack_arrays([table,t],usemask=False,asrecarray=True)
logger.info("Found %i simulations."%len(table))
outfile = join(outdir,"merged_sims.fits")
logger.info("Writing %s..."%outfile)
fitsio.write(outfile,table,clobber=True)
if 'plot' in self.opts.run:
logger.info("Running 'plot'...")
import ugali.utils.plotting
import pylab as plt
plotdir = mkdir(self.config['output']['plotdir'])
data = fitsio.read(join(outdir,"merged_sims.fits"))
data = data[~np.isnan(data['ts'])]
bigfig,bigax = plt.subplots()
for dist in np.unique(data['fit_distance']):
logger.info(' Plotting distance: %s'%dist)
ts = data['ts'][data['fit_distance'] == dist]
ugali.utils.plotting.drawChernoff(bigax,ts,bands='none',color='gray')
fig,ax = plt.subplots(1,2,figsize=(10,5))
ugali.utils.plotting.drawChernoff(ax[0],ts,bands='none',pdf=True)
ugali.utils.plotting.drawChernoff(ax[1],ts)
fig.suptitle(r'Chernoff ($\mu = %g$)'%dist)
ax[0].annotate(r"$N=%i$"%len(ts), xy=(0.15,0.85), xycoords='axes fraction',
bbox={'boxstyle':"round",'fc':'1'})
basename = 'chernoff_u%g.png'%dist
outfile = os.path.join(plotdir,basename)
plt.savefig(outfile)
bigfig.suptitle('Chernoff!')
basename = 'chernoff_all.png'
outfile = os.path.join(plotdir,basename)
plt.savefig(outfile)
#idx=np.random.randint(len(data['ts'])-1,size=400)
#idx=slice(400)
#ugali.utils.plotting.plotChernoff(data['ts'][idx])
#ugali.utils.plotting.plotChernoff(data['fit_ts'])
plt.ion()
"""
def mergeSparseHealpixMaps(infiles,
outfile=None,
pix_data_extension='PIX_DATA',
pix_field='PIX',
distance_modulus_extension='DISTANCE_MODULUS',
distance_modulus_field='DISTANCE_MODULUS',
default_value=healpy.UNSEEN):
"""
Use the first infile to determine the basic contents to expect for the other files.
"""
# Setup
if isinstance(infiles, str): infiles = [infiles]
distance_modulus_array = None
pix_array = []
data_dict = {}
reader = pyfits.open(infiles[0])
nside = reader[pix_data_extension].header['NSIDE']
for ii in range(0, len(reader)):
if reader[ii].name == distance_modulus_extension:
distance_modulus_array = reader[
distance_modulus_extension].data.field(distance_modulus_field)
for key in reader[pix_data_extension].data.names:
if key == pix_field:
continue
data_dict[key] = []
#if distance_modulus_array is None:
# data_dict[key] = default_value * numpy.ones(healpy.nside2npix(nside))
#else:
# data_dict[key] = default_value * numpy.ones([len(distance_modulus_array),
# healpy.nside2npix(nside)])
reader.close()
# Now loop over the infiles
for ii in range(0, len(infiles)):
logger.debug('(%i/%i) %s' % (ii + 1, len(infiles), infiles[ii]))
reader = pyfits.open(infiles[ii])
distance_modulus_array_current = numpy.array(
reader[distance_modulus_extension].data.field(
distance_modulus_field),
copy=True)
if not numpy.array_equal(distance_modulus_array_current,
distance_modulus_array):
logger.warning("Distance moduli do not match; skipping...")
continue
reader.close()
pix_array_current = readSparseHealpixMap(infiles[ii],
pix_field,
extension=pix_data_extension,
construct_map=False)[0]
pix_array.append(pix_array_current)
for key in data_dict.keys():
value_array_current = readSparseHealpixMap(
infiles[ii],
key,
extension=pix_data_extension,
construct_map=False)[1]
data_dict[key].append(value_array_current)
#if distance_modulus_array is None:
# data_dict[key][pix_array_current] = value
#else:
# for jj in range(0, len(distance_modulus_array)):
# data_dict[key][jj] = value[jj]
gc.collect()
pix_master = numpy.concatenate(pix_array)
n_conflicting_pixels = len(pix_master) - len(numpy.unique(pix_master))
if n_conflicting_pixels != 0:
logger.warning('%i conflicting pixels during merge.' %
(n_conflicting_pixels))
for key in data_dict.keys():
if distance_modulus_array is not None:
data_dict[key] = numpy.concatenate(data_dict[key],
axis=1).transpose()
else:
data_dict[key] = numpy.concatenate(data_dict[key])
if outfile is not None:
writeSparseHealpixMap(pix_master,
data_dict,
nside,
outfile,
distance_modulus_array=distance_modulus_array,
coordsys='NULL',
ordering='NULL')
else:
return data_dict
def load_file(kwargs):
""" Load a FITS file with kwargs. """
logger.debug("Loading %s..." % kwargs['filename'])
return fitsio.read(**kwargs)
def pixelize(infiles,
outdir='hpx',
outbase=HPXBASE,
nside=16,
gzip=False,
force=False,
float32=False):
"""
Break catalog up into a set of healpix files.
"""
mkdir(outdir)
outfiles = glob.glob(outdir + '/*.fits')
if len(outfiles) and not force:
msg = "Found files: %s" % glob.glob(outdir + '/*.fits')
raise Exception(msg)
#if len(outfiles):
# print("Removing existing files...")
# map(os.remove,outfiles)
for ii, infile in enumerate(infiles):
logger.info('(%i/%i) %s' % (ii + 1, len(infiles), infile))
data = readfile(infile, float32)
if data is None: continue
catalog_pix = ang2pix(nside, data['RA'], data['DEC'])
#### Add object pixel (hack to get correct byte order)
#object_pix = ang2pix(NSIDE_OBJ,data['RA'],data['DEC'],nest=True)
#name = 'HPX%i'%NSIDE_OBJ; dtype = '>i4'
#data = recfuncs.rec_append_fields(data,name,object_pix,dtype)
for pix in np.unique(catalog_pix):
logger.debug("Processing pixel %s" % pix)
if 'BAND' in data.dtype.names:
band = np.unique(data['BAND'])
if len(band) != 1 and not force:
msg = "Found bands: %s" % band
raise Exception(msg)
band = ','.join([b.strip() for b in band])
else:
band = 'None'
outfile = os.path.join(outdir, outbase % pix)
arr = data[catalog_pix == pix]
if not os.path.exists(outfile):
logger.debug("Creating %s" % outfile)
out = fitsio.FITS(outfile, mode='rw')
out.write(arr)
out[1].write_key('COORDSYS',
'CEL',
comment='Coordinate system')
out[1].write_key('ORDERING',
'RING',
comment='HEALPix ordering scheme')
out[1].write_key('NSIDE', nside, comment='HEALPix nside')
out[1].write_key('HPX', pix, comment='HEALPix pixel (RING)')
out[1].write_key('BAND', band, comment='Photometric band')
else:
out = fitsio.FITS(outfile, mode='rw')
out[1].append(arr)
logger.debug("Writing %s" % outfile)
out.close()
