def abundanceMatchSnapshot(proxy,
scatter,
lf,
box_size,
minmag=-25.,
maxmag=10.,
debug=False,
figname=None):
af = AbundanceFunction(lf['mag'],
lf['phi'],
ext_range=(minmag, maxmag),
nbin=2000,
faint_end_fit_points=6)
# check the abundance function
if debug:
plt.clf()
plt.semilogy(lf['mag'], lf['phi'])
x = np.linspace(minmag, maxmag, 101)
plt.semilogy(x, af(x))
plt.savefig('abundance_fcn.png')
# deconvolution and check results (it's a good idea to always check this)
remainder = af.deconvolute(scatter * LF_SCATTER_MULT, 40)
x, nd = af.get_number_density_table()
if debug:
plt.clf()
plt.semilogy(x, np.abs(remainder / nd))
plt.savefig('nd_remainder.png')
# get number densities of the halo catalog
nd_halos = calc_number_densities(proxy, box_size)
# do abundance matching with some scatter
catalog_sc = af.match(nd_halos, scatter * LF_SCATTER_MULT)
if debug:
plt.clf()
c, e = np.histogram(catalog_sc[~np.isnan(catalog_sc)],
bins=np.linspace(minmag, maxmag, 101))
c = c / box_size**3 / (e[1:] - e[:-1])
me = (e[:-1] + e[1:]) / 2
plt.semilogy(me, c)
plt.semilogy(me, af(me))
plt.savefig('lf_in_v_out.png')
return catalog_sc
def deconvolve_scatter(self, scatter):
"""
"""
# tabulate stellar mass function
if self.gal_log_prop is True:
msample = np.logspace(self.gal_min_mass, self.gal_max_mass, self.gal_nbins)
nsample = self._stellar_mass_function(msample)
self.af = AbundanceFunction(np.log10(msample), nsample, faint_end_first=True)
else:
msample = np.linspace(self.gal_min_mass, self.gal_max_mass, self.gal_nbins)
nsample = self._stellar_mass_function(msample)
self.af = AbundanceFunction(msample, nsample, faint_end_first=True)
remainder = self.af.deconvolute(scatter, 100)
return remainder
def comp_deconv_steps(lf, scatters, deconv_repeats, m_max=-25):
""" Generate the projected 2D correlation by abundance matching galaxies
Parameters:
lf: The luminosity function. The first column is the magnitudes and the
second column is the density in units of 1/Mpc^3.
scatters: The scatters to deconvolve / re-introduce in the am (must
be a list)
Returns:
w_p(r_p) at the r_p values specified by r_p_data.
"""
# Initialize abundance function and calculate the number density of the
# halos in the box
af = AbundanceFunction(lf[:, 0], lf[:, 1], (-25, -5))
f, ax = plt.subplots(len(scatters),
1,
sharex='col',
sharey='row',
figsize=(13, 16))
ax[-1].set_xlabel('Magnitude (M - 5 log h)')
x, nd = af.get_number_density_table()
# For each scatter and each number of deconvolution steps, plot the
# dependence of the remainder on the step.
for s_i in range(len(scatters)):
scatter = scatters[s_i]
y_max = 0
legend = []
for deconv_repeat in deconv_repeats:
remainder = af.deconvolute(scatter * LF_SCATTER_MULT,
deconv_repeat) / nd
ax[s_i].plot(x,
remainder,
lw=3,
c=custom_blues_complement[len(legend)])
y_max = max(y_max, np.max(remainder[x > m_max]))
legend.append('Deconvolution Steps = %d' % (deconv_repeat))
ax[s_i].set_ylabel('(LF (deconv $\Rightarrow$ conv) - LF) / LF')
ax[s_i].set_xlim([np.max(lf[:, 0]) - 2, m_max])
ax[s_i].set_ylim([-1.2, y_max * 1.2])
ax[s_i].set_title('Luminosity Function Remainder %.2f Scatter' %
(scatter))
ax[0].legend(legend)
def __getAbundanceFunc(self, MFobj):
"""
Returns the abundance matching function object.
Takes considers only part of MF above log10(6.8)
"""
IDS = np.where(MFobj[:, 0] > 6.8)
# Abundances are not logged, whereas masses are logged
af = AbundanceFunction(MFobj[:, 0][IDS], MFobj[:, 1][IDS], (5, 13), faint_end_first=True)
return af
class DeconvolveSHAM(object):
"""
class to assign stellar mass, or some other primary galaxy property,
based on the de-convolution SHAM method, based on Yau-Yun Mao's
implementation of Peter Behroozi's deconvolution package.
"""
def __init__(self,
stellar_mass_function,
prim_haloprop='halo_mpeak',
prim_galprop='stellar_mass',
scatter=0.0,
gal_log_prop=True,
gal_min_mass=2.0,
gal_max_mass=12.5,
gal_nbins=200,
**kwargs):
"""
Parameters
----------
stellar_mass_function : callable object
prim_haloprop : string, optional
key indicating halo property used to assign the primary
galaxy property
prim_galprop : string, optional
key indicating primary galaxy property
gal_log_prop : boolean, optional
boolean indicating whther the scatter in the primary
galaxy property is modelled as a log-normal.
gal_min_mass, gal_max_mass, gal_nbins: float, float, int
parameters used to bin and span galaxy abundance function.
If gal_log_prop is True, these should also be logged,
and the spacing will be even in log.
redshift : float
redshit for which the model is applicable
"""
if 'redshift' in list(kwargs.keys()):
self.redshift = kwargs['redshift']
else:
self.redshift = 0.0
self.prim_haloprop = prim_haloprop
self.prim_galprop = prim_galprop
self._galprop_dtypes_to_allocate = np.dtype([(str(self.prim_galprop), 'f4')])
self._mock_generation_calling_sequence = ['inherit_halocat_properties', 'assign_stellar_mass']
self.list_of_haloprops_needed = [prim_haloprop]
# set default box size.
if 'Lbox' in kwargs.keys():
self._Lbox = kwargs['Lbox']
else:
self._Lbox = np.inf
# update Lbox if a halo catalog object is passed.
self._additional_kwargs_dict = dict(inherit_halocat_properties=['Lbox'])
# store model parametrs
self.gal_log_prop = gal_log_prop
self.param_dict = ({'scatter': scatter})
self.gal_min_mass = gal_min_mass
self.gal_max_mass = gal_max_mass
self.gal_nbins = gal_nbins
# deconvolve abundance function
self._stellar_mass_function = stellar_mass_function
self.deconvolve_scatter(self.param_dict['scatter'])
def inherit_halocat_properties(self, seed=None, **kwargs):
"""
Inherit the box size during mock population
"""
try:
Lbox = kwargs['Lbox']
self._Lbox = Lbox
except KeyError:
print("Error automatically detecting Lbox.")
def deconvolve_scatter(self, scatter):
"""
"""
# tabulate stellar mass function
if self.gal_log_prop is True:
msample = np.logspace(self.gal_min_mass, self.gal_max_mass, self.gal_nbins)
nsample = self._stellar_mass_function(msample)
self.af = AbundanceFunction(np.log10(msample), nsample, faint_end_first=True)
else:
msample = np.linspace(self.gal_min_mass, self.gal_max_mass, self.gal_nbins)
nsample = self._stellar_mass_function(msample)
self.af = AbundanceFunction(msample, nsample, faint_end_first=True)
remainder = self.af.deconvolute(scatter, 100)
return remainder
def assign_stellar_mass(self, **kwargs):
"""
assign stellar mass
"""
if 'table' in kwargs.keys():
table = kwargs['table']
try:
Lbox = kwargs['Lbox']
except KeyError:
Lbox = self._Lbox
else:
try:
Lbox = kwargs['Lbox']
except KeyError:
Lbox = self._Lbox
Lbox = np.atleast_1d(Lbox)
if len(Lbox) == 3:
Lbox = (np.prod(Lbox))**(1.0/3.0)
else:
Lbox = Lbox[0]
nd_halos = calc_number_densities(table[self.prim_haloprop], Lbox)
mstar = self.af.match(nd_halos, self.param_dict['scatter'])
if self.gal_log_prop is True:
mstar = 10.0**mstar
table[self.prim_galprop] = mstar
else:
table[self.prim_galprop] = mstar
if 'table' in kwargs.keys():
return table
else:
return mstar
def __init__(self,
lf_list,
halos,
af_criteria,
box_size,
r_p_data,
mag_cuts,
wp_data_list,
wp_cov_list,
pimax,
nthreads,
deconv_repeat,
wp_save_path,
n_k_tree_cut=None):
""" Initialize AMLikelihood object. This involves initializing an
AbundanceFunction object for each luminosity function.
Parameters:
lf_list: List of luminosity functions
halos: Halos dictionairy. halos[af_criteria] should return
a numpy array of values for the abundance matching
criteria for the halos.
af_criteria: A string that will be used to index into halos
for the abundance matching criteria data.
box_size: The size of the box being used (length)
r_p_data: The positions at which to calculate the 2D correlation
function.
mag_cuts: The magnitude cuts for w_p(r_p) (must be a list)
wp_data_list: The list of wp_data corresponding to lf_list
to be used for the likelihood function
wp_cov_list: The list of covariance matrices for w_p. Also
important for the covariance function.
pimax: The maximum redshift seperation to use in w_p(r_p)
calculation
nthreads: The number of threads to use for CorrFunc
deconv_repeat: The number of deconvolution steps to conduct
wp_save_path: A unique path to which to save the parameter
values and 2d projected correlation functions.
n_k_tree_cut: An integer for the number of halos to cut from
the catalog in the k nearest neighbor step. This will reduce
accuracy at small scales but speed up computation. If set
to None this step will not be done.
Returns:
Initialized class
"""
# Save dictionairy parameter along with box size object
# pre-sort halos to speed up computations
self.halos = halos[np.argsort(halos[af_criteria])]
self.af_criteria = af_criteria
self.box_size = box_size
self.r_p_data = r_p_data
self.mag_cuts = mag_cuts
self.wp_data_list = wp_data_list
self.wp_cov_list = wp_cov_list
self.pimax = pimax
self.nthreads = nthreads
self.deconv_repeat = deconv_repeat
# Generate list of abundance matching functions
self.af_list = []
for lf in lf_list:
af = AbundanceFunction(lf[:, 0], lf[:, 1], (-25, -5))
self.af_list.append(af)
# Generate rbins so that the average falls at r_p_data
rbins = np.zeros(len(r_p_data) + 1)
rbins[1:-1] = 0.5 * (r_p_data[:-1] + r_p_data[1:])
rbins[0] = 2 * r_p_data[0] - rbins[1]
rbins[-1] = 2 * r_p_data[-1] - rbins[-2]
self.rbins = rbins
self.wp_save_path = wp_save_path
# K nearest neighbors calculation for the cut. This only needs
# to be done once since the cut is not dependent on the AM
# parameters.
# This code does not work! I need to reimplement this with weighting
# and a mapping from original halos to the reduced catalog.
# if n_k_tree_cut is not None:
# neigh_pos = np.transpose(np.vstack([
# self.halos['px'],self.halos['py']]))
# # Epsilon in case some galaxies are cataloges as being at the edge
# # of the box.
# epsilon = 1e-12
# # Set up the tree
# tree = cKDTree(neigh_pos,boxsize=box_size+epsilon)
# # Query the 2nd nearest neighbor.
# dist, locs = tree.query(neigh_pos,k=2)
# keep = np.argsort(dist[:,1])[n_k_tree_cut:]
# # A bool array to use for indexing
# self.wp_keep = np.zeros(len(halos),dtype=bool)
# self.wp_keep[keep] = True
# else:
self.wp_keep = None
self.ll_dict = {}
def generate_wp(lf_list,
halos,
af_criteria,
r_p_data,
box_size,
mag_cuts,
pimax=40.0,
nthreads=1,
scatters=None,
deconv_repeat=20,
verbose=False):
""" Generate the projected 2D correlation by abundance matching galaxies
Parameters:
lf_list: A list of luminosity functions for each mag_cut. The first
column is the magnitudes and thesecond column is the density in
units of 1/Mpc^3.
halos: A catalog of the halos in the n-body sim that can be indexed
into using the quantity name.
af_criteria: The galaxy property (i.e. vpeak) to use for abundance
matching.
r_p_data: The positions at which to calculate the 2D correlation
function.
box_size: The size of the box (box length not volume)
mag_cuts: The magnitude cuts for w_p(r_p) (must be a list)
pimax: The maximum redshift seperation to use in w_p(r_p) calculation
nthreads: The number of threads to use for CorrFunc
scatters: The scatters to deconvolve / re-introduce in the am (must
be a list)
deconv_repeat: The number of deconvolution steps to conduct
verbose: If set to true, will generate plots for visual inspection
of am outputs.
Returns:
w_p(r_p) at the r_p values specified by r_p_data.
"""
# Repeat once for each magnitude cut
wp_binneds = []
for mag_cut_i in range(len(mag_cuts)):
mag_cut = mag_cuts[mag_cut_i]
lf = lf_list[mag_cut_i]
# Initialize abundance function and calculate the number density of the
# halos in the box
af = AbundanceFunction(lf[:, 0], lf[:, 1], (-25, -5))
nd_halos = calc_number_densities(halos[af_criteria], box_size)
if scatters is not None:
remainders = []
for scatter in scatters:
remainders.append(
af.deconvolute(scatter * LF_SCATTER_MULT, deconv_repeat))
# If verbose output the match between abundance function and input data
if verbose:
matplotlib.rcParams.update({'font.size': 18})
plt.figure(figsize=(10, 8))
plt.plot(lf[:, 0], lf[:, 1], lw=7, c=custom_blues[1])
x = np.linspace(np.min(lf[:, 0]) - 2, np.max(lf[:, 0]) + 2, 101)
plt.semilogy(x, af(x), lw=3, c=custom_blues[4])
plt.xlim([np.max(lf[:, 0]) + 2, np.min(lf[:, 0])])
plt.ylim([1e-5, 1])
plt.xlabel('Magnitude (M - 5 log h)')
plt.ylabel('Number Density (1/ (Mpc^3 h))')
plt.legend(['Input', 'Fit'])
plt.title('Luminosity Function')
plt.yscale('log')
plt.show()
# Plot remainder to ensure the deconvolution returned reasonable results
if verbose and scatters is not None:
f, ax = plt.subplots(2,
1,
sharex='col',
sharey='row',
figsize=(15, 12),
gridspec_kw={'height_ratios': [2, 1]})
x, nd = af.get_number_density_table()
ax[0].plot(x, nd, lw=3, c=custom_blues[4])
legend = []
for scatter in scatters:
ax[0].plot(af._x_deconv[float(scatter * LF_SCATTER_MULT)],
nd,
lw=3,
c=custom_blues_complement[2 * len(legend)])
legend.append('Scatter = %.2f' % (scatter))
ax[0].set_xlim([np.max(lf[:, 0]) + 2, np.min(lf[:, 0])])
ax[0].set_ylim([1e-5, 1])
ax[0].set_ylabel('Number Density (1/ (Mpc^3 h))')
ax[0].legend(['Fit'] + legend)
ax[0].set_title('Deconvolved Luminosity Function')
ax[0].set_yscale('log')
ax[1].set_xlabel('Magnitude (M - 5 log h)')
ax[1].set_ylabel('(LF (deconv $\Rightarrow$ conv) - LF) / LF')
ax[1].set_xlim([np.max(lf[:, 0]) + 2, np.min(lf[:, 0])])
y_max = 0
for r_i in range(len(remainders)):
remainder = remainders[r_i] / nd
ax[1].plot(x,
remainder,
lw=3,
c=custom_blues_complement[2 * r_i])
y_max = max(y_max, np.max(remainder[x > np.min(lf[:, 0])]))
ax[1].set_ylim([-1.2, y_max * 1.2])
plt.show()
# Conduct the abundance matching
catalogs = []
if scatters is not None:
for scatter in scatters:
catalogs.append(
af.match(nd_halos,
scatter * LF_SCATTER_MULT,
do_rematch=False))
else:
catalogs = [af.match(nd_halos)]
wp_scatts = []
for catalog in catalogs:
# A luminosity cutoff to use for the correlation function.
sub_catalog = catalog < mag_cut
print('Scatter %.2f catalog has %d galaxies' %
(scatters[len(wp_scatts)], np.sum(sub_catalog)))
x = halos['px'][sub_catalog]
y = halos['py'][sub_catalog]
z = halos['pz'][sub_catalog]
# Generate rbins so that the average falls at r_p_data
rbins = np.zeros(len(r_p_data) + 1)
rbins[1:-1] = 0.5 * (r_p_data[:-1] + r_p_data[1:])
rbins[0] = 2 * r_p_data[0] - rbins[1]
rbins[-1] = 2 * r_p_data[-1] - rbins[-2]
# Calculate the projected correlation function
wp_results = wp(box_size,
pimax,
nthreads,
rbins,
x,
y,
z,
verbose=False,
output_rpavg=True)
# Extract the results
wp_binned = np.zeros(len(wp_results))
for i in range(len(wp_results)):
wp_binned[i] = wp_results[i][3]
wp_scatts.append(wp_binned)
wp_binneds.append(wp_scatts)
return wp_binneds
import os
from urllib import urlopen, urlretrieve
import numpy as np
from AbundanceMatching import AbundanceFunction, LF_SCATTER_MULT
test_file = urlretrieve('http://slac.stanford.edu/~yymao/tmp/am_deconv_test.npz')[0]
tests = np.load(test_file)
M, phi_log = np.loadtxt(urlopen('http://arxiv.org/src/1304.7778v2/anc/LF_SerExp.dat'), usecols=(0,1)).T
phi = (10.**phi_log)*0.4/(0.7**3)
af = AbundanceFunction(M, phi, ext_range=(-30.0, -5.0))
for scatter in (0.05, 0.1, 0.15, 0.2, 0.25):
scatter *= LF_SCATTER_MULT
af.deconvolute(scatter)
assert np.allclose(af._x_deconv[scatter], tests['{0:g}'.format(scatter)], rtol=0.001)
# assert (af._x_deconv[scatter] == tests['{0:g}'.format(scatter)]).all()
# diff = (af._x_deconv[scatter] == tests['{0:g}'.format(scatter)])
# for val in diff:
# if val != 0:
# print(val)
# ineq = np.where(diff != 0)[0]
# print(len(diff[ineq]), len(diff))
os.remove(test_file)
moster13_halocat_keys = ('halo_upid', 'halo_mpeak', 'halo_scale_factor_mpeak',
'halo_x', 'halo_y', 'halo_z', 'halo_zpeak', 'halo_vmax_mpeak', 'halo_mvir',
'halo_scale_factor_firstacc', 'halo_mvir_firstacc', 'halo_halfmass_scale_factor',
'halo_vx', 'halo_vy', 'halo_vz', 'halo_rvir_zpeak', 'halo_vmax_at_mpeak_percentile',
'halo_mvir_host_halo', 'halo_spin', 'halo_uran')
smf_dirname = "/Users/aphearin/work/UniverseMachine/code/UniverseMachine/obs"
smf_basename = "moustakas_z0.01_z0.20.smf"
smf_fname = os.path.join(smf_dirname, smf_basename)
smf = np.loadtxt(smf_fname)
log10_sm_table_h0p7 = 0.5*(smf[:, 0] + smf[:, 1])
dn_dlog10_sm_h0p7 = smf[:, 2]
sham_ext_range = (8, 12.75)
moustakas_af = AbundanceFunction(log10_sm_table_h0p7, dn_dlog10_sm_h0p7,
sham_ext_range, faint_end_first=True)
def load_baseline_halocat(simname='bolplanck', redshift=0, fixed_seed=411):
halocat = CachedHaloCatalog(simname=simname, redshift=redshift)
halocat.halo_table['halo_zpeak'] = 1./halocat.halo_table['halo_scale_factor_mpeak'] - 1.
rvir_peak_physical_unity_h = halo_mass_to_halo_radius(halocat.halo_table['halo_mpeak'],
halocat.cosmology, halocat.halo_table['halo_zpeak'], 'vir')
rvir_peak_physical = rvir_peak_physical_unity_h/halocat.cosmology.h
halocat.halo_table['halo_rvir_zpeak'] = rvir_peak_physical*1000.
nhalos = len(halocat.halo_table)
with NumpyRNGContext(fixed_seed):
halocat.halo_table['halo_uran'] = np.random.rand(nhalos)
def test_deconv():
af = AbundanceFunction(*_lf.T, ext_range=(-25, -16))
af.deconvolute(0.2 * LF_SCATTER_MULT, 20)
def test_AbundanceFunction():
AbundanceFunction(*_lf.T, ext_range=(-25, -16))