def calc_lnoverlaps(group_pars, star_pars, nstars):
"""Find the lnoverlaps given the parameters of a group
Parameters
----------
group_pars : [npars] array
Group parameters (internal encoding, 1/dX... no nstars)
star_pars : dict
stars: (nstars) high astropy table including columns as
documented in the Traceback class.
times : [ntimes] numpy array
times that have been traced back, in Myr
xyzuvw : [nstars, ntimes, 6] array
XYZ in pc and UVW in km/s
xyzuvw_cov : [nstars, ntimes, 6, 6] array
covariance of xyzuvw
nstars : int
number of stars in traceback
Returns
-------
lnols : [nstars] array
The log of the overlap of each star with the provided group
"""
group_mn = group_pars[0:6]
group_cov = utils.generate_cov(group_pars)
assert np.min(np.linalg.eigvalsh(group_cov)) >= 0
# interpolate star data to modelled age
age = group_pars[13]
interp_covs, interp_mns = gf.interp_cov(age, star_pars)
lnols = ol.get_lnoverlaps(group_cov, group_mn, interp_covs, interp_mns,
nstars)
return lnols
def calc_lnoverlaps(group_pars, star_pars, nstars):
"""Find the lnoverlaps given the parameters of a group
Parameters
----------
group_pars : [npars] array
Group parameters (internal encoding, 1/dX... no nstars)
star_pars : dict
stars: (nstars) high astropy table including columns as
documented in the Traceback class.
times : [ntimes] numpy array
times that have been traced back, in Myr
xyzuvw : [nstars, ntimes, 6] array
XYZ in pc and UVW in km/s
xyzuvw_cov : [nstars, ntimes, 6, 6] array
covariance of xyzuvw
nstars : int
number of stars in traceback
Returns
-------
lnols : [nstars] array
The log of the overlap of each star with the provided group
"""
group_mn = group_pars[0:6]
group_cov = utils.generate_cov(group_pars)
assert np.min( np.linalg.eigvalsh(group_cov) ) >= 0
# interpolate star data to modelled age
age = group_pars[13]
interp_covs, interp_mns = gf.interp_cov(age, star_pars)
lnols = ol.get_lnoverlaps(
group_cov, group_mn, interp_covs, interp_mns, nstars
)
return lnols
def test_pythonFuncs():
xyzuvw_file = "../data/fed_stars_20_xyzuvw.fits"
xyzuvw_dict = gf.loadXYZUVW(xyzuvw_file)
star_means = xyzuvw_dict['xyzuvw']
star_covs = xyzuvw_dict['xyzuvw_cov']
nstars = star_means.shape[0]
group_mean = np.mean(star_means, axis=0)
group_cov = np.cov(star_means.T)
co1s = []
co2s = []
for i, (scov, smn) in enumerate(zip(star_covs, star_means)):
print(i)
co1s.append(co1(group_cov, group_mean, scov, smn))
co2s.append(co2(group_cov, group_mean, scov, smn))
ol.get_lnoverlaps(group_cov, group_mean,
np.array([scov]),
np.array([smn]), 1)
co1s = np.array(co1s)
co2s = np.array(co2s)
co3s = np.exp(sclno(group_cov, group_mean, star_covs, star_means, nstars))
assert np.allclose(co1s, co2s)
assert np.allclose(co2s, co3s)
assert np.allclose(co1s, co3s)
# note that most overlaps go to 0, but the log overlaps retains the
# information
co1s = []
co2s = []
for i, (scov, smn) in enumerate(zip(star_covs, star_means)):
co1s.append(co1(star_covs[15], star_means[15], scov, smn))
co2s.append(co2(star_covs[15], star_means[15], scov, smn))
co1s = np.array(co1s)
co2s = np.array(co2s)
lnos = sclno(star_covs[15], star_means[15], star_covs, star_means, 1)
co3s = np.exp(lnos)
assert np.allclose(co1s, co2s)
assert np.allclose(co2s, co3s)
assert np.allclose(co1s, co3s)
def correctness(group_icov, group_mean, group_icov_det, star_icovs,
star_means, star_icov_dets, nstars):
"""
Displays the result for each function, no differences should
occur.
"""
# Using swig-numpy module with multiple stars per call
swig_np_ms_ols = overlap.get_overlaps(
group_icov, group_mean, group_icov_det, star_icovs,
star_means, star_icov_dets, nstars)
# Using swig-numpy module with multiple stars per call and log results
group_cov = np.linalg.inv(group_icov)
star_covs = np.zeros(star_icovs.shape)
for i in range(nstars):
star_covs[i] = np.linalg.inv(star_icovs[i])
swig_np_ms_lnols = overlap.get_lnoverlaps(
group_cov, group_mean, star_covs,
star_means, nstars
)
# Compare with various implementations of calculating one star at a time
for i in range(nstars):
# Using numpy
numpy_ols = compute_overlap(
group_icov, group_mean, group_icov_det,
star_icovs[i], star_means[i], star_icov_dets[i])
# Using swig module (input passed as python list)
swig_ols = overlap.get_overlap2(
group_icov.flatten().tolist(), group_mean.flatten().tolist(),
group_icov_det, star_icovs[i].flatten().tolist(),
star_means[i].flatten().tolist(), star_icov_dets[i])
# Using swig module (input passed as numpy arrays)
swig_np_ols = overlap.get_overlap(
group_icov, group_mean, group_icov_det, star_icovs[i],
star_means[i], star_icov_dets[i])
# # Using swig module (input passed as numpy arrays, output in log)
# swig_np_lnol = overlap.get_lnoverlap(
# group_icov, group_mean, group_icov_det, star_icovs[i],
# star_means[i], star_icov_dets[i])
assert np.isclose(numpy_ols, swig_np_ms_ols[i], rtol=1e-8)
assert np.isclose(numpy_ols, swig_ols, rtol=1e-8)
assert np.isclose(numpy_ols, swig_np_ols, rtol=1e-8)
assert np.isclose(numpy_ols, np.exp(swig_np_ms_lnols[i]), rtol=1e-8)
print("All implementations return same result to 8 sigfigs")
def correctness(group_icov, group_mean, group_icov_det, star_icovs, star_means,
star_icov_dets, nstars):
"""
Displays the result for each function, no differences should
occur.
"""
# Using swig-numpy module with multiple stars per call
swig_np_ms_ols = overlap.get_overlaps(group_icov, group_mean,
group_icov_det, star_icovs,
star_means, star_icov_dets, nstars)
# Using swig-numpy module with multiple stars per call and log results
group_cov = np.linalg.inv(group_icov)
star_covs = np.zeros(star_icovs.shape)
for i in range(nstars):
star_covs[i] = np.linalg.inv(star_icovs[i])
swig_np_ms_lnols = overlap.get_lnoverlaps(group_cov, group_mean, star_covs,
star_means, nstars)
# Compare with various implementations of calculating one star at a time
for i in range(nstars):
# Using numpy
numpy_ols = compute_overlap(group_icov, group_mean, group_icov_det,
star_icovs[i], star_means[i],
star_icov_dets[i])
# Using swig module (input passed as python list)
swig_ols = overlap.get_overlap2(group_icov.flatten().tolist(),
group_mean.flatten().tolist(),
group_icov_det,
star_icovs[i].flatten().tolist(),
star_means[i].flatten().tolist(),
star_icov_dets[i])
# Using swig module (input passed as numpy arrays)
swig_np_ols = overlap.get_overlap(group_icov, group_mean,
group_icov_det, star_icovs[i],
star_means[i], star_icov_dets[i])
# # Using swig module (input passed as numpy arrays, output in log)
# swig_np_lnol = overlap.get_lnoverlap(
# group_icov, group_mean, group_icov_det, star_icovs[i],
# star_means[i], star_icov_dets[i])
assert np.isclose(numpy_ols, swig_np_ms_ols[i], rtol=1e-8)
assert np.isclose(numpy_ols, swig_ols, rtol=1e-8)
assert np.isclose(numpy_ols, swig_np_ols, rtol=1e-8)
assert np.isclose(numpy_ols, np.exp(swig_np_ms_lnols[i]), rtol=1e-8)
print("All implementations return same result to 8 sigfigs")
dt.mvGauss(later_groups[i].mean, later_groups[i].mean,
later_groups[i].generateSphericalCovMatrix(),
later_z[:, i].sum())))
print("bg dens during run for reference:")
print(np.exp(em.backgroundLogOverlap(init_groups[0].mean, hists_from_run)))
# --------------------------------------------------
# ----- an aside, does a delta function at ---------
# ----- group mean return same as peak val ---------
# --------------------------------------------------
# spoiler... yes, yes it does
lg0 = later_groups[0]
delta_ol_at_mean =\
np.exp(get_lnoverlaps(lg0.generateSphericalCovMatrix(), lg0.mean,
np.array([1e-10*np.eye(6)]), np.array([lg0.mean]),
1))[0]
peak_dens = dt.mvGauss(lg0.mean, lg0.mean, lg0.generateSphericalCovMatrix())
assert np.isclose(delta_ol_at_mean, peak_dens)
# demo synthesising some bg stars
my_bg_dens = 1e-7
spread = np.std(gaia_xyzuvw, axis=0) / 8.
ubound = bpmg_mean + 0.5 * spread
lbound = bpmg_mean - 0.5 * spread
nbg_stars = int(my_bg_dens * np.prod(spread))
print("There are {} stars in bg".format(nbg_stars))
my_bg_stars = np.random.uniform(-1, 1,
size=(nbg_stars, 6)) * spread + bpmg_mean
def timings(group_icov, group_mean, group_icov_det,
star_icovs, star_means, star_icov_dets, batch_size, noverlaps=10000):
"""
Executes each function a fixed number of times, timing for how
long it takes.
"""
if (noverlaps <= 100000):
npstart = time.clock()
for i in range(noverlaps):
result = compute_overlap(group_icov, group_mean, group_icov_det,
star_icovs[0], star_means[0],
star_icov_dets[0])
print "Numpy: " + str(time.clock() - npstart)
else:
print "Numpy: practically infinity seconds"
print " -> (approximately 5x 'Swig')"
swigstart = time.clock()
for i in range(noverlaps):
result = overlap.get_overlap2(group_icov.flatten().tolist(),
group_mean.flatten().tolist(),
group_icov_det,
star_icovs[0].flatten().tolist(),
star_means[0].flatten().tolist(),
star_icov_dets[0])
print "Swig: {} s".format(time.clock() - swigstart)
swignpstart = time.clock()
for i in range(noverlaps):
result = overlap.get_overlap(group_icov, group_mean, group_icov_det,
star_icovs[0], star_means[0], star_icov_dets[0])
end = time.clock()
print "Swigging numpy: {} s".format(end - swignpstart)
swignpmultistart = time.clock()
for i in range(noverlaps/batch_size):
result = overlap.get_overlaps(
group_icov, group_mean, group_icov_det,
star_icovs, star_means, star_icov_dets, batch_size)
end = time.clock()
print("Swigging numpy multi: {} s".format(end - swignpmultistart))
print(" -> total module calls: {}".format(noverlaps/batch_size))
print(" -> {} microsec per overlap".\
format((end - swignpmultistart)/noverlaps*1e6))
print(" -> {} stars per module call".format(batch_size))
group_cov = np.linalg.inv(group_icov)
star_covs = np.linalg.inv(star_icovs)
newswignpmultistart = time.clock()
for i in range(noverlaps/batch_size):
result = overlap.get_lnoverlaps(group_cov, group_mean,
star_covs, star_means, batch_size)
end = time.clock()
print("Swigging numpy multi logged: {} s".format(end - newswignpmultistart))
print(" -> total module calls: {}".format(noverlaps/batch_size))
print(" -> {} microsec per overlap".\
format((end - newswignpmultistart)/noverlaps*1e6))
print(" -> {} stars per module call".format(batch_size))
# BROADCAST CONSTANTS
nstars = comm.bcast(nstars, root=0)
background_means = comm.bcast(background_means, root=0)
background_covs = comm.bcast(background_covs, root=0)
# SCATTER DATA
star_means = comm.scatter(star_means, root=0)
star_covs = comm.scatter(star_covs, root=0)
#print(rank, len(star_means))
# EVERY PROCESS DOES THIS FOR ITS DATA
bg_ln_ols = []
for star_cov, star_mean in zip(star_covs, star_means):
try:
bg_lnol = get_lnoverlaps(star_cov, star_mean, background_covs,
background_means, nstars)
bg_lnol = logsumexp(bg_lnol) # sum in linear space
except:
# TC: Changed sign to negative (surely if it fails, we want it to
# have a neglible background overlap?
print('bg ln overlap failed, setting it to -inf')
bg_lnol = -np.inf
bg_ln_ols.append(bg_lnol)
#print(rank, bg_ln_ols)
# GATHER DATA
bg_ln_ols_result = comm.gather(bg_ln_ols, root=0)
if rank == 0:
bg_ln_ols_result = list(itertools.chain.from_iterable(bg_ln_ols_result))
np.savetxt('bgols_multiprocessing_%d.dat' % NI, bg_ln_ols_result)
def timings(group_icov,
group_mean,
group_icov_det,
star_icovs,
star_means,
star_icov_dets,
batch_size,
noverlaps=10000):
"""
Executes each function a fixed number of times, timing for how
long it takes.
"""
if (noverlaps <= 100000):
npstart = time.clock()
for i in range(noverlaps):
result = compute_overlap(group_icov, group_mean, group_icov_det,
star_icovs[0], star_means[0],
star_icov_dets[0])
print "Numpy: " + str(time.clock() - npstart)
else:
print "Numpy: practically infinity seconds"
print " -> (approximately 5x 'Swig')"
swigstart = time.clock()
for i in range(noverlaps):
result = overlap.get_overlap2(group_icov.flatten().tolist(),
group_mean.flatten().tolist(),
group_icov_det,
star_icovs[0].flatten().tolist(),
star_means[0].flatten().tolist(),
star_icov_dets[0])
print "Swig: {} s".format(time.clock() - swigstart)
swignpstart = time.clock()
for i in range(noverlaps):
result = overlap.get_overlap(group_icov, group_mean, group_icov_det,
star_icovs[0], star_means[0],
star_icov_dets[0])
end = time.clock()
print "Swigging numpy: {} s".format(end - swignpstart)
swignpmultistart = time.clock()
for i in range(noverlaps / batch_size):
result = overlap.get_overlaps(group_icov, group_mean, group_icov_det,
star_icovs, star_means, star_icov_dets,
batch_size)
end = time.clock()
print("Swigging numpy multi: {} s".format(end - swignpmultistart))
print(" -> total module calls: {}".format(noverlaps / batch_size))
print(" -> {} microsec per overlap".\
format((end - swignpmultistart)/noverlaps*1e6))
print(" -> {} stars per module call".format(batch_size))
group_cov = np.linalg.inv(group_icov)
star_covs = np.linalg.inv(star_icovs)
newswignpmultistart = time.clock()
for i in range(noverlaps / batch_size):
result = overlap.get_lnoverlaps(group_cov, group_mean, star_covs,
star_means, batch_size)
end = time.clock()
print("Swigging numpy multi logged: {} s".format(end -
newswignpmultistart))
print(" -> total module calls: {}".format(noverlaps / batch_size))
print(" -> {} microsec per overlap".\
format((end - newswignpmultistart)/noverlaps*1e6))
print(" -> {} stars per module call".format(batch_size))
xyzuvw_file = "../data/fed_stars_20_xyzuvw.fits"
xyzuvw_dict = gf.loadXYZUVW(xyzuvw_file)
star_means = xyzuvw_dict['xyzuvw']
star_covs = xyzuvw_dict['xyzuvw_cov']
nstars = star_means.shape[0]
gmn = np.mean(star_means, axis=0)
gcov = np.cov(star_means.T)
if DEBUG_SINGULAR:
scov = star_covs[20]
smn = star_means[20]
py_lnol = sclno(gcov, gmn, np.array([scov]), np.array([smn]), 1)
ol.get_lnoverlaps(gcov, gmn,
np.array([scov]),
np.array([smn]), 1)
if DEBUG_UNDERFLOW:
scov = star_covs[0]
smn = star_means[0]
#py_lnol = sclno(gcov, gmn, np.array([scov]), np.array([smn]), 1)
py_lnol = clno(gcov, gmn, scov, smn)
# gcovi = np.linalg.inv(gcov)
# scovi = np.linalg.inv(scov)
# gcovi_det = np.linalg.det(gcovi)
# scovi_det = np.linalg.det(scovi)
# c_ol1 = ol.get_overlap(gcovi, gmn, gcovi_det,
# scovi, smn, scovi_det)
# c_ol2 = ol.get_overlaps(gcovi, gmn, gcovi_det,
# np.array([scovi]), np.array([smn]),
