def calc_lnoverlaps(group_pars, star_pars, nstars): """Find the lnoverlaps given the parameters of a group Parameters ---------- group_pars : [npars] array Group parameters (internal encoding, 1/dX... no nstars) star_pars : dict stars: (nstars) high astropy table including columns as documented in the Traceback class. times : [ntimes] numpy array times that have been traced back, in Myr xyzuvw : [nstars, ntimes, 6] array XYZ in pc and UVW in km/s xyzuvw_cov : [nstars, ntimes, 6, 6] array covariance of xyzuvw nstars : int number of stars in traceback Returns ------- lnols : [nstars] array The log of the overlap of each star with the provided group """ group_mn = group_pars[0:6] group_cov = utils.generate_cov(group_pars) assert np.min(np.linalg.eigvalsh(group_cov)) >= 0 # interpolate star data to modelled age age = group_pars[13] interp_covs, interp_mns = gf.interp_cov(age, star_pars) lnols = ol.get_lnoverlaps(group_cov, group_mn, interp_covs, interp_mns, nstars) return lnols
def calc_lnoverlaps(group_pars, star_pars, nstars): """Find the lnoverlaps given the parameters of a group Parameters ---------- group_pars : [npars] array Group parameters (internal encoding, 1/dX... no nstars) star_pars : dict stars: (nstars) high astropy table including columns as documented in the Traceback class. times : [ntimes] numpy array times that have been traced back, in Myr xyzuvw : [nstars, ntimes, 6] array XYZ in pc and UVW in km/s xyzuvw_cov : [nstars, ntimes, 6, 6] array covariance of xyzuvw nstars : int number of stars in traceback Returns ------- lnols : [nstars] array The log of the overlap of each star with the provided group """ group_mn = group_pars[0:6] group_cov = utils.generate_cov(group_pars) assert np.min( np.linalg.eigvalsh(group_cov) ) >= 0 # interpolate star data to modelled age age = group_pars[13] interp_covs, interp_mns = gf.interp_cov(age, star_pars) lnols = ol.get_lnoverlaps( group_cov, group_mn, interp_covs, interp_mns, nstars ) return lnols
def test_pythonFuncs(): xyzuvw_file = "../data/fed_stars_20_xyzuvw.fits" xyzuvw_dict = gf.loadXYZUVW(xyzuvw_file) star_means = xyzuvw_dict['xyzuvw'] star_covs = xyzuvw_dict['xyzuvw_cov'] nstars = star_means.shape[0] group_mean = np.mean(star_means, axis=0) group_cov = np.cov(star_means.T) co1s = [] co2s = [] for i, (scov, smn) in enumerate(zip(star_covs, star_means)): print(i) co1s.append(co1(group_cov, group_mean, scov, smn)) co2s.append(co2(group_cov, group_mean, scov, smn)) ol.get_lnoverlaps(group_cov, group_mean, np.array([scov]), np.array([smn]), 1) co1s = np.array(co1s) co2s = np.array(co2s) co3s = np.exp(sclno(group_cov, group_mean, star_covs, star_means, nstars)) assert np.allclose(co1s, co2s) assert np.allclose(co2s, co3s) assert np.allclose(co1s, co3s) # note that most overlaps go to 0, but the log overlaps retains the # information co1s = [] co2s = [] for i, (scov, smn) in enumerate(zip(star_covs, star_means)): co1s.append(co1(star_covs[15], star_means[15], scov, smn)) co2s.append(co2(star_covs[15], star_means[15], scov, smn)) co1s = np.array(co1s) co2s = np.array(co2s) lnos = sclno(star_covs[15], star_means[15], star_covs, star_means, 1) co3s = np.exp(lnos) assert np.allclose(co1s, co2s) assert np.allclose(co2s, co3s) assert np.allclose(co1s, co3s)
def correctness(group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, nstars): """ Displays the result for each function, no differences should occur. """ # Using swig-numpy module with multiple stars per call swig_np_ms_ols = overlap.get_overlaps( group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, nstars) # Using swig-numpy module with multiple stars per call and log results group_cov = np.linalg.inv(group_icov) star_covs = np.zeros(star_icovs.shape) for i in range(nstars): star_covs[i] = np.linalg.inv(star_icovs[i]) swig_np_ms_lnols = overlap.get_lnoverlaps( group_cov, group_mean, star_covs, star_means, nstars ) # Compare with various implementations of calculating one star at a time for i in range(nstars): # Using numpy numpy_ols = compute_overlap( group_icov, group_mean, group_icov_det, star_icovs[i], star_means[i], star_icov_dets[i]) # Using swig module (input passed as python list) swig_ols = overlap.get_overlap2( group_icov.flatten().tolist(), group_mean.flatten().tolist(), group_icov_det, star_icovs[i].flatten().tolist(), star_means[i].flatten().tolist(), star_icov_dets[i]) # Using swig module (input passed as numpy arrays) swig_np_ols = overlap.get_overlap( group_icov, group_mean, group_icov_det, star_icovs[i], star_means[i], star_icov_dets[i]) # # Using swig module (input passed as numpy arrays, output in log) # swig_np_lnol = overlap.get_lnoverlap( # group_icov, group_mean, group_icov_det, star_icovs[i], # star_means[i], star_icov_dets[i]) assert np.isclose(numpy_ols, swig_np_ms_ols[i], rtol=1e-8) assert np.isclose(numpy_ols, swig_ols, rtol=1e-8) assert np.isclose(numpy_ols, swig_np_ols, rtol=1e-8) assert np.isclose(numpy_ols, np.exp(swig_np_ms_lnols[i]), rtol=1e-8) print("All implementations return same result to 8 sigfigs")
def correctness(group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, nstars): """ Displays the result for each function, no differences should occur. """ # Using swig-numpy module with multiple stars per call swig_np_ms_ols = overlap.get_overlaps(group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, nstars) # Using swig-numpy module with multiple stars per call and log results group_cov = np.linalg.inv(group_icov) star_covs = np.zeros(star_icovs.shape) for i in range(nstars): star_covs[i] = np.linalg.inv(star_icovs[i]) swig_np_ms_lnols = overlap.get_lnoverlaps(group_cov, group_mean, star_covs, star_means, nstars) # Compare with various implementations of calculating one star at a time for i in range(nstars): # Using numpy numpy_ols = compute_overlap(group_icov, group_mean, group_icov_det, star_icovs[i], star_means[i], star_icov_dets[i]) # Using swig module (input passed as python list) swig_ols = overlap.get_overlap2(group_icov.flatten().tolist(), group_mean.flatten().tolist(), group_icov_det, star_icovs[i].flatten().tolist(), star_means[i].flatten().tolist(), star_icov_dets[i]) # Using swig module (input passed as numpy arrays) swig_np_ols = overlap.get_overlap(group_icov, group_mean, group_icov_det, star_icovs[i], star_means[i], star_icov_dets[i]) # # Using swig module (input passed as numpy arrays, output in log) # swig_np_lnol = overlap.get_lnoverlap( # group_icov, group_mean, group_icov_det, star_icovs[i], # star_means[i], star_icov_dets[i]) assert np.isclose(numpy_ols, swig_np_ms_ols[i], rtol=1e-8) assert np.isclose(numpy_ols, swig_ols, rtol=1e-8) assert np.isclose(numpy_ols, swig_np_ols, rtol=1e-8) assert np.isclose(numpy_ols, np.exp(swig_np_ms_lnols[i]), rtol=1e-8) print("All implementations return same result to 8 sigfigs")
dt.mvGauss(later_groups[i].mean, later_groups[i].mean, later_groups[i].generateSphericalCovMatrix(), later_z[:, i].sum()))) print("bg dens during run for reference:") print(np.exp(em.backgroundLogOverlap(init_groups[0].mean, hists_from_run))) # -------------------------------------------------- # ----- an aside, does a delta function at --------- # ----- group mean return same as peak val --------- # -------------------------------------------------- # spoiler... yes, yes it does lg0 = later_groups[0] delta_ol_at_mean =\ np.exp(get_lnoverlaps(lg0.generateSphericalCovMatrix(), lg0.mean, np.array([1e-10*np.eye(6)]), np.array([lg0.mean]), 1))[0] peak_dens = dt.mvGauss(lg0.mean, lg0.mean, lg0.generateSphericalCovMatrix()) assert np.isclose(delta_ol_at_mean, peak_dens) # demo synthesising some bg stars my_bg_dens = 1e-7 spread = np.std(gaia_xyzuvw, axis=0) / 8. ubound = bpmg_mean + 0.5 * spread lbound = bpmg_mean - 0.5 * spread nbg_stars = int(my_bg_dens * np.prod(spread)) print("There are {} stars in bg".format(nbg_stars)) my_bg_stars = np.random.uniform(-1, 1, size=(nbg_stars, 6)) * spread + bpmg_mean
def timings(group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, batch_size, noverlaps=10000): """ Executes each function a fixed number of times, timing for how long it takes. """ if (noverlaps <= 100000): npstart = time.clock() for i in range(noverlaps): result = compute_overlap(group_icov, group_mean, group_icov_det, star_icovs[0], star_means[0], star_icov_dets[0]) print "Numpy: " + str(time.clock() - npstart) else: print "Numpy: practically infinity seconds" print " -> (approximately 5x 'Swig')" swigstart = time.clock() for i in range(noverlaps): result = overlap.get_overlap2(group_icov.flatten().tolist(), group_mean.flatten().tolist(), group_icov_det, star_icovs[0].flatten().tolist(), star_means[0].flatten().tolist(), star_icov_dets[0]) print "Swig: {} s".format(time.clock() - swigstart) swignpstart = time.clock() for i in range(noverlaps): result = overlap.get_overlap(group_icov, group_mean, group_icov_det, star_icovs[0], star_means[0], star_icov_dets[0]) end = time.clock() print "Swigging numpy: {} s".format(end - swignpstart) swignpmultistart = time.clock() for i in range(noverlaps/batch_size): result = overlap.get_overlaps( group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, batch_size) end = time.clock() print("Swigging numpy multi: {} s".format(end - swignpmultistart)) print(" -> total module calls: {}".format(noverlaps/batch_size)) print(" -> {} microsec per overlap".\ format((end - swignpmultistart)/noverlaps*1e6)) print(" -> {} stars per module call".format(batch_size)) group_cov = np.linalg.inv(group_icov) star_covs = np.linalg.inv(star_icovs) newswignpmultistart = time.clock() for i in range(noverlaps/batch_size): result = overlap.get_lnoverlaps(group_cov, group_mean, star_covs, star_means, batch_size) end = time.clock() print("Swigging numpy multi logged: {} s".format(end - newswignpmultistart)) print(" -> total module calls: {}".format(noverlaps/batch_size)) print(" -> {} microsec per overlap".\ format((end - newswignpmultistart)/noverlaps*1e6)) print(" -> {} stars per module call".format(batch_size))
# BROADCAST CONSTANTS nstars = comm.bcast(nstars, root=0) background_means = comm.bcast(background_means, root=0) background_covs = comm.bcast(background_covs, root=0) # SCATTER DATA star_means = comm.scatter(star_means, root=0) star_covs = comm.scatter(star_covs, root=0) #print(rank, len(star_means)) # EVERY PROCESS DOES THIS FOR ITS DATA bg_ln_ols = [] for star_cov, star_mean in zip(star_covs, star_means): try: bg_lnol = get_lnoverlaps(star_cov, star_mean, background_covs, background_means, nstars) bg_lnol = logsumexp(bg_lnol) # sum in linear space except: # TC: Changed sign to negative (surely if it fails, we want it to # have a neglible background overlap? print('bg ln overlap failed, setting it to -inf') bg_lnol = -np.inf bg_ln_ols.append(bg_lnol) #print(rank, bg_ln_ols) # GATHER DATA bg_ln_ols_result = comm.gather(bg_ln_ols, root=0) if rank == 0: bg_ln_ols_result = list(itertools.chain.from_iterable(bg_ln_ols_result)) np.savetxt('bgols_multiprocessing_%d.dat' % NI, bg_ln_ols_result)
def timings(group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, batch_size, noverlaps=10000): """ Executes each function a fixed number of times, timing for how long it takes. """ if (noverlaps <= 100000): npstart = time.clock() for i in range(noverlaps): result = compute_overlap(group_icov, group_mean, group_icov_det, star_icovs[0], star_means[0], star_icov_dets[0]) print "Numpy: " + str(time.clock() - npstart) else: print "Numpy: practically infinity seconds" print " -> (approximately 5x 'Swig')" swigstart = time.clock() for i in range(noverlaps): result = overlap.get_overlap2(group_icov.flatten().tolist(), group_mean.flatten().tolist(), group_icov_det, star_icovs[0].flatten().tolist(), star_means[0].flatten().tolist(), star_icov_dets[0]) print "Swig: {} s".format(time.clock() - swigstart) swignpstart = time.clock() for i in range(noverlaps): result = overlap.get_overlap(group_icov, group_mean, group_icov_det, star_icovs[0], star_means[0], star_icov_dets[0]) end = time.clock() print "Swigging numpy: {} s".format(end - swignpstart) swignpmultistart = time.clock() for i in range(noverlaps / batch_size): result = overlap.get_overlaps(group_icov, group_mean, group_icov_det, star_icovs, star_means, star_icov_dets, batch_size) end = time.clock() print("Swigging numpy multi: {} s".format(end - swignpmultistart)) print(" -> total module calls: {}".format(noverlaps / batch_size)) print(" -> {} microsec per overlap".\ format((end - swignpmultistart)/noverlaps*1e6)) print(" -> {} stars per module call".format(batch_size)) group_cov = np.linalg.inv(group_icov) star_covs = np.linalg.inv(star_icovs) newswignpmultistart = time.clock() for i in range(noverlaps / batch_size): result = overlap.get_lnoverlaps(group_cov, group_mean, star_covs, star_means, batch_size) end = time.clock() print("Swigging numpy multi logged: {} s".format(end - newswignpmultistart)) print(" -> total module calls: {}".format(noverlaps / batch_size)) print(" -> {} microsec per overlap".\ format((end - newswignpmultistart)/noverlaps*1e6)) print(" -> {} stars per module call".format(batch_size))
xyzuvw_file = "../data/fed_stars_20_xyzuvw.fits" xyzuvw_dict = gf.loadXYZUVW(xyzuvw_file) star_means = xyzuvw_dict['xyzuvw'] star_covs = xyzuvw_dict['xyzuvw_cov'] nstars = star_means.shape[0] gmn = np.mean(star_means, axis=0) gcov = np.cov(star_means.T) if DEBUG_SINGULAR: scov = star_covs[20] smn = star_means[20] py_lnol = sclno(gcov, gmn, np.array([scov]), np.array([smn]), 1) ol.get_lnoverlaps(gcov, gmn, np.array([scov]), np.array([smn]), 1) if DEBUG_UNDERFLOW: scov = star_covs[0] smn = star_means[0] #py_lnol = sclno(gcov, gmn, np.array([scov]), np.array([smn]), 1) py_lnol = clno(gcov, gmn, scov, smn) # gcovi = np.linalg.inv(gcov) # scovi = np.linalg.inv(scov) # gcovi_det = np.linalg.det(gcovi) # scovi_det = np.linalg.det(scovi) # c_ol1 = ol.get_overlap(gcovi, gmn, gcovi_det, # scovi, smn, scovi_det) # c_ol2 = ol.get_overlaps(gcovi, gmn, gcovi_det, # np.array([scovi]), np.array([smn]),