def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if isinstance(data, Observable):
assert (data.shape[0] == 1)
self._archive.update({name: data}) # rw
elif isinstance(data, Field):
assert (data.shape[0] == 1)
self._archive.update(
{name: Observable(data.domain, data.local_data)}) # rw
# reading from numpy array takes information only on master node
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(int(1)), RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(int(1)), HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)}) # rw
else:
raise TypeError('unsupported data type')
log.debug('measurements-dict appends data %s' % str(name))
def test_without_simcov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock covariance
dtuple = DomainTuple.make((RGSpace(shape=(48, 48))))
arr_c = np.random.rand(48, 48)
comm.Bcast(arr_c, root=0)
cov = Field.from_global_data(dtuple, arr_c)
covdict.append(('test', 'nan', '2', 'nan'), cov)
# mock observable with repeated single realisation
dtuple = DomainTuple.make((RGSpace(5*mpisize), HPSpace(nside=2)))
arr_b = np.random.rand(1, 48)
comm.Bcast(arr_b, root=0)
arr_ens = np.zeros((5, 48))
for i in range(len(arr_ens)):
arr_ens[i] = arr_b
sim = Observable(dtuple, arr_ens)
simdict.append(('test', 'nan', '2', 'nan'), sim)
# simplelikelihood
lh_simple = SimpleLikelihood(meadict, covdict)
rslt_simple = lh_simple(simdict)
# ensemblelikelihood
lh_ensemble = EnsembleLikelihood(meadict, covdict)
rslt_ensemble = lh_ensemble(simdict)
self.assertEqual(rslt_ensemble, rslt_simple)
def test_with_cov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
arr_a = np.random.rand(1, 4 * mpisize)
comm.Bcast(arr_a, root=0)
mea = Observable(arr_a, 'measured')
meadict.append(('test', 'nan', str(4 * mpisize), 'nan'), mea, True)
# mock sims
arr_b = np.random.rand(5, 4 * mpisize)
sim = Observable(arr_b, 'simulated')
simdict.append(('test', 'nan', str(4 * mpisize), 'nan'), sim, True)
# mock covariance
arr_c = np.random.rand(4, 4 * mpisize)
cov = Observable(arr_c, 'covariance')
covdict.append(('test', 'nan', str(4 * mpisize), 'nan'), cov, True)
# with covariance
lh = SimpleLikelihood(meadict, covdict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
full_b = np.vstack(comm.allgather(arr_b)) # global arr_b
diff = (np.mean(full_b, axis=0) - arr_a)
full_cov = np.vstack(comm.allgather(arr_c)) # global covariance
(sign, logdet) = np.linalg.slogdet(full_cov * 2. * np.pi)
baseline = -float(0.5) * float(
np.vdot(diff, np.linalg.solve(full_cov, diff.T)) + sign * logdet)
self.assertAlmostEqual(rslt, baseline)
def test_without_cov(self):
simdict = Simulations()
meadict = Measurements()
# mock measurements
arr_a = np.random.rand(1, 12 * mpisize**2)
comm.Bcast(arr_a, root=0)
mea = Observable(arr_a, 'measured')
meadict.append(('test', 'nan', str(mpisize), 'nan'), mea)
# mock observable with repeated single realisation
arr_b = np.random.rand(1, 12 * mpisize**2)
comm.Bcast(arr_b, root=0)
if not mpirank:
arr_ens = np.zeros((3, 12 * mpisize**2))
else:
arr_ens = np.zeros((2, 12 * mpisize**2))
for i in range(len(arr_ens)):
arr_ens[i] = arr_b
sim = Observable(arr_ens, 'simulated')
simdict.append(('test', 'nan', str(mpisize), 'nan'), sim)
# simplelikelihood
lh_simple = SimpleLikelihood(meadict)
rslt_simple = lh_simple(simdict)
# ensemblelikelihood
lh_ensemble = EnsembleLikelihood(meadict)
rslt_ensemble = lh_ensemble(simdict)
self.assertEqual(rslt_ensemble, rslt_simple)
def test_without_simcov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
arr_a = np.random.rand(1, 4 * mpisize)
comm.Bcast(arr_a, root=0)
mea = Observable(arr_a, 'measured')
meadict.append(('test', 'nan', str(4 * mpisize), 'nan'), mea, True)
# mock covariance
arr_c = np.random.rand(4, 4 * mpisize)
cov = Observable(arr_c, 'covariance')
covdict.append(('test', 'nan', str(4 * mpisize), 'nan'), cov, True)
# mock observable with repeated single realisation
arr_b = np.random.rand(1, 4 * mpisize)
comm.Bcast(arr_b, root=0)
arr_ens = np.zeros((2, 4 * mpisize))
for i in range(len(arr_ens)):
arr_ens[i] = arr_b
sim = Observable(arr_ens, 'simulated')
simdict.append(('test', 'nan', str(4 * mpisize), 'nan'), sim, True)
# simplelikelihood
lh_simple = SimpleLikelihood(meadict, covdict)
rslt_simple = lh_simple(simdict)
# ensemblelikelihood
lh_ensemble = EnsembleLikelihood(meadict, covdict)
rslt_ensemble = lh_ensemble(simdict)
self.assertEqual(rslt_ensemble, rslt_simple)
def append(self, *args, **kwargs):
log.debug('@ observable_dict::Covariances::append')
name, _, data, otype, coords = super().append(*args, **kwargs)
if isinstance(data, Observable):
self._archive.update({name: data})
elif isinstance(data, np.ndarray):
# Covariances case
if len(data.shape) == 2:
if otype == 'HEALPix':
assert (data.shape[1] == _Nside_to_Npixels(name[2]))
self._archive.update({
name:
Observable(data,
dtype='covariance',
coords=coords,
otype=otype)
})
# Variances case
else:
self._archive.update({
name:
Observable(data,
dtype='variance',
coords=coords,
otype=otype)
})
else:
raise TypeError('unsupported data type')
def test_init(self):
# initialize observable
dtuple = DomainTuple.make((RGSpace(3 * mpisize), HPSpace(nside=2)))
val = np.random.rand(3, 48)
obs = Observable(dtuple, val)
# collect local val at master
val_master = None
if mpirank == 0:
val_master = np.zeros((3 * mpisize, 48))
comm.Gather(val, val_master, root=0)
# test domain/field shape
self.assertEqual(obs.domain, dtuple)
if mpirank == 0:
self.assertEqual(obs.shape, val_master.shape)
# test function to_global_data()
raw = obs.to_global_data()
if mpirank == 0:
for i in range(len(val_master)):
self.assertListEqual(list(raw[i]), list(val_master[i]))
# test function ensemble_mean
mean = obs.ensemble_mean
if mpirank == 0:
val_mean = np.mean(val_master, axis=0)
for i in range(val_mean.size):
self.assertAlmostEqual(mean[0][i], val_mean[i])
def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if name in self._archive.keys(): # app
self._archive[name].append(data)
else: # new
if isinstance(data, Observable):
self._archive.update({name: data})
elif isinstance(data, Field):
self._archive.update(
{name: Observable(data.domain, data.local_data)})
elif isinstance(data, np.ndarray): # distributed data
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(data.shape[0] * mpisize),
RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(data.shape[0] * mpisize),
HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)})
else:
raise TypeError('unsupported data type')
log.debug('observable-dict appends data %s' % str(name))
def test_with_cov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), RGSpace(12)))
arr_a = np.random.rand(1, 12)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '12', 'nan'), mea, True)
# mock sims
dtuple = DomainTuple.make((RGSpace(5*mpisize), RGSpace(12)))
arr_b = np.random.rand(5, 12)
sim = Observable(dtuple, arr_b)
simdict.append(('test', 'nan', '12', 'nan'), sim, True)
# mock covariance
arr_c = np.random.rand(12, 12)
comm.Bcast(arr_c, root=0)
dtuple = DomainTuple.make((RGSpace(shape=arr_c.shape)))
cov = Field.from_global_data(dtuple, arr_c)
covdict.append(('test', 'nan', '12', 'nan'), cov, True)
# with covariance
lh = SimpleLikelihood(meadict, covdict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
arr_b = sim.to_global_data() # get global arr_b
diff = (np.mean(arr_b, axis=0) - arr_a)
(sign, logdet) = np.linalg.slogdet(arr_c*2.*np.pi)
baseline = -float(0.5)*float(np.vdot(diff, np.linalg.solve(arr_c, diff.T))+sign*logdet)
self.assertAlmostEqual(rslt, baseline)
def test_1dinit(self):
# initialize observable
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
val = np.random.rand(1, 48)
comm.Bcast(val, root=0)
obs = Observable(dtuple, val)
# matches val at master
raw = obs.to_global_data()
self.assertListEqual(list(raw[0]), list(val[0]))
def test_append_with_rewrite(self):
if not mpirank:
arr = np.random.rand(2, 128)
else:
arr = np.random.rand(1, 128)
test_obs = Observable(arr, 'simulated')
test_obs.rw_flag = True
brr = np.random.rand(1, 128)
test_obs.append(brr)
global_shape = test_obs.shape
globalrr = test_obs.global_data
if not mpirank:
self.assertEqual(global_shape, globalrr.shape)
self.assertListEqual(list(test_obs.data[0]), list(brr[0]))
def test_append_ndarray(self):
if not mpirank:
arr = np.random.rand(2, 128)
else:
arr = np.random.rand(1, 128)
test_obs = Observable(arr, 'simulated')
brr = np.random.rand(1, 128)
test_obs.append(brr)
global_shape = test_obs.shape
globalrr = test_obs.global_data
if not mpirank:
self.assertEqual(global_shape, globalrr.shape)
fullrr = np.vstack([arr, brr])
for i in range(fullrr.shape[0]):
self.assertListEqual(list(fullrr[i]), list(test_obs.data[i]))
def test_init_measure(self):
arr = np.random.rand(1, 128)
test_obs = Observable(arr, 'measured')
self.assertEqual(test_obs.dtype, 'measured')
self.assertEqual(test_obs.shape, (mpisize, 128))
self.assertListEqual(list(arr[0]), list(test_obs.data[0]))
self.assertListEqual(list(arr[0]), list(test_obs.ensemble_mean[0]))
def test_init_covariance(self):
arr = np.random.rand(1, mpisize)
test_obs = Observable(arr, 'covariance')
self.assertEqual(test_obs.dtype, 'covariance')
self.assertEqual(test_obs.shape, (mpisize, mpisize))
self.assertListEqual(list(arr[0]), list(test_obs.data[0]))
self.assertEqual(test_obs.size, mpisize)
def append(self, name, new, plain=False):
"""
Adds/updates name and data
Parameters
----------
name : str tuple
Should follow the convention:
``(data-name,str(data-freq),str(data-Nside/size),str(ext))``.
If data is independent from frequency, set 'nan'.
`ext` can be 'I','Q','U','PI','PA', 'nan' or other customized tags.
data
distributed/copied ndarray/Observable
plain : bool
If True, means unstructured data.
If False (default case), means HEALPix-like sky map.
"""
log.debug('@ observable_dict::Covariances::append')
assert (len(name) == 4)
if isinstance(new, Observable): # always rewrite
if plain:
assert (new.size == np.uint(name[2]))
else:
assert (new.size == 12 * np.uint(name[2])**2)
self._archive.update({name: new}) # rw
elif isinstance(new, np.ndarray):
if plain:
assert (new.shape[1] == np.uint(name[2]))
else:
assert (new.shape[1] == 12 * np.uint(name[2])**2)
self._archive.update({name: Observable(new, 'covariance')})
else:
raise TypeError('unsupported data type')
def test_append_ndarray(self):
dtuple = DomainTuple.make((RGSpace(3 * mpisize), HPSpace(nside=2)))
val = np.random.rand(3, 48)
obs = Observable(dtuple, val)
# test function append with nd array
new_data = np.random.rand(6, 48)
obs.append(new_data)
raw_obs = obs.to_global_data()
new_val = np.vstack([val, new_data])
# collect new val at master
new_val_master = None
if mpirank == 0:
new_val_master = np.zeros((9 * mpisize, 48))
comm.Gather(new_val, new_val_master, root=0)
# do comparison at master
if mpirank == 0:
for i in range(new_val_master.shape[0]):
self.assertListEqual(list(raw_obs[i]), list(new_val_master[i]))
def test_append_obs(self):
if not mpirank:
arr = np.random.rand(2, 128)
else:
arr = np.random.rand(1, 128)
test_obs = Observable(arr, 'simulated')
if not mpirank:
brr = np.random.rand(1, 128)
else:
brr = np.random.rand(1, 128)
test_obs2 = Observable(brr, 'simulated')
test_obs2.append(test_obs)
global_shape = test_obs2.shape
globalrr = test_obs2.global_data
if not mpirank:
self.assertEqual(global_shape, globalrr.shape)
fullrr = np.vstack([arr, brr])
for i in range(fullrr.shape[0]):
self.assertTrue(test_obs2.data[i] in fullrr)
def test_measuredict_append_observable(self):
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
hrr = np.random.rand(1, 48)
obs1 = Observable(dtuple, hrr)
measuredict = Measurements()
measuredict.append(('test', 'nan', '2', 'nan'),
obs1) # healpix Observable
if mpirank == 0:
self.assertListEqual(
list(measuredict[('test', 'nan', '2',
'nan')].to_global_data()[0]), list(hrr[0]))
dtuple = DomainTuple.make((RGSpace(1), RGSpace(3)))
arr = np.random.rand(1, 3)
obs2 = Observable(dtuple, arr)
measuredict.append(('test', 'nan', '3', 'nan'),
obs2) # plain Observable
if mpirank == 0:
self.assertListEqual(
list(measuredict[('test', 'nan', '3',
'nan')].to_global_data()[0]), list(arr[0]))
def append(self, *args, **kwargs):
log.debug('@ observable_dict::Covariances::append')
name, _, data, otype, _ = super().append(*args, **kwargs)
if isinstance(data, Observable): # always rewrite
self._archive.update({name: data}) # rw
elif isinstance(data, np.ndarray):
if otype == 'HEALPix':
assert (data.shape[1] == _Nside_to_Npixels(name[2]))
self._archive.update({name: Observable(data, 'covariance')})
else:
raise TypeError('unsupported data type')
def test_without_cov(self):
simdict = Simulations()
meadict = Measurements()
# mock measurements
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(arr_a, 'measured')
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock sims
arr_b = np.random.rand(3, 48)
sim = Observable(arr_b, 'simulated')
simdict.append(('test', 'nan', '2', 'nan'), sim)
# no covariance
lh = SimpleLikelihood(meadict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
full_b = np.vstack(comm.allgather(arr_b)) # global arr_b
diff = (np.mean(full_b, axis=0) - arr_a)
baseline = -float(0.5) * float(np.vdot(diff, diff))
# comapre
self.assertAlmostEqual(rslt, baseline)
def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if isinstance(data, Observable):
assert (data.shape[0] == mpisize)
raw_data = data.local_data
for i in raw_data[0]:
assert (i == float(0) or i == float(1))
self._archive.update({name: data})
elif isinstance(data, Field):
assert (data.shape[0] == mpisize)
raw_data = data.local_data
for i in raw_data[0]:
assert (i == float(0) or i == float(1))
self._archive.update(
{name: Observable(data.domain, data.local_data)})
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
for i in data[0]:
assert (i == float(0) or i == float(1))
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(mpisize), RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(mpisize), HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)})
else:
raise TypeError('unsupported data type')
log.debug('mask-dict appends data %s' % str(name))
def test_simdict_append_observable(self):
dtuple = DomainTuple.make((RGSpace(2 * mpisize), HPSpace(nside=2)))
hrr = np.random.rand(2, 48)
obs1 = Observable(dtuple, hrr)
simdict = Simulations()
simdict.append(('test', 'nan', '2', 'nan'), obs1) # healpix Observable
self.assertEqual(simdict[('test', 'nan', '2', 'nan')].shape,
(2 * mpisize, 48))
for i in range(len(hrr)):
self.assertListEqual(
list((simdict[('test', 'nan', '2', 'nan')].local_data)[i]),
list(hrr[i]))
dtuple = DomainTuple.make((RGSpace(5 * mpisize), RGSpace(3)))
arr = np.random.rand(5, 3)
obs2 = Observable(dtuple, arr)
simdict.append(('test', 'nan', '3', 'nan'), obs2,
True) # plain Observable
self.assertEqual(simdict[('test', 'nan', '3', 'nan')].shape,
(5 * mpisize, 3))
for i in range(len(arr)):
self.assertListEqual(
list((simdict[('test', 'nan', '3', 'nan')].local_data)[i]),
list(arr[i]))
def test_without_cov(self):
simdict = Simulations()
meadict = Measurements()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock sims
dtuple = DomainTuple.make((RGSpace(3*mpisize), HPSpace(nside=2)))
arr_b = np.random.rand(3, 48)
sim = Observable(dtuple, arr_b)
simdict.append(('test', 'nan', '2', 'nan'), sim)
# no covariance
lh = SimpleLikelihood(meadict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
arr_b = sim.to_global_data() # global arr_b
diff = (np.mean(arr_b, axis=0) - arr_a)
baseline = -float(0.5)*float(np.vdot(diff, diff))
# comapre
self.assertAlmostEqual(rslt, baseline)
def test_append_twice(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
val = np.random.rand(1, 48)
obs = Observable(dtuple, val)
# test function append with 1d array
new_data = np.random.rand(1, 48)
obs.append(new_data)
self.assertEqual(obs.shape, (2 * mpisize, 48))
obs.append(new_data)
self.assertEqual(obs.shape, (3 * mpisize, 48))
def test_append_after_replace(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
obs = Observable(dtuple)
self.assertTrue(obs.rw_flag)
# test function append with 1d array
new_data = np.random.rand(1, 48)
obs.append(new_data)
self.assertEqual(obs.shape, (1 * mpisize, 48))
obs.append(new_data)
self.assertEqual(obs.shape, (2 * mpisize, 48))
def test_oas(self):
# mock observable
arr_a = np.random.rand(1, 4)
comm.Bcast(arr_a, root=0)
arr_ens = np.zeros((3, 4))
null_cov = np.zeros((4, 4))
# ensemble with identical realisations
for i in range(len(arr_ens)):
arr_ens[i] = arr_a
dtuple = DomainTuple.make((RGSpace(3 * mpisize), RGSpace(4)))
obs = Observable(dtuple, arr_ens)
test_mean, test_cov = oas_mcov(obs)
for i in range(len(arr_a)):
self.assertAlmostEqual(test_mean[0][i], arr_a[0][i])
for j in range(len(arr_a)):
self.assertAlmostEqual(test_cov[i][j], null_cov[i][j])
def append(self, *args, **kwargs):
log.debug('@ observable_dict::Masks::append')
name, data, _, otype, _ = super().append(*args, **kwargs)
if isinstance(data, Observable):
assert (data.dtype == 'measured')
if otype == 'HEALPix':
assert (data.size == 12*np.uint(name[2])**2)
self._archive.update({name: data})
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
if otype == 'HEALPix':
assert (data.shape[1] == _Nside_to_Npixels(name[2]))
self._archive.update({name: Observable(data, 'measured')})
else:
raise TypeError('unsupported data type')
def append(self, *args, **kwargs):
log.debug('@ observable_dict::Simulations::append')
name, data, _, otype, coords = super().append(*args, **kwargs)
if name in self._archive.keys(): # app
self._archive[name].rw_flag = False
self._archive[name].append(data)
else: # data
if isinstance(data, Observable):
self._archive.update({name: data})
elif isinstance(data, np.ndarray): # distributed data
self._archive.update({name: Observable(data=data,
dtype='simulated',
coords=coords,
otype=otype)})
else:
raise TypeError('unsupported data type')
def test_append_after_rewrite(self):
arr = np.random.rand(1, 128)
test_obs = Observable(arr, 'simulated')
if not mpirank:
brr = np.random.rand(2, 128)
else:
brr = np.random.rand(1, 128)
test_obs.rw_flag = True
test_obs.append(brr)
crr = np.random.rand(1, 128)
# rw_flag must have be switched off
test_obs.append(crr)
global_shape = test_obs.shape
globalrr = test_obs.global_data
if not mpirank:
self.assertEqual(global_shape, globalrr.shape)
fullrr = np.vstack([brr, crr])
for i in range(fullrr.shape[0]):
self.assertTrue(test_obs.data[i] in fullrr)
def append(self, *args, **kwargs):
log.debug('@ observable_dict::Measurements::append')
name, data, cov, otype, coords = super().append(*args, **kwargs)
if cov is not None:
if self.cov is None:
self.cov = Covariances()
self.cov.append(*args, **kwargs)
if isinstance(data, Observable):
assert (data.dtype == 'measured')
self._archive.update({name: data})
elif isinstance(data, np.ndarray):
if otype == 'HEALPix':
assert (data.shape[1] == _Nside_to_Npixels(name[2]))
self._archive.update({name: Observable(data=data,
dtype='measured',
coords=coords,
otype=otype)})
else:
raise TypeError('Unsupported data type')