def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if name in self._archive.keys(): # app
self._archive[name].append(data)
else: # new
if isinstance(data, Observable):
self._archive.update({name: data})
elif isinstance(data, Field):
self._archive.update(
{name: Observable(data.domain, data.local_data)})
elif isinstance(data, np.ndarray): # distributed data
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(data.shape[0] * mpisize),
RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(data.shape[0] * mpisize),
HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)})
else:
raise TypeError('unsupported data type')
log.debug('observable-dict appends data %s' % str(name))
def append(self, name, data, plain=False):
"""
matrix coming in, numpy.matrix is not recommended
matrix must be distributed
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if plain:
assert (data.shape[1] == int(name[2]))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
if isinstance(data, Field):
assert (data.shape[0] == data.shape[1])
assert (len(data.domain) == 1) # single domain
self._archive.update({name: data}) # rw
elif isinstance(data, np.ndarray):
if data.shape[0] < data.shape[1] // mpisize + int(
2): # distributed
domain = DomainTuple.make(
RGSpace(shape=(data.shape[1], data.shape[1])))
self._archive.update(
{name: Field.from_local_data(domain, data)}) # rw
elif data.shape[0] == data.shape[1]: # non-distributed
domain = DomainTuple.make(RGSpace(shape=data.shape))
self._archive.update(
{name: Field.from_global_data(domain, data)}) # rw
else:
raise ValueError('unsupported data shape')
else:
raise TypeError('unsupported data type')
log.debug('covariances-dict appends data %s' % str(name))
def test_with_cov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), RGSpace(12)))
arr_a = np.random.rand(1, 12)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '12', 'nan'), mea, True)
# mock sims
dtuple = DomainTuple.make((RGSpace(5*mpisize), RGSpace(12)))
arr_b = np.random.rand(5, 12)
sim = Observable(dtuple, arr_b)
simdict.append(('test', 'nan', '12', 'nan'), sim, True)
# mock covariance
arr_c = np.random.rand(12, 12)
comm.Bcast(arr_c, root=0)
dtuple = DomainTuple.make((RGSpace(shape=arr_c.shape)))
cov = Field.from_global_data(dtuple, arr_c)
covdict.append(('test', 'nan', '12', 'nan'), cov, True)
# with covariance
lh = SimpleLikelihood(meadict, covdict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
arr_b = sim.to_global_data() # get global arr_b
diff = (np.mean(arr_b, axis=0) - arr_a)
(sign, logdet) = np.linalg.slogdet(arr_c*2.*np.pi)
baseline = -float(0.5)*float(np.vdot(diff, np.linalg.solve(arr_c, diff.T))+sign*logdet)
self.assertAlmostEqual(rslt, baseline)
def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if isinstance(data, Observable):
assert (data.shape[0] == 1)
self._archive.update({name: data}) # rw
elif isinstance(data, Field):
assert (data.shape[0] == 1)
self._archive.update(
{name: Observable(data.domain, data.local_data)}) # rw
# reading from numpy array takes information only on master node
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(int(1)), RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(int(1)), HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)}) # rw
else:
raise TypeError('unsupported data type')
log.debug('measurements-dict appends data %s' % str(name))
def test_without_simcov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock covariance
dtuple = DomainTuple.make((RGSpace(shape=(48, 48))))
arr_c = np.random.rand(48, 48)
comm.Bcast(arr_c, root=0)
cov = Field.from_global_data(dtuple, arr_c)
covdict.append(('test', 'nan', '2', 'nan'), cov)
# mock observable with repeated single realisation
dtuple = DomainTuple.make((RGSpace(5*mpisize), HPSpace(nside=2)))
arr_b = np.random.rand(1, 48)
comm.Bcast(arr_b, root=0)
arr_ens = np.zeros((5, 48))
for i in range(len(arr_ens)):
arr_ens[i] = arr_b
sim = Observable(dtuple, arr_ens)
simdict.append(('test', 'nan', '2', 'nan'), sim)
# simplelikelihood
lh_simple = SimpleLikelihood(meadict, covdict)
rslt_simple = lh_simple(simdict)
# ensemblelikelihood
lh_ensemble = EnsembleLikelihood(meadict, covdict)
rslt_ensemble = lh_ensemble(simdict)
self.assertEqual(rslt_ensemble, rslt_simple)
def append(self, new_data):
"""
append new_data from all nodes
if the observable is not distributed, self.shape[0] % mpisize != 0,
it should be a measurement/observational data, in this case,
there is no need to append/rewrite new data,
nor it is convenient to do so
:param new_data: new data in type numpy array, NIFTy5.Field, Observable
:return:
since Field is read only, to append new data
we need to strip ndarray out, extend, then update Field
append also handle ._flag is True case
which means instead of append new data
we should rewrite
"""
assert(self._field.shape[0] % mpisize == 0) # non-distributed, do not append
# strip data
if isinstance(new_data, (Field, Observable)):
raw_new = new_data.local_data # return to each node
elif isinstance(new_data, np.ndarray):
raw_new = new_data
else:
raise TypeError('unsupported type')
# assemble new_cache
if self.rw_flag:
local_cache = raw_new
self.rw_flag = False # rw only once by default
else:
local_cache = np.vstack([self._field.local_data, raw_new])
# update new_cache to ._field, first need to get global_size
new_domain = DomainTuple.make((RGSpace(local_cache.shape[0]*mpisize), self._domain[1]))
self._field = Field.from_local_data(new_domain, local_cache)
def apply_mask(self, mask_dict):
if mask_dict is None:
pass
else:
assert isinstance(mask_dict, Masks)
for name, msk in mask_dict._archive.items():
if name in self._archive.keys():
masked = mask_cov(self._archive[name].local_data,
msk.local_data)
remain = msk.local_data.sum()
# assemble masked pieces
if mpirank == 0:
for i in range(1, mpisize):
req = comm.irecv(source=i)
newslice = req.wait()
masked = np.vstack([masked, newslice])
else:
comm.isend(masked, dest=0)
masked = np.zeros((remain, remain))
comm.Bcast(masked, root=0)
new_name = (name[0], name[1], str(remain), name[3])
self._archive.pop(name, None) # pop out obsolete
domain = DomainTuple.make(RGSpace(shape=(remain, remain)))
self._archive.update(
{new_name: Field.from_global_data(domain, masked)})
def test_init(self):
# initialize observable
dtuple = DomainTuple.make((RGSpace(3 * mpisize), HPSpace(nside=2)))
val = np.random.rand(3, 48)
obs = Observable(dtuple, val)
# collect local val at master
val_master = None
if mpirank == 0:
val_master = np.zeros((3 * mpisize, 48))
comm.Gather(val, val_master, root=0)
# test domain/field shape
self.assertEqual(obs.domain, dtuple)
if mpirank == 0:
self.assertEqual(obs.shape, val_master.shape)
# test function to_global_data()
raw = obs.to_global_data()
if mpirank == 0:
for i in range(len(val_master)):
self.assertListEqual(list(raw[i]), list(val_master[i]))
# test function ensemble_mean
mean = obs.ensemble_mean
if mpirank == 0:
val_mean = np.mean(val_master, axis=0)
for i in range(val_mean.size):
self.assertAlmostEqual(mean[0][i], val_mean[i])
def test_1dinit(self):
# initialize observable
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
val = np.random.rand(1, 48)
comm.Bcast(val, root=0)
obs = Observable(dtuple, val)
# matches val at master
raw = obs.to_global_data()
self.assertListEqual(list(raw[0]), list(val[0]))
def test_append_twice(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
val = np.random.rand(1, 48)
obs = Observable(dtuple, val)
# test function append with 1d array
new_data = np.random.rand(1, 48)
obs.append(new_data)
self.assertEqual(obs.shape, (2 * mpisize, 48))
obs.append(new_data)
self.assertEqual(obs.shape, (3 * mpisize, 48))
def test_append_after_replace(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
obs = Observable(dtuple)
self.assertTrue(obs.rw_flag)
# test function append with 1d array
new_data = np.random.rand(1, 48)
obs.append(new_data)
self.assertEqual(obs.shape, (1 * mpisize, 48))
obs.append(new_data)
self.assertEqual(obs.shape, (2 * mpisize, 48))
def test_append_with_replace(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
obs = Observable(dtuple)
self.assertTrue(obs.rw_flag)
# test with empty observable
dtuple = DomainTuple.make((RGSpace(8 * mpisize), HPSpace(nside=2)))
new_data = np.random.rand(8, 48)
new_obs = Observable(dtuple, new_data)
obs.append(new_obs)
raw_obs = obs.to_global_data()
# collect new val at master
new_val_master = None
if mpirank == 0:
new_val_master = np.zeros((8 * mpisize, 48))
comm.Gather(new_data, new_val_master, root=0)
# do comparison at master
if mpirank == 0:
for i in range(new_val_master.shape[0]):
self.assertListEqual(list(raw_obs[i]), list(new_val_master[i]))
def test_measuredict_append_observable(self):
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
hrr = np.random.rand(1, 48)
obs1 = Observable(dtuple, hrr)
measuredict = Measurements()
measuredict.append(('test', 'nan', '2', 'nan'),
obs1) # healpix Observable
if mpirank == 0:
self.assertListEqual(
list(measuredict[('test', 'nan', '2',
'nan')].to_global_data()[0]), list(hrr[0]))
dtuple = DomainTuple.make((RGSpace(1), RGSpace(3)))
arr = np.random.rand(1, 3)
obs2 = Observable(dtuple, arr)
measuredict.append(('test', 'nan', '3', 'nan'),
obs2) # plain Observable
if mpirank == 0:
self.assertListEqual(
list(measuredict[('test', 'nan', '3',
'nan')].to_global_data()[0]), list(arr[0]))
def test_append_observable(self):
dtuple = DomainTuple.make((RGSpace(3 * mpisize), HPSpace(nside=2)))
val = np.random.rand(3, 48)
obs = Observable(dtuple, val)
# test function append with Observable
dtuple = DomainTuple.make((RGSpace(5 * mpisize), HPSpace(nside=2)))
new_data = np.random.rand(5, 48)
new_obs = Observable(dtuple, new_data)
obs.append(new_obs)
raw_obs = obs.to_global_data()
new_val = np.vstack([val, new_data])
# collect new val at master
new_val_master = None
if mpirank == 0:
new_val_master = np.zeros((8 * mpisize, 48))
comm.Gather(new_val, new_val_master, root=0)
# do comparison at master
if mpirank == 0:
for i in range(new_val_master.shape[0]):
self.assertListEqual(list(raw_obs[i]), list(new_val_master[i]))
def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if isinstance(data, Observable):
assert (data.shape[0] == mpisize)
raw_data = data.local_data
for i in raw_data[0]:
assert (i == float(0) or i == float(1))
self._archive.update({name: data})
elif isinstance(data, Field):
assert (data.shape[0] == mpisize)
raw_data = data.local_data
for i in raw_data[0]:
assert (i == float(0) or i == float(1))
self._archive.update(
{name: Observable(data.domain, data.local_data)})
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
for i in data[0]:
assert (i == float(0) or i == float(1))
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(mpisize), RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(mpisize), HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)})
else:
raise TypeError('unsupported data type')
log.debug('mask-dict appends data %s' % str(name))
def test_simdict_append_observable(self):
dtuple = DomainTuple.make((RGSpace(2 * mpisize), HPSpace(nside=2)))
hrr = np.random.rand(2, 48)
obs1 = Observable(dtuple, hrr)
simdict = Simulations()
simdict.append(('test', 'nan', '2', 'nan'), obs1) # healpix Observable
self.assertEqual(simdict[('test', 'nan', '2', 'nan')].shape,
(2 * mpisize, 48))
for i in range(len(hrr)):
self.assertListEqual(
list((simdict[('test', 'nan', '2', 'nan')].local_data)[i]),
list(hrr[i]))
dtuple = DomainTuple.make((RGSpace(5 * mpisize), RGSpace(3)))
arr = np.random.rand(5, 3)
obs2 = Observable(dtuple, arr)
simdict.append(('test', 'nan', '3', 'nan'), obs2,
True) # plain Observable
self.assertEqual(simdict[('test', 'nan', '3', 'nan')].shape,
(5 * mpisize, 3))
for i in range(len(arr)):
self.assertListEqual(
list((simdict[('test', 'nan', '3', 'nan')].local_data)[i]),
list(arr[i]))
def test_without_cov(self):
simdict = Simulations()
meadict = Measurements()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock sims
dtuple = DomainTuple.make((RGSpace(3*mpisize), HPSpace(nside=2)))
arr_b = np.random.rand(3, 48)
sim = Observable(dtuple, arr_b)
simdict.append(('test', 'nan', '2', 'nan'), sim)
# no covariance
lh = SimpleLikelihood(meadict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
arr_b = sim.to_global_data() # global arr_b
diff = (np.mean(arr_b, axis=0) - arr_a)
baseline = -float(0.5)*float(np.vdot(diff, diff))
# comapre
self.assertAlmostEqual(rslt, baseline)
def test_oas(self):
# mock observable
arr_a = np.random.rand(1, 4)
comm.Bcast(arr_a, root=0)
arr_ens = np.zeros((3, 4))
null_cov = np.zeros((4, 4))
# ensemble with identical realisations
for i in range(len(arr_ens)):
arr_ens[i] = arr_a
dtuple = DomainTuple.make((RGSpace(3 * mpisize), RGSpace(4)))
obs = Observable(dtuple, arr_ens)
test_mean, test_cov = oas_mcov(obs)
for i in range(len(arr_a)):
self.assertAlmostEqual(test_mean[0][i], arr_a[0][i])
for j in range(len(arr_a)):
self.assertAlmostEqual(test_cov[i][j], null_cov[i][j])
def test_mask(self):
msk_arr = np.array([0., 1., 0., 1., 1., 0.]).reshape(1, 6)
obs_arr = np.random.rand(1, 6)
cov_arr = (Field.from_global_data(
DomainTuple.make(RGSpace(shape=(6, 6))),
np.random.rand(6, 6))).local_data
# mask by methods
test_obs = mask_obs(obs_arr, msk_arr)
test_cov = mask_cov(cov_arr, msk_arr)
# mask manually
fid_obs = np.hstack([obs_arr[0, 1], obs_arr[0, 3], obs_arr[0, 4]])
# gather global cov
fid_cov = np.zeros((6, 6))
comm.Gather(cov_arr, fid_cov, root=0)
# mask cov by hand
fid_cov = np.delete(fid_cov, [0, 2, 5], 0)
fid_cov = np.delete(fid_cov, [0, 2, 5], 1)
comm.Bcast(fid_cov, root=0)
# compare mask on matrix
for i in test_cov:
self.assertTrue(i in fid_cov)
# compare mask on array
self.assertListEqual(list(test_obs[0]), list(fid_obs))