def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if name in self._archive.keys(): # app
self._archive[name].append(data)
else: # new
if isinstance(data, Observable):
self._archive.update({name: data})
elif isinstance(data, Field):
self._archive.update(
{name: Observable(data.domain, data.local_data)})
elif isinstance(data, np.ndarray): # distributed data
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(data.shape[0] * mpisize),
RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(data.shape[0] * mpisize),
HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)})
else:
raise TypeError('unsupported data type')
log.debug('observable-dict appends data %s' % str(name))
def test_without_simcov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock covariance
dtuple = DomainTuple.make((RGSpace(shape=(48, 48))))
arr_c = np.random.rand(48, 48)
comm.Bcast(arr_c, root=0)
cov = Field.from_global_data(dtuple, arr_c)
covdict.append(('test', 'nan', '2', 'nan'), cov)
# mock observable with repeated single realisation
dtuple = DomainTuple.make((RGSpace(5*mpisize), HPSpace(nside=2)))
arr_b = np.random.rand(1, 48)
comm.Bcast(arr_b, root=0)
arr_ens = np.zeros((5, 48))
for i in range(len(arr_ens)):
arr_ens[i] = arr_b
sim = Observable(dtuple, arr_ens)
simdict.append(('test', 'nan', '2', 'nan'), sim)
# simplelikelihood
lh_simple = SimpleLikelihood(meadict, covdict)
rslt_simple = lh_simple(simdict)
# ensemblelikelihood
lh_ensemble = EnsembleLikelihood(meadict, covdict)
rslt_ensemble = lh_ensemble(simdict)
self.assertEqual(rslt_ensemble, rslt_simple)
def test_with_cov(self):
simdict = Simulations()
meadict = Measurements()
covdict = Covariances()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), RGSpace(12)))
arr_a = np.random.rand(1, 12)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '12', 'nan'), mea, True)
# mock sims
dtuple = DomainTuple.make((RGSpace(5*mpisize), RGSpace(12)))
arr_b = np.random.rand(5, 12)
sim = Observable(dtuple, arr_b)
simdict.append(('test', 'nan', '12', 'nan'), sim, True)
# mock covariance
arr_c = np.random.rand(12, 12)
comm.Bcast(arr_c, root=0)
dtuple = DomainTuple.make((RGSpace(shape=arr_c.shape)))
cov = Field.from_global_data(dtuple, arr_c)
covdict.append(('test', 'nan', '12', 'nan'), cov, True)
# with covariance
lh = SimpleLikelihood(meadict, covdict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
arr_b = sim.to_global_data() # get global arr_b
diff = (np.mean(arr_b, axis=0) - arr_a)
(sign, logdet) = np.linalg.slogdet(arr_c*2.*np.pi)
baseline = -float(0.5)*float(np.vdot(diff, np.linalg.solve(arr_c, diff.T))+sign*logdet)
self.assertAlmostEqual(rslt, baseline)
def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if isinstance(data, Observable):
assert (data.shape[0] == 1)
self._archive.update({name: data}) # rw
elif isinstance(data, Field):
assert (data.shape[0] == 1)
self._archive.update(
{name: Observable(data.domain, data.local_data)}) # rw
# reading from numpy array takes information only on master node
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(int(1)), RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(int(1)), HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)}) # rw
else:
raise TypeError('unsupported data type')
log.debug('measurements-dict appends data %s' % str(name))
def append(self, name, data, plain=False):
"""
matrix coming in, numpy.matrix is not recommended
matrix must be distributed
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if plain:
assert (data.shape[1] == int(name[2]))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
if isinstance(data, Field):
assert (data.shape[0] == data.shape[1])
assert (len(data.domain) == 1) # single domain
self._archive.update({name: data}) # rw
elif isinstance(data, np.ndarray):
if data.shape[0] < data.shape[1] // mpisize + int(
2): # distributed
domain = DomainTuple.make(
RGSpace(shape=(data.shape[1], data.shape[1])))
self._archive.update(
{name: Field.from_local_data(domain, data)}) # rw
elif data.shape[0] == data.shape[1]: # non-distributed
domain = DomainTuple.make(RGSpace(shape=data.shape))
self._archive.update(
{name: Field.from_global_data(domain, data)}) # rw
else:
raise ValueError('unsupported data shape')
else:
raise TypeError('unsupported data type')
log.debug('covariances-dict appends data %s' % str(name))
def append(self, new_data):
"""
append new_data from all nodes
if the observable is not distributed, self.shape[0] % mpisize != 0,
it should be a measurement/observational data, in this case,
there is no need to append/rewrite new data,
nor it is convenient to do so
:param new_data: new data in type numpy array, NIFTy5.Field, Observable
:return:
since Field is read only, to append new data
we need to strip ndarray out, extend, then update Field
append also handle ._flag is True case
which means instead of append new data
we should rewrite
"""
assert(self._field.shape[0] % mpisize == 0) # non-distributed, do not append
# strip data
if isinstance(new_data, (Field, Observable)):
raw_new = new_data.local_data # return to each node
elif isinstance(new_data, np.ndarray):
raw_new = new_data
else:
raise TypeError('unsupported type')
# assemble new_cache
if self.rw_flag:
local_cache = raw_new
self.rw_flag = False # rw only once by default
else:
local_cache = np.vstack([self._field.local_data, raw_new])
# update new_cache to ._field, first need to get global_size
new_domain = DomainTuple.make((RGSpace(local_cache.shape[0]*mpisize), self._domain[1]))
self._field = Field.from_local_data(new_domain, local_cache)
def test_oas(self):
# mock observable
arr_a = np.random.rand(1, 4)
comm.Bcast(arr_a, root=0)
arr_ens = np.zeros((3, 4))
null_cov = np.zeros((4, 4))
# ensemble with identical realisations
for i in range(len(arr_ens)):
arr_ens[i] = arr_a
dtuple = DomainTuple.make((RGSpace(3 * mpisize), RGSpace(4)))
obs = Observable(dtuple, arr_ens)
test_mean, test_cov = oas_mcov(obs)
for i in range(len(arr_a)):
self.assertAlmostEqual(test_mean[0][i], arr_a[0][i])
for j in range(len(arr_a)):
self.assertAlmostEqual(test_cov[i][j], null_cov[i][j])
def apply_mask(self, mask_dict):
if mask_dict is None:
pass
else:
assert isinstance(mask_dict, Masks)
for name, msk in mask_dict._archive.items():
if name in self._archive.keys():
masked = mask_cov(self._archive[name].local_data,
msk.local_data)
remain = msk.local_data.sum()
# assemble masked pieces
if mpirank == 0:
for i in range(1, mpisize):
req = comm.irecv(source=i)
newslice = req.wait()
masked = np.vstack([masked, newslice])
else:
comm.isend(masked, dest=0)
masked = np.zeros((remain, remain))
comm.Bcast(masked, root=0)
new_name = (name[0], name[1], str(remain), name[3])
self._archive.pop(name, None) # pop out obsolete
domain = DomainTuple.make(RGSpace(shape=(remain, remain)))
self._archive.update(
{new_name: Field.from_global_data(domain, masked)})
def test_init(self):
# initialize observable
dtuple = DomainTuple.make((RGSpace(3 * mpisize), HPSpace(nside=2)))
val = np.random.rand(3, 48)
obs = Observable(dtuple, val)
# collect local val at master
val_master = None
if mpirank == 0:
val_master = np.zeros((3 * mpisize, 48))
comm.Gather(val, val_master, root=0)
# test domain/field shape
self.assertEqual(obs.domain, dtuple)
if mpirank == 0:
self.assertEqual(obs.shape, val_master.shape)
# test function to_global_data()
raw = obs.to_global_data()
if mpirank == 0:
for i in range(len(val_master)):
self.assertListEqual(list(raw[i]), list(val_master[i]))
# test function ensemble_mean
mean = obs.ensemble_mean
if mpirank == 0:
val_mean = np.mean(val_master, axis=0)
for i in range(val_mean.size):
self.assertAlmostEqual(mean[0][i], val_mean[i])
def test_covdict_append_field(self):
cov_field = Field.from_global_data(RGSpace(shape=(3, 3)),
np.random.rand(3, 3))
covdict = Covariances()
covdict.append(('test', 'nan', '3', 'nan'), cov_field,
True) # plain covariance
for i in range(len(cov_field.local_data)):
self.assertListEqual(
list((covdict[('test', 'nan', '3', 'nan')].local_data)[i]),
list((cov_field.local_data)[i]))
cov_field = Field.from_global_data(RGSpace(shape=(48, 48)),
np.random.rand(48, 48))
covdict.append(('test', 'nan', '2', 'nan'),
cov_field) # healpix covariance
for i in range(len(cov_field.local_data)):
self.assertListEqual(
list((covdict[('test', 'nan', '2', 'nan')].local_data)[i]),
list((cov_field.local_data)[i]))
def test_1dinit(self):
# initialize observable
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
val = np.random.rand(1, 48)
comm.Bcast(val, root=0)
obs = Observable(dtuple, val)
# matches val at master
raw = obs.to_global_data()
self.assertListEqual(list(raw[0]), list(val[0]))
def test_append_twice(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
val = np.random.rand(1, 48)
obs = Observable(dtuple, val)
# test function append with 1d array
new_data = np.random.rand(1, 48)
obs.append(new_data)
self.assertEqual(obs.shape, (2 * mpisize, 48))
obs.append(new_data)
self.assertEqual(obs.shape, (3 * mpisize, 48))
def test_append_after_replace(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
obs = Observable(dtuple)
self.assertTrue(obs.rw_flag)
# test function append with 1d array
new_data = np.random.rand(1, 48)
obs.append(new_data)
self.assertEqual(obs.shape, (1 * mpisize, 48))
obs.append(new_data)
self.assertEqual(obs.shape, (2 * mpisize, 48))
def test_append_with_replace(self):
dtuple = DomainTuple.make((RGSpace(1 * mpisize), HPSpace(nside=2)))
obs = Observable(dtuple)
self.assertTrue(obs.rw_flag)
# test with empty observable
dtuple = DomainTuple.make((RGSpace(8 * mpisize), HPSpace(nside=2)))
new_data = np.random.rand(8, 48)
new_obs = Observable(dtuple, new_data)
obs.append(new_obs)
raw_obs = obs.to_global_data()
# collect new val at master
new_val_master = None
if mpirank == 0:
new_val_master = np.zeros((8 * mpisize, 48))
comm.Gather(new_data, new_val_master, root=0)
# do comparison at master
if mpirank == 0:
for i in range(new_val_master.shape[0]):
self.assertListEqual(list(raw_obs[i]), list(new_val_master[i]))
def test_measuredict_append_observable(self):
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
hrr = np.random.rand(1, 48)
obs1 = Observable(dtuple, hrr)
measuredict = Measurements()
measuredict.append(('test', 'nan', '2', 'nan'),
obs1) # healpix Observable
if mpirank == 0:
self.assertListEqual(
list(measuredict[('test', 'nan', '2',
'nan')].to_global_data()[0]), list(hrr[0]))
dtuple = DomainTuple.make((RGSpace(1), RGSpace(3)))
arr = np.random.rand(1, 3)
obs2 = Observable(dtuple, arr)
measuredict.append(('test', 'nan', '3', 'nan'),
obs2) # plain Observable
if mpirank == 0:
self.assertListEqual(
list(measuredict[('test', 'nan', '3',
'nan')].to_global_data()[0]), list(arr[0]))
def test_append_observable(self):
dtuple = DomainTuple.make((RGSpace(3 * mpisize), HPSpace(nside=2)))
val = np.random.rand(3, 48)
obs = Observable(dtuple, val)
# test function append with Observable
dtuple = DomainTuple.make((RGSpace(5 * mpisize), HPSpace(nside=2)))
new_data = np.random.rand(5, 48)
new_obs = Observable(dtuple, new_data)
obs.append(new_obs)
raw_obs = obs.to_global_data()
new_val = np.vstack([val, new_data])
# collect new val at master
new_val_master = None
if mpirank == 0:
new_val_master = np.zeros((8 * mpisize, 48))
comm.Gather(new_val, new_val_master, root=0)
# do comparison at master
if mpirank == 0:
for i in range(new_val_master.shape[0]):
self.assertListEqual(list(raw_obs[i]), list(new_val_master[i]))
def append(self, name, data, plain=False):
"""
:param name:
:param data: distributed
:param plain:
:return:
"""
assert (len(name) == 4)
if isinstance(data, Observable):
assert (data.shape[0] == mpisize)
raw_data = data.local_data
for i in raw_data[0]:
assert (i == float(0) or i == float(1))
self._archive.update({name: data})
elif isinstance(data, Field):
assert (data.shape[0] == mpisize)
raw_data = data.local_data
for i in raw_data[0]:
assert (i == float(0) or i == float(1))
self._archive.update(
{name: Observable(data.domain, data.local_data)})
elif isinstance(data, np.ndarray):
assert (data.shape[0] == 1)
for i in data[0]:
assert (i == float(0) or i == float(1))
if plain:
assert (data.shape[1] == int(name[2]))
domain = DomainTuple.make(
(RGSpace(mpisize), RGSpace(data.shape[1])))
else:
assert (data.shape[1] == 12 * int(name[2]) * int(name[2]))
domain = DomainTuple.make(
(RGSpace(mpisize), HPSpace(nside=int(name[2]))))
self._archive.update({name: Observable(domain, data)})
else:
raise TypeError('unsupported data type')
log.debug('mask-dict appends data %s' % str(name))
def test_simdict_append_observable(self):
dtuple = DomainTuple.make((RGSpace(2 * mpisize), HPSpace(nside=2)))
hrr = np.random.rand(2, 48)
obs1 = Observable(dtuple, hrr)
simdict = Simulations()
simdict.append(('test', 'nan', '2', 'nan'), obs1) # healpix Observable
self.assertEqual(simdict[('test', 'nan', '2', 'nan')].shape,
(2 * mpisize, 48))
for i in range(len(hrr)):
self.assertListEqual(
list((simdict[('test', 'nan', '2', 'nan')].local_data)[i]),
list(hrr[i]))
dtuple = DomainTuple.make((RGSpace(5 * mpisize), RGSpace(3)))
arr = np.random.rand(5, 3)
obs2 = Observable(dtuple, arr)
simdict.append(('test', 'nan', '3', 'nan'), obs2,
True) # plain Observable
self.assertEqual(simdict[('test', 'nan', '3', 'nan')].shape,
(5 * mpisize, 3))
for i in range(len(arr)):
self.assertListEqual(
list((simdict[('test', 'nan', '3', 'nan')].local_data)[i]),
list(arr[i]))
def test_without_cov(self):
simdict = Simulations()
meadict = Measurements()
# mock measurements
dtuple = DomainTuple.make((RGSpace(1), HPSpace(nside=2)))
arr_a = np.random.rand(1, 48)
comm.Bcast(arr_a, root=0)
mea = Observable(dtuple, arr_a)
meadict.append(('test', 'nan', '2', 'nan'), mea)
# mock sims
dtuple = DomainTuple.make((RGSpace(3*mpisize), HPSpace(nside=2)))
arr_b = np.random.rand(3, 48)
sim = Observable(dtuple, arr_b)
simdict.append(('test', 'nan', '2', 'nan'), sim)
# no covariance
lh = SimpleLikelihood(meadict)
# calc by likelihood
rslt = lh(simdict) # feed variable value, not parameter value
# calc by hand
arr_b = sim.to_global_data() # global arr_b
diff = (np.mean(arr_b, axis=0) - arr_a)
baseline = -float(0.5)*float(np.vdot(diff, diff))
# comapre
self.assertAlmostEqual(rslt, baseline)
def test_covdict_apply_mask(self):
msk = np.array([0, 1, 0, 1, 1]).reshape(1, 5)
mskdict = Masks()
mskdict.append(('test', 'nan', '5', 'nan'), msk, True)
cov_field = Field.from_global_data(RGSpace(shape=(5, 5)),
np.random.rand(5, 5))
arr = cov_field.local_data
covdict = Covariances()
covdict.append(('test', 'nan', '5', 'nan'), arr, True)
covdict.apply_mask(mskdict)
arr = np.delete(cov_field.to_global_data(), [0, 2], 0)
arr = np.delete(arr, [0, 2], 1)
for i in range(arr.shape[0]):
self.assertListEqual(
list((covdict[('test', 'nan', '3',
'nan')].to_global_data())[i]), list(arr[i]))
def test_covdict_append_array(self):
cov = (Field.from_global_data(RGSpace(shape=(5, 5)),
np.random.rand(5, 5))).local_data
covdict = Covariances()
covdict.append(('test', 'nan', '5', 'nan'), cov,
True) # plain covariance
self.assertEqual(covdict[('test', 'nan', '5', 'nan')].shape, (5, 5))
for i in range(len(cov)):
self.assertListEqual(
list((covdict[('test', 'nan', '5', 'nan')].local_data)[i]),
list(cov[i]))
cov = np.random.rand(48, 48)
comm.Bcast(cov, root=0)
covdict.append(('test', 'nan', '2', 'nan'), cov) # healpix covariance
self.assertEqual(covdict[('test', 'nan', '2', 'nan')].shape, (48, 48))
for i in range(len(cov)):
self.assertListEqual(
list((covdict[('test', 'nan', '2',
'nan')].to_global_data())[i]), list(cov[i]))
def test_mask(self):
msk_arr = np.array([0., 1., 0., 1., 1., 0.]).reshape(1, 6)
obs_arr = np.random.rand(1, 6)
cov_arr = (Field.from_global_data(
DomainTuple.make(RGSpace(shape=(6, 6))),
np.random.rand(6, 6))).local_data
# mask by methods
test_obs = mask_obs(obs_arr, msk_arr)
test_cov = mask_cov(cov_arr, msk_arr)
# mask manually
fid_obs = np.hstack([obs_arr[0, 1], obs_arr[0, 3], obs_arr[0, 4]])
# gather global cov
fid_cov = np.zeros((6, 6))
comm.Gather(cov_arr, fid_cov, root=0)
# mask cov by hand
fid_cov = np.delete(fid_cov, [0, 2, 5], 0)
fid_cov = np.delete(fid_cov, [0, 2, 5], 1)
comm.Bcast(fid_cov, root=0)
# compare mask on matrix
for i in test_cov:
self.assertTrue(i in fid_cov)
# compare mask on array
self.assertListEqual(list(test_obs[0]), list(fid_obs))
def wmap_errfix():
#log.basicConfig(filename='imagine.log', level=log.DEBUG)
"""
only WMAP regular magnetic field model in test, @ 23GHz
Faraday rotation provided by YMW16 free electron model
full WMAP parameter set {b0, psi0, psi1, chi0}
"""
# hammurabi parameter base file
xmlpath = './params_fullsky_regular.xml'
# we take three active parameters
true_b0 = 6.0
true_psi0 = 27.0
true_psi1 = 0.9
true_chi0 = 25.
true_alpha = 3.0
true_r0 = 5.0
true_z0 = 1.0
truths = [
true_b0, true_psi0, true_psi1, true_chi0, true_alpha, true_r0, true_z0
]
mea_nside = 2 # observable Nside
mea_pix = 12 * mea_nside**2 # observable pixel number
"""
# step 1, prepare mock data
"""
x = np.zeros((1, mea_pix)) # only for triggering simulator
trigger = Measurements()
trigger.append(('sync', '23', str(mea_nside), 'I'), x) # only I map
# initialize simulator
error = 0.1 # theoretical raltive uncertainty for each (active) parameter
mocker = Hammurabi(measurements=trigger, xml_path=xmlpath)
# start simulation
# BregWMAP field
paramlist = {
'b0': true_b0,
'psi0': true_psi0,
'psi1': true_psi1,
'chi0': true_chi0
} # inactive parameters at default
breg_wmap = BregWMAP(paramlist, 1)
# CREAna field
paramlist = {
'alpha': true_alpha,
'beta': 0.0,
'theta': 0.0,
'r0': true_r0,
'z0': true_z0,
'E0': 20.6,
'j0': 0.0217
} # inactive parameters at default
cre_ana = CREAna(paramlist, 1)
# FEregYMW16 field
fereg_ymw16 = FEregYMW16(dict(), 1)
# collect mock data and covariance
outputs = mocker([breg_wmap, cre_ana, fereg_ymw16])
Imap = outputs[('sync', '23', str(mea_nside), 'I')].local_data
# collect mean and cov from simulated results
mock_data = Measurements()
mock_cov = Covariances()
mock_data.append(('sync', '23', str(mea_nside), 'I'), Imap)
mock_cov.append(('sync', '23', str(mea_nside), 'I'),
Field.from_global_data(RGSpace(shape=(mea_pix, mea_pix)),
(error**2 * (np.mean(Imap))**2) *
np.eye(mea_pix)))
"""
# step 2, prepare pipeline and execute analysis
"""
#likelihood = EnsembleLikelihood(mock_data, mock_cov)
likelihood = SimpleLikelihood(mock_data, mock_cov)
breg_factory = BregWMAPFactory(active_parameters=('b0', 'psi0', 'psi1',
'chi0'))
breg_factory.parameter_ranges = {
'b0': (0., 10.),
'psi0': (0., 50.),
'psi1': (0., 2.),
'chi0': (0., 50.)
}
cre_factory = CREAnaFactory(active_parameters=('alpha', 'r0', 'z0'))
cre_factory.parameter_ranges = {
'alpha': (1., 5.),
'r0': (1., 10.),
'z0': (0.1, 5.)
}
fereg_factory = FEregYMW16Factory()
factory_list = [breg_factory, cre_factory, fereg_factory]
prior = FlatPrior()
simer = Hammurabi(measurements=mock_data, xml_path=xmlpath)
ensemble_size = 1
pipe = MultinestPipeline(simer, factory_list, likelihood, prior,
ensemble_size)
pipe.random_type = 'free'
pipe.sampling_controllers = {
'n_live_points': 4000,
'resume': False,
'verbose': True
}
results = pipe()
"""
# step 3, visualize (with corner package)
"""
if mpirank == 0:
samples = results['samples']
np.savetxt('posterior_fullsky_regular_errfix.txt', samples)
"""