def ensemble_mean(self):
log.debug('@ observable::ensemble_mean')
if (self._dtype == 'measured'):
assert (self._data.shape[0] == 1) # single realization
return self._data # since each node has a full copy
elif (self._dtype == 'simulated'):
return mpi_mean(self._data) # calculate mean from all nodes
else:
raise TypeError('unsupported data type')
def pmean(data):
"""
:py:func:`imagine.tools.mpi_helper.mpi_mean` or :py:func:`numpy.mean`
depending on :py:data:`imagine.rc['distributed_arrays']`.
"""
if rc['distributed_arrays']:
return m.mpi_mean(data)
else:
return (np.mean(data, axis=0)).reshape(1, -1)
def test_mean(self):
if not mpirank:
arr = np.random.rand(2, 128)
else:
arr = np.random.rand(1, 128)
full_arr = np.vstack(comm.allgather(arr))
test_arr = (np.mean(full_arr, axis=0)).reshape(1, -1)
test_mean = mpi_mean(arr)
# check if almost equal since we forced the array datatype into numpy.float64
for i in range(len(test_mean[0])):
self.assertAlmostEqual(test_mean[0][i], test_arr[0][i])
def mpi_mean_timing(ensemble_size, data_size):
local_ensemble_size = mpi_arrange(ensemble_size)[1] - mpi_arrange(
ensemble_size)[0]
random_data = np.random.rand(local_ensemble_size, data_size)
tmr = Timer()
tmr.tick('mpi_mean')
ensemble_mean = mpi_mean(random_data)
tmr.tock('mpi_mean')
if not mpirank:
print('@ tools_profiles::mpi_mean_timing with ' + str(mpisize) +
' nodes')
print('global array shape (' + str(ensemble_size) + ',' +
str(data_size) + ')')
print('elapse time ' + str(tmr.record['mpi_mean']) + '\n')
def oas_mcov(data):
"""
Estimate covariance with the Oracle Approximating Shrinkage algorithm.
See `imagine.tools.covariance_estimator.oas_cov` for details. This
function aditionally returns the computed ensemble mean.
Parameters
----------
data : numpy.ndarray
distributed data in global shape (ensemble_size, data_size)
Returns
-------
mean : numpy.ndarray
copied ensemble mean (on all nodes)
cov : numpy.ndarray
distributed covariance matrix in shape (data_size, data_size)
"""
log.debug('@ covariance_estimator::oas_mcov')
assert isinstance(data, np.ndarray)
assert (len(data.shape) == 2)
# Finds ensemble size and data size
data_size = data.shape[1]
ensemble_size = np.array(0, dtype=np.uint)
comm.Allreduce([np.array(data.shape[0], dtype=np.uint), MPI.LONG],
[ensemble_size, MPI.LONG],
op=MPI.SUM)
# Calculates OAS covariance extimator from empirical covariance estimator
mean = mpi_mean(data)
u = data - mean
s = mpi_mult(mpi_trans(u), u) / ensemble_size
trs = mpi_trace(s)
trs2 = mpi_trace(mpi_mult(s, s))
numerator = (1.0 - 2.0 / data_size) * trs2 + trs * trs
denominator = (ensemble_size + 1.0 -
2.0 / data_size) * (trs2 - (trs * trs) / data_size)
if denominator == 0:
rho = 1
else:
rho = np.min([1, numerator / denominator])
cov = (1. - rho) * s + mpi_eye(data_size) * rho * trs / data_size
return mean, cov
def empirical_cov(data):
r"""
Empirical covariance estimator
Given some data matrix, :math:`D`, where rows are different samples
and columns different properties, the covariance can be
estimated from
.. math::
U_{ij} = D_{ij} - \overline{D}_j\,,\;
\text{with}\; \overline{D}_j=\tfrac{1}{N} \sum_{i=1}^N D_{ij}
.. math::
\text{cov} = \tfrac{1}{N} U^T U
Notes
-----
While conceptually simple, this is usually not the
best option.
Parameters
----------
data : numpy.ndarray
ensemble of observables, in global shape (ensemble size, data size)
Returns
-------
cov : numpy.ndarray
distributed (not copied) covariance matrix in global shape
(data size, data size) each node takes part of the rows
"""
log.debug('@ covariance_estimator::empirical_cov')
assert isinstance(data, np.ndarray)
assert (len(data.shape) == 2)
# Get ensemble size (i.e. the number of rows)
ensemble_size = np.array(0, dtype=np.uint)
comm.Allreduce([np.array(data.shape[0], dtype=np.uint), MPI.LONG],
[ensemble_size, MPI.LONG],
op=MPI.SUM)
# Calculates covariance
u = data - mpi_mean(data)
cov = mpi_mult(mpi_trans(u), u) / ensemble_size
return cov
def lsa_errfix():
#log.basicConfig(filename='imagine.log', level=log.DEBUG)
"""
only LSA regular magnetic field model in test, @ 23GHz
Faraday rotation provided by YMW16 thermal electron model
full LSA parameter set {b0, psi0, psi1, chi0}
"""
# hammurabi parameter base file
xmlpath = './params.xml'
# we take three active parameters
true_b0 = 6.0
true_psi0 = 27.0
true_psi1 = 0.9
true_chi0 = 25.
true_alpha = 3.0
true_r0 = 5.0
true_z0 = 1.0
mea_nside = 2 # observable Nside
mea_pix = 12 * mea_nside**2 # observable pixel number
"""
# step 1, prepare mock data
"""
x = np.zeros((1, mea_pix)) # only for triggering simulator
trigger = Measurements()
trigger.append(('sync', '23', str(mea_nside), 'I'), x) # only I map
# initialize simulator
error = 0.1 # theoretical raltive uncertainty for each (active) parameter
mocker = Hammurabi(measurements=trigger, xml_path=xmlpath)
# start simulation
# BregLSA field
paramlist = {
'b0': true_b0,
'psi0': true_psi0,
'psi1': true_psi1,
'chi0': true_chi0
} # inactive parameters at default
breg_lsa = BregLSA(paramlist, 1)
# CREAna field
paramlist = {
'alpha': true_alpha,
'beta': 0.0,
'theta': 0.0,
'r0': true_r0,
'z0': true_z0,
'E0': 20.6,
'j0': 0.0217
} # inactive parameters at default
cre_ana = CREAna(paramlist, 1)
# TEregYMW16 field
tereg_ymw16 = TEregYMW16(dict(), 1)
# collect mock data and covariance
outputs = mocker([breg_lsa, cre_ana, tereg_ymw16])
Imap = outputs[('sync', '23', str(mea_nside), 'I')].local_data
# collect mean and cov from simulated results
mock_data = Measurements()
mock_cov = Covariances()
mock_data.append(('sync', '23', str(mea_nside), 'I'), Imap)
mock_cov.append(('sync', '23', str(mea_nside), 'I'),
(error**2 * (mpi_mean(Imap))**2) * mpi_eye(mea_pix))
"""
# step 2, prepare pipeline and execute analysis
"""
likelihood = EnsembleLikelihood(mock_data, mock_cov)
breg_factory = BregLSAFactory(active_parameters=('b0', 'psi0', 'psi1',
'chi0'))
breg_factory.parameter_ranges = {
'b0': (0., 10.),
'psi0': (0., 50.),
'psi1': (0., 2.),
'chi0': (0., 50.)
}
cre_factory = CREAnaFactory(active_parameters=('alpha', 'r0', 'z0'))
cre_factory.parameter_ranges = {
'alpha': (1., 5.),
'r0': (1., 10.),
'z0': (0.1, 5.)
}
tereg_factory = TEregYMW16Factory()
factory_list = [breg_factory, cre_factory, tereg_factory]
prior = FlatPrior()
simer = Hammurabi(measurements=mock_data, xml_path=xmlpath)
ensemble_size = 10
pipe = DynestyPipeline(simer, factory_list, likelihood, prior,
ensemble_size)
pipe.random_type = 'free'
pipe.sampling_controllers = {'nlive': 4000}
results = pipe()
"""
# step 3, visualize (with corner package)
"""
if mpirank == 0:
samples = results['samples']
np.savetxt('posterior_fullsky_regular_errfix.txt', samples)
"""