def test_crossover_prob_estimation(self):
"""Test that crossover probabilities are updated as expected when changing or not changing parameter locations and giving points that give a greater jump distance."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
dream = Dream(model=model, save_history=False)
starting_crossover = dream.CR_probabilities
crossover_probabilities = mp.Array('d', starting_crossover)
n = mp.Value('i', 0)
nCR = dream.nCR
CR_vals = dream.CR_values
ncrossover_updates = mp.Array('d', [0] * nCR)
current_position_arr = mp.Array(
'd', [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4])
dream.nchains = 5
delta_m = mp.Array('d', [0] * nCR)
dream.chain_n = 0
pydream.Dream_shared_vars.cross_probs = crossover_probabilities
pydream.Dream_shared_vars.count = n
pydream.Dream_shared_vars.ncr_updates = ncrossover_updates
pydream.Dream_shared_vars.current_positions = current_position_arr
pydream.Dream_shared_vars.delta_m = delta_m
q0 = np.array([1, 2, 3, 4])
q_new = np.array([1, 2, 3, 4])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[0])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
True)
q_new = np.array([1.2, 2.2, 3.3, 4.4])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[0])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
True)
q_new = np.array([2, 3, 4, 5])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[1])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
True)
q_new = np.array([11, -15, 20, 9])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[2])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
False)
self.assertGreater(new_cr_probs[2], starting_crossover[2])
self.assertAlmostEqual(np.sum(new_cr_probs), 1.0, places=1)
old_cr_probs = new_cr_probs
for i, q_new in zip(list(range(5)), [
np.array([15]),
np.array([17]),
np.array([19]),
np.array([21]),
np.array([23])
]):
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[1])
self.assertEqual(np.array_equal(new_cr_probs, old_cr_probs), False)
def test():
manager = processing.Manager()
gc.disable()
print('\n\t######## testing Queue.Queue\n')
test_queuespeed(threading.Thread, Queue.Queue(),
threading.Condition())
print('\n\t######## testing processing.Queue\n')
test_queuespeed(processing.Process, processing.Queue(),
processing.Condition())
print('\n\t######## testing Queue managed by server process\n')
test_queuespeed(processing.Process, manager.Queue(),
manager.Condition())
print('\n\t######## testing processing.Pipe\n')
test_pipespeed()
print
print('\n\t######## testing list\n')
test_seqspeed(range(10))
print('\n\t######## testing list managed by server process\n')
test_seqspeed(manager.list(range(10)))
print('\n\t######## testing Array("i", ..., lock=False)\n')
test_seqspeed(processing.Array('i', range(10), lock=False))
print('\n\t######## testing Array("i", ..., lock=True)\n')
test_seqspeed(processing.Array('i', range(10), lock=True))
print()
print('\n\t######## testing threading.Lock\n')
test_lockspeed(threading.Lock())
print('\n\t######## testing threading.RLock\n')
test_lockspeed(threading.RLock())
print('\n\t######## testing processing.Lock\n')
test_lockspeed(processing.Lock())
print('\n\t######## testing processing.RLock\n')
test_lockspeed(processing.RLock())
print('\n\t######## testing lock managed by server process\n')
test_lockspeed(manager.Lock())
print('\n\t######## testing rlock managed by server process\n')
test_lockspeed(manager.RLock())
print()
print('\n\t######## testing threading.Condition\n')
test_conditionspeed(threading.Thread, threading.Condition())
print('\n\t######## testing processing.Condition\n')
test_conditionspeed(processing.Process, processing.Condition())
print('\n\t######## testing condition managed by a server process\n')
test_conditionspeed(processing.Process, manager.Condition())
gc.enable()
def test_history_recording_multidim_model(self):
"""Test that history in memory matches with that recorded for test multi-dimensional model."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
dream = Dream(model=model, model_name='test_history_recording')
history_arr = mp.Array('d', [0] * 4 * dream.total_var_dimension * 3)
n = mp.Value('i', 0)
nchains = mp.Value('i', 3)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
pydream.Dream_shared_vars.nchains = nchains
test_history = np.array([[[1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8]],
[[7, 8, 9, 10], [9, 12, 18, 20],
[11, 14, 18, 8]],
[[13, 14, 18, 4], [15, 17, 11, 8],
[17, 28, 50, 4]],
[[19, 21, 1, 18], [21, 19, 19, 11],
[23, 4, 3, 2]]])
for chainpoint in test_history:
for point in chainpoint:
dream.record_history(nseedchains=0,
ndimensions=dream.total_var_dimension,
q_new=point,
len_history=len(history_arr))
history_arr_np = np.frombuffer(
pydream.Dream_shared_vars.history.get_obj())
history_arr_np_reshaped = history_arr_np.reshape(
np.shape(test_history))
self.assertIs(np.array_equal(history_arr_np_reshaped, test_history),
True)
remove('test_history_recording_DREAM_chain_history.npy')
remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy')
remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy')
def test_history_recording_simple_model(self):
"""Test that history in memory matches with that recorded for test one-dimensional model."""
self.param, self.like = onedmodel()
model = Model(self.like, self.param)
step = Dream(model=model, model_name='test_history_recording')
history_arr = mp.Array('d', [0] * 4 * step.total_var_dimension)
n = mp.Value('i', 0)
nchains = mp.Value('i', 3)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
pydream.Dream_shared_vars.nchains = nchains
test_history = np.array([[1], [3], [5], [7]])
for chainpoint in test_history:
for point in chainpoint:
step.record_history(nseedchains=0,
ndimensions=step.total_var_dimension,
q_new=point,
len_history=len(history_arr))
history_arr_np = np.frombuffer(
pydream.Dream_shared_vars.history.get_obj())
history_arr_np_reshaped = history_arr_np.reshape(
np.shape(test_history))
self.assertIs(np.array_equal(history_arr_np_reshaped, test_history),
True)
remove('test_history_recording_DREAM_chain_history.npy')
remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy')
remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy')
def test_proposal_generation_snooker(self):
"""Test that proposal generation with a snooker update returns values of the expected shape."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
step = Dream(model=model)
history_arr = mp.Array('d', list(range(120)))
n = mp.Value('i', 0)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
step.nseedchains = 20
q0 = np.array([2, 3, 4, 5])
proposed_pt, snooker_logp, z = step.generate_proposal_points(
n_proposed_pts=1,
q0=q0,
CR=1,
DEpairs=1,
gamma_level=1,
snooker=True)
self.assertEqual(len(proposed_pt), step.total_var_dimension)
proposed_pts, snooker_logp, z = step.generate_proposal_points(
n_proposed_pts=5,
q0=q0,
CR=1,
DEpairs=1,
gamma_level=1,
snooker=True)
self.assertEqual(len(proposed_pts), 5)
def test_chain_sampling_multidim_model(self):
"""Test that sampling from DREAM history for multi-dimensional model when the history is known matches with expected possible samples."""
self.params, self.like = multidmodel()
model = Model(likelihood=self.like, sampled_parameters=self.params)
dream = Dream(model=model)
history_arr = mp.Array('d', [0] * 2 * dream.total_var_dimension)
n = mp.Value('i', 0)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
chains_added_to_history = []
for i in range(2):
start = i * dream.total_var_dimension
end = start + dream.total_var_dimension
chain = dream.draw_from_prior(model.sampled_parameters)
pydream.Dream_shared_vars.history[start:end] = chain
chains_added_to_history.append(chain)
sampled_chains = dream.sample_from_history(
nseedchains=2, DEpairs=1, ndimensions=dream.total_var_dimension)
sampled_chains = np.array(sampled_chains)
chains_added_to_history = np.array(chains_added_to_history)
self.assertIs(
np.array_equal(
chains_added_to_history[chains_added_to_history[:,
0].argsort()],
sampled_chains[sampled_chains[:, 0].argsort()]), True)
def __init__(self, device, ctx=mp.get_context("spawn")):
""" Constructor method
"""
self.device = device
self.ctx = ctx
res = self.device.im_size
self.frame_shared = mp.Array(ctypes.c_uint8, res[1] * res[0] * 3)
self.frame = np.frombuffer(self.frame_shared.get_obj(),
dtype="uint8").reshape(res[1], res[0], 3)
self.frame_time_shared = mp.Array(ctypes.c_double, 1)
self.frame_time = np.frombuffer(self.frame_time_shared.get_obj(),
dtype="d")
self.q_to_process = ClearableMPQueue(ctx=self.ctx)
self.q_from_process = ClearableMPQueue(ctx=self.ctx)
self.write_frame_queue = ClearableMPQueue(ctx=self.ctx)
self.capture_process = None
self.writer_process = None
def test_sharedvalues():
values = [('i', 10), ('h', -2), ('d', 1.25)]
arrays = [('i', range(100)), ('d', [0.25 * i for i in range(100)]),
('H', range(1000))]
shared_values = [processing.Value(id, v) for id, v in values]
shared_arrays = [processing.Array(id, a) for id, a in arrays]
p = processing.Process(target=sharedvalues_func,
args=(values, arrays, shared_values, shared_arrays))
p.start()
p.join()
assert p.exitcode == 0
def test_proposal_generation_nosnooker_CR66(self):
"""Test proposal generation without a snooker update with a single or multiple proposed points and a crossover value of 2/3 gives 2/3 of all dimensions changed on average as expected."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
step = Dream(model=model)
history_arr = mp.Array('d', list(range(120)))
n = mp.Value('i', 0)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
step.nseedchains = 20
q0 = np.array([2, 3, 4, 5])
dims_kept = 0
for iteration in range(100000):
proposed_pt = step.generate_proposal_points(n_proposed_pts=1,
q0=q0,
CR=.66,
DEpairs=1,
gamma_level=1,
snooker=False)
if iteration == 1:
self.assertEqual(len(proposed_pt), 1)
dims_change_vec = np.squeeze(q0 == proposed_pt)
for dim in dims_change_vec:
if dim:
dims_kept += 1
frac_kept = dims_kept / (step.total_var_dimension * 100000.0)
self.assertAlmostEqual(frac_kept, 1 - .66, places=1)
dims_kept = 0
for iteration in range(10000):
proposed_pts = step.generate_proposal_points(n_proposed_pts=5,
q0=q0,
CR=.66,
DEpairs=1,
gamma_level=1,
snooker=False)
if iteration == 1:
self.assertEqual(len(proposed_pts), 5)
for pt in proposed_pts:
dims_change_vec = (q0 == pt)
for dim in dims_change_vec:
if dim:
dims_kept += 1
frac_kept = dims_kept / (step.total_var_dimension * 10000.0 * 5)
self.assertAlmostEqual(frac_kept, 1 - .66, places=1)
def _setup_mp_dream_pool(nchains, niterations, step_instance, start_pt=None):
min_njobs = (2 * len(step_instance.DEpairs)) + 1
if nchains < min_njobs:
raise Exception(
'Dream should be run with at least (2*DEpairs)+1 number of chains. For current algorithmic settings, set njobs>=%s.'
% str(min_njobs))
if step_instance.history_file != False:
old_history = np.load(step_instance.history_file)
len_old_history = len(old_history.flatten())
nold_history_records = len_old_history / step_instance.total_var_dimension
step_instance.nseedchains = nold_history_records
if niterations < step_instance.history_thin:
arr_dim = (
(np.floor(nchains * niterations / step_instance.history_thin) +
nchains) *
step_instance.total_var_dimension) + len_old_history
else:
arr_dim = np.floor(((
(nchains * niterations) * step_instance.total_var_dimension) /
step_instance.history_thin)) + len_old_history
else:
if niterations < step_instance.history_thin:
arr_dim = (
(np.floor(nchains * niterations / step_instance.history_thin) +
nchains) * step_instance.total_var_dimension
) + (step_instance.nseedchains * step_instance.total_var_dimension)
else:
arr_dim = np.floor(
((nchains * niterations / step_instance.history_thin) *
step_instance.total_var_dimension)
) + (step_instance.nseedchains * step_instance.total_var_dimension)
min_nseedchains = 2 * len(step_instance.DEpairs) * nchains
if step_instance.nseedchains < min_nseedchains:
raise Exception(
'The size of the seeded starting history is insufficient. Increase nseedchains>=%s.'
% str(min_nseedchains))
current_position_dim = nchains * step_instance.total_var_dimension
history_arr = mp.Array('d', [0] * int(arr_dim))
if step_instance.history_file != False:
history_arr[0:len_old_history] = old_history.flatten()
nCR = step_instance.nCR
ngamma = step_instance.ngamma
crossover_setting = step_instance.CR_probabilities
crossover_probabilities = mp.Array('d', crossover_setting)
ncrossover_updates = mp.Array('d', [0] * nCR)
delta_m = mp.Array('d', [0] * nCR)
gamma_level_setting = step_instance.gamma_probabilities
gamma_probabilities = mp.Array('d', gamma_level_setting)
ngamma_updates = mp.Array('d', [0] * ngamma)
delta_m_gamma = mp.Array('d', [0] * ngamma)
current_position_arr = mp.Array('d', [0] * current_position_dim)
shared_nchains = mp.Value('i', nchains)
n = mp.Value('i', 0)
tf = mp.Value('c', b'F')
if step_instance.crossover_burnin == None:
step_instance.crossover_burnin = int(np.floor(niterations / 10))
if start_pt != None:
if step_instance.start_random:
print(
'Warning: start position provided but random_start set to True. Overrode random_start value and starting walk at provided start position.'
)
step_instance.start_random = False
p = DreamPool(nchains,
initializer=_mp_dream_init,
initargs=(
history_arr,
current_position_arr,
shared_nchains,
crossover_probabilities,
ncrossover_updates,
delta_m,
gamma_probabilities,
ngamma_updates,
delta_m_gamma,
n,
tf,
))
#p = mp.pool.ThreadPool(nchains, initializer=_mp_dream_init, initargs=(history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, ))
#p = mp.Pool(nchains, initializer=_mp_dream_init, initargs=(history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, ))
return p
