def test_crossover_prob_estimation(self): """Test that crossover probabilities are updated as expected when changing or not changing parameter locations and giving points that give a greater jump distance.""" self.param, self.like = multidmodel() model = Model(self.like, self.param) dream = Dream(model=model, save_history=False) starting_crossover = dream.CR_probabilities crossover_probabilities = mp.Array('d', starting_crossover) n = mp.Value('i', 0) nCR = dream.nCR CR_vals = dream.CR_values ncrossover_updates = mp.Array('d', [0] * nCR) current_position_arr = mp.Array( 'd', [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4]) dream.nchains = 5 delta_m = mp.Array('d', [0] * nCR) dream.chain_n = 0 pydream.Dream_shared_vars.cross_probs = crossover_probabilities pydream.Dream_shared_vars.count = n pydream.Dream_shared_vars.ncr_updates = ncrossover_updates pydream.Dream_shared_vars.current_positions = current_position_arr pydream.Dream_shared_vars.delta_m = delta_m q0 = np.array([1, 2, 3, 4]) q_new = np.array([1, 2, 3, 4]) new_cr_probs = dream.estimate_crossover_probabilities( dream.total_var_dimension, q0, q_new, CR_vals[0]) self.assertEqual(np.array_equal(new_cr_probs, starting_crossover), True) q_new = np.array([1.2, 2.2, 3.3, 4.4]) new_cr_probs = dream.estimate_crossover_probabilities( dream.total_var_dimension, q0, q_new, CR_vals[0]) self.assertEqual(np.array_equal(new_cr_probs, starting_crossover), True) q_new = np.array([2, 3, 4, 5]) new_cr_probs = dream.estimate_crossover_probabilities( dream.total_var_dimension, q0, q_new, CR_vals[1]) self.assertEqual(np.array_equal(new_cr_probs, starting_crossover), True) q_new = np.array([11, -15, 20, 9]) new_cr_probs = dream.estimate_crossover_probabilities( dream.total_var_dimension, q0, q_new, CR_vals[2]) self.assertEqual(np.array_equal(new_cr_probs, starting_crossover), False) self.assertGreater(new_cr_probs[2], starting_crossover[2]) self.assertAlmostEqual(np.sum(new_cr_probs), 1.0, places=1) old_cr_probs = new_cr_probs for i, q_new in zip(list(range(5)), [ np.array([15]), np.array([17]), np.array([19]), np.array([21]), np.array([23]) ]): new_cr_probs = dream.estimate_crossover_probabilities( dream.total_var_dimension, q0, q_new, CR_vals[1]) self.assertEqual(np.array_equal(new_cr_probs, old_cr_probs), False)
def test(): manager = processing.Manager() gc.disable() print('\n\t######## testing Queue.Queue\n') test_queuespeed(threading.Thread, Queue.Queue(), threading.Condition()) print('\n\t######## testing processing.Queue\n') test_queuespeed(processing.Process, processing.Queue(), processing.Condition()) print('\n\t######## testing Queue managed by server process\n') test_queuespeed(processing.Process, manager.Queue(), manager.Condition()) print('\n\t######## testing processing.Pipe\n') test_pipespeed() print print('\n\t######## testing list\n') test_seqspeed(range(10)) print('\n\t######## testing list managed by server process\n') test_seqspeed(manager.list(range(10))) print('\n\t######## testing Array("i", ..., lock=False)\n') test_seqspeed(processing.Array('i', range(10), lock=False)) print('\n\t######## testing Array("i", ..., lock=True)\n') test_seqspeed(processing.Array('i', range(10), lock=True)) print() print('\n\t######## testing threading.Lock\n') test_lockspeed(threading.Lock()) print('\n\t######## testing threading.RLock\n') test_lockspeed(threading.RLock()) print('\n\t######## testing processing.Lock\n') test_lockspeed(processing.Lock()) print('\n\t######## testing processing.RLock\n') test_lockspeed(processing.RLock()) print('\n\t######## testing lock managed by server process\n') test_lockspeed(manager.Lock()) print('\n\t######## testing rlock managed by server process\n') test_lockspeed(manager.RLock()) print() print('\n\t######## testing threading.Condition\n') test_conditionspeed(threading.Thread, threading.Condition()) print('\n\t######## testing processing.Condition\n') test_conditionspeed(processing.Process, processing.Condition()) print('\n\t######## testing condition managed by a server process\n') test_conditionspeed(processing.Process, manager.Condition()) gc.enable()
def test_history_recording_multidim_model(self): """Test that history in memory matches with that recorded for test multi-dimensional model.""" self.param, self.like = multidmodel() model = Model(self.like, self.param) dream = Dream(model=model, model_name='test_history_recording') history_arr = mp.Array('d', [0] * 4 * dream.total_var_dimension * 3) n = mp.Value('i', 0) nchains = mp.Value('i', 3) pydream.Dream_shared_vars.history = history_arr pydream.Dream_shared_vars.count = n pydream.Dream_shared_vars.nchains = nchains test_history = np.array([[[1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8]], [[7, 8, 9, 10], [9, 12, 18, 20], [11, 14, 18, 8]], [[13, 14, 18, 4], [15, 17, 11, 8], [17, 28, 50, 4]], [[19, 21, 1, 18], [21, 19, 19, 11], [23, 4, 3, 2]]]) for chainpoint in test_history: for point in chainpoint: dream.record_history(nseedchains=0, ndimensions=dream.total_var_dimension, q_new=point, len_history=len(history_arr)) history_arr_np = np.frombuffer( pydream.Dream_shared_vars.history.get_obj()) history_arr_np_reshaped = history_arr_np.reshape( np.shape(test_history)) self.assertIs(np.array_equal(history_arr_np_reshaped, test_history), True) remove('test_history_recording_DREAM_chain_history.npy') remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy') remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy')
def test_history_recording_simple_model(self): """Test that history in memory matches with that recorded for test one-dimensional model.""" self.param, self.like = onedmodel() model = Model(self.like, self.param) step = Dream(model=model, model_name='test_history_recording') history_arr = mp.Array('d', [0] * 4 * step.total_var_dimension) n = mp.Value('i', 0) nchains = mp.Value('i', 3) pydream.Dream_shared_vars.history = history_arr pydream.Dream_shared_vars.count = n pydream.Dream_shared_vars.nchains = nchains test_history = np.array([[1], [3], [5], [7]]) for chainpoint in test_history: for point in chainpoint: step.record_history(nseedchains=0, ndimensions=step.total_var_dimension, q_new=point, len_history=len(history_arr)) history_arr_np = np.frombuffer( pydream.Dream_shared_vars.history.get_obj()) history_arr_np_reshaped = history_arr_np.reshape( np.shape(test_history)) self.assertIs(np.array_equal(history_arr_np_reshaped, test_history), True) remove('test_history_recording_DREAM_chain_history.npy') remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy') remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy')
def test_proposal_generation_snooker(self): """Test that proposal generation with a snooker update returns values of the expected shape.""" self.param, self.like = multidmodel() model = Model(self.like, self.param) step = Dream(model=model) history_arr = mp.Array('d', list(range(120))) n = mp.Value('i', 0) pydream.Dream_shared_vars.history = history_arr pydream.Dream_shared_vars.count = n step.nseedchains = 20 q0 = np.array([2, 3, 4, 5]) proposed_pt, snooker_logp, z = step.generate_proposal_points( n_proposed_pts=1, q0=q0, CR=1, DEpairs=1, gamma_level=1, snooker=True) self.assertEqual(len(proposed_pt), step.total_var_dimension) proposed_pts, snooker_logp, z = step.generate_proposal_points( n_proposed_pts=5, q0=q0, CR=1, DEpairs=1, gamma_level=1, snooker=True) self.assertEqual(len(proposed_pts), 5)
def test_chain_sampling_multidim_model(self): """Test that sampling from DREAM history for multi-dimensional model when the history is known matches with expected possible samples.""" self.params, self.like = multidmodel() model = Model(likelihood=self.like, sampled_parameters=self.params) dream = Dream(model=model) history_arr = mp.Array('d', [0] * 2 * dream.total_var_dimension) n = mp.Value('i', 0) pydream.Dream_shared_vars.history = history_arr pydream.Dream_shared_vars.count = n chains_added_to_history = [] for i in range(2): start = i * dream.total_var_dimension end = start + dream.total_var_dimension chain = dream.draw_from_prior(model.sampled_parameters) pydream.Dream_shared_vars.history[start:end] = chain chains_added_to_history.append(chain) sampled_chains = dream.sample_from_history( nseedchains=2, DEpairs=1, ndimensions=dream.total_var_dimension) sampled_chains = np.array(sampled_chains) chains_added_to_history = np.array(chains_added_to_history) self.assertIs( np.array_equal( chains_added_to_history[chains_added_to_history[:, 0].argsort()], sampled_chains[sampled_chains[:, 0].argsort()]), True)
def __init__(self, device, ctx=mp.get_context("spawn")): """ Constructor method """ self.device = device self.ctx = ctx res = self.device.im_size self.frame_shared = mp.Array(ctypes.c_uint8, res[1] * res[0] * 3) self.frame = np.frombuffer(self.frame_shared.get_obj(), dtype="uint8").reshape(res[1], res[0], 3) self.frame_time_shared = mp.Array(ctypes.c_double, 1) self.frame_time = np.frombuffer(self.frame_time_shared.get_obj(), dtype="d") self.q_to_process = ClearableMPQueue(ctx=self.ctx) self.q_from_process = ClearableMPQueue(ctx=self.ctx) self.write_frame_queue = ClearableMPQueue(ctx=self.ctx) self.capture_process = None self.writer_process = None
def test_sharedvalues(): values = [('i', 10), ('h', -2), ('d', 1.25)] arrays = [('i', range(100)), ('d', [0.25 * i for i in range(100)]), ('H', range(1000))] shared_values = [processing.Value(id, v) for id, v in values] shared_arrays = [processing.Array(id, a) for id, a in arrays] p = processing.Process(target=sharedvalues_func, args=(values, arrays, shared_values, shared_arrays)) p.start() p.join() assert p.exitcode == 0
def test_proposal_generation_nosnooker_CR66(self): """Test proposal generation without a snooker update with a single or multiple proposed points and a crossover value of 2/3 gives 2/3 of all dimensions changed on average as expected.""" self.param, self.like = multidmodel() model = Model(self.like, self.param) step = Dream(model=model) history_arr = mp.Array('d', list(range(120))) n = mp.Value('i', 0) pydream.Dream_shared_vars.history = history_arr pydream.Dream_shared_vars.count = n step.nseedchains = 20 q0 = np.array([2, 3, 4, 5]) dims_kept = 0 for iteration in range(100000): proposed_pt = step.generate_proposal_points(n_proposed_pts=1, q0=q0, CR=.66, DEpairs=1, gamma_level=1, snooker=False) if iteration == 1: self.assertEqual(len(proposed_pt), 1) dims_change_vec = np.squeeze(q0 == proposed_pt) for dim in dims_change_vec: if dim: dims_kept += 1 frac_kept = dims_kept / (step.total_var_dimension * 100000.0) self.assertAlmostEqual(frac_kept, 1 - .66, places=1) dims_kept = 0 for iteration in range(10000): proposed_pts = step.generate_proposal_points(n_proposed_pts=5, q0=q0, CR=.66, DEpairs=1, gamma_level=1, snooker=False) if iteration == 1: self.assertEqual(len(proposed_pts), 5) for pt in proposed_pts: dims_change_vec = (q0 == pt) for dim in dims_change_vec: if dim: dims_kept += 1 frac_kept = dims_kept / (step.total_var_dimension * 10000.0 * 5) self.assertAlmostEqual(frac_kept, 1 - .66, places=1)
def _setup_mp_dream_pool(nchains, niterations, step_instance, start_pt=None): min_njobs = (2 * len(step_instance.DEpairs)) + 1 if nchains < min_njobs: raise Exception( 'Dream should be run with at least (2*DEpairs)+1 number of chains. For current algorithmic settings, set njobs>=%s.' % str(min_njobs)) if step_instance.history_file != False: old_history = np.load(step_instance.history_file) len_old_history = len(old_history.flatten()) nold_history_records = len_old_history / step_instance.total_var_dimension step_instance.nseedchains = nold_history_records if niterations < step_instance.history_thin: arr_dim = ( (np.floor(nchains * niterations / step_instance.history_thin) + nchains) * step_instance.total_var_dimension) + len_old_history else: arr_dim = np.floor((( (nchains * niterations) * step_instance.total_var_dimension) / step_instance.history_thin)) + len_old_history else: if niterations < step_instance.history_thin: arr_dim = ( (np.floor(nchains * niterations / step_instance.history_thin) + nchains) * step_instance.total_var_dimension ) + (step_instance.nseedchains * step_instance.total_var_dimension) else: arr_dim = np.floor( ((nchains * niterations / step_instance.history_thin) * step_instance.total_var_dimension) ) + (step_instance.nseedchains * step_instance.total_var_dimension) min_nseedchains = 2 * len(step_instance.DEpairs) * nchains if step_instance.nseedchains < min_nseedchains: raise Exception( 'The size of the seeded starting history is insufficient. Increase nseedchains>=%s.' % str(min_nseedchains)) current_position_dim = nchains * step_instance.total_var_dimension history_arr = mp.Array('d', [0] * int(arr_dim)) if step_instance.history_file != False: history_arr[0:len_old_history] = old_history.flatten() nCR = step_instance.nCR ngamma = step_instance.ngamma crossover_setting = step_instance.CR_probabilities crossover_probabilities = mp.Array('d', crossover_setting) ncrossover_updates = mp.Array('d', [0] * nCR) delta_m = mp.Array('d', [0] * nCR) gamma_level_setting = step_instance.gamma_probabilities gamma_probabilities = mp.Array('d', gamma_level_setting) ngamma_updates = mp.Array('d', [0] * ngamma) delta_m_gamma = mp.Array('d', [0] * ngamma) current_position_arr = mp.Array('d', [0] * current_position_dim) shared_nchains = mp.Value('i', nchains) n = mp.Value('i', 0) tf = mp.Value('c', b'F') if step_instance.crossover_burnin == None: step_instance.crossover_burnin = int(np.floor(niterations / 10)) if start_pt != None: if step_instance.start_random: print( 'Warning: start position provided but random_start set to True. Overrode random_start value and starting walk at provided start position.' ) step_instance.start_random = False p = DreamPool(nchains, initializer=_mp_dream_init, initargs=( history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, )) #p = mp.pool.ThreadPool(nchains, initializer=_mp_dream_init, initargs=(history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, )) #p = mp.Pool(nchains, initializer=_mp_dream_init, initargs=(history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, )) return p