def test_history_recording_simple_model(self):
"""Test that history in memory matches with that recorded for test one-dimensional model."""
self.param, self.like = onedmodel()
model = Model(self.like, self.param)
step = Dream(model=model, model_name='test_history_recording')
history_arr = mp.Array('d', [0] * 4 * step.total_var_dimension)
n = mp.Value('i', 0)
nchains = mp.Value('i', 3)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
pydream.Dream_shared_vars.nchains = nchains
test_history = np.array([[1], [3], [5], [7]])
for chainpoint in test_history:
for point in chainpoint:
step.record_history(nseedchains=0,
ndimensions=step.total_var_dimension,
q_new=point,
len_history=len(history_arr))
history_arr_np = np.frombuffer(
pydream.Dream_shared_vars.history.get_obj())
history_arr_np_reshaped = history_arr_np.reshape(
np.shape(test_history))
self.assertIs(np.array_equal(history_arr_np_reshaped, test_history),
True)
remove('test_history_recording_DREAM_chain_history.npy')
remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy')
remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy')
def test_history_recording_multidim_model(self):
"""Test that history in memory matches with that recorded for test multi-dimensional model."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
dream = Dream(model=model, model_name='test_history_recording')
history_arr = mp.Array('d', [0] * 4 * dream.total_var_dimension * 3)
n = mp.Value('i', 0)
nchains = mp.Value('i', 3)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
pydream.Dream_shared_vars.nchains = nchains
test_history = np.array([[[1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8]],
[[7, 8, 9, 10], [9, 12, 18, 20],
[11, 14, 18, 8]],
[[13, 14, 18, 4], [15, 17, 11, 8],
[17, 28, 50, 4]],
[[19, 21, 1, 18], [21, 19, 19, 11],
[23, 4, 3, 2]]])
for chainpoint in test_history:
for point in chainpoint:
dream.record_history(nseedchains=0,
ndimensions=dream.total_var_dimension,
q_new=point,
len_history=len(history_arr))
history_arr_np = np.frombuffer(
pydream.Dream_shared_vars.history.get_obj())
history_arr_np_reshaped = history_arr_np.reshape(
np.shape(test_history))
self.assertIs(np.array_equal(history_arr_np_reshaped, test_history),
True)
remove('test_history_recording_DREAM_chain_history.npy')
remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy')
remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy')
def fit_two_poolable(file, two_pulse_fit, pulse_params, height_th, sigma0):
counter = mp.Value('i', 0)
try:
time = pu.time_vector(file)
signal = trcp.trace_extr(file, height_th)
# fname = file.split('/')[-1].split('.trc')[0]
fname = file
# print('success')
result = pfp.fit_two_cw(time, signal, two_pulse_fit, pulse_params,
height_th, sigma0)
# Extract results
results_summary, keys = results_extr(result)
df = results_packager(results_summary, keys, fname)
# Sends Telegram Message to Update Status
# counter.value+=1
# times_toupdate = 20
# text_tosend = '\nProcess finished:'+ '%.1f'%(counter.value/len(tasks)*100) + "%"
# print text_tosend
# try:
# if counter.value % int(len(tasks)/times_toupdate) == 0:
# sender_bot.sendMessage(chat_id=uid,text=text_tosend)
# except:
# pass
# Extracts useful data for saving
return df
except:
# print fname+' failed' #if file is corrupt
# raise
pass
def start_train(resume):
urllib.request.urlretrieve(const.url + '/network_file',
'deliverables/network.py')
urllib.request.urlretrieve(const.url + '/config_file',
'deliverables/input_params.py')
urllib.request.urlretrieve(const.url + '/observation_file',
'deliverables/observation.py')
urllib.request.urlretrieve(const.url + '/curriculum_file',
'deliverables/curriculum.py')
num_workers = mp.cpu_count() - 1
should_stop = mp.Value(c_bool, False)
while True:
worker_processes = []
# create_worker(0, should_stop)
# Start process 1 - n, running in other processes
for w_num in range(0, num_workers):
process = mp.Process(target=create_worker,
args=(w_num, should_stop))
process.start()
sleep(0.5)
worker_processes.append(process)
try:
for p in worker_processes:
p.join()
except KeyboardInterrupt:
should_stop.value = True
print("Looks like we're done")
def test_proposal_generation_snooker(self):
"""Test that proposal generation with a snooker update returns values of the expected shape."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
step = Dream(model=model)
history_arr = mp.Array('d', list(range(120)))
n = mp.Value('i', 0)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
step.nseedchains = 20
q0 = np.array([2, 3, 4, 5])
proposed_pt, snooker_logp, z = step.generate_proposal_points(
n_proposed_pts=1,
q0=q0,
CR=1,
DEpairs=1,
gamma_level=1,
snooker=True)
self.assertEqual(len(proposed_pt), step.total_var_dimension)
proposed_pts, snooker_logp, z = step.generate_proposal_points(
n_proposed_pts=5,
q0=q0,
CR=1,
DEpairs=1,
gamma_level=1,
snooker=True)
self.assertEqual(len(proposed_pts), 5)
def test_chain_sampling_multidim_model(self):
"""Test that sampling from DREAM history for multi-dimensional model when the history is known matches with expected possible samples."""
self.params, self.like = multidmodel()
model = Model(likelihood=self.like, sampled_parameters=self.params)
dream = Dream(model=model)
history_arr = mp.Array('d', [0] * 2 * dream.total_var_dimension)
n = mp.Value('i', 0)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
chains_added_to_history = []
for i in range(2):
start = i * dream.total_var_dimension
end = start + dream.total_var_dimension
chain = dream.draw_from_prior(model.sampled_parameters)
pydream.Dream_shared_vars.history[start:end] = chain
chains_added_to_history.append(chain)
sampled_chains = dream.sample_from_history(
nseedchains=2, DEpairs=1, ndimensions=dream.total_var_dimension)
sampled_chains = np.array(sampled_chains)
chains_added_to_history = np.array(chains_added_to_history)
self.assertIs(
np.array_equal(
chains_added_to_history[chains_added_to_history[:,
0].argsort()],
sampled_chains[sampled_chains[:, 0].argsort()]), True)
def __init__(self, value=0):
"""a counter where the value can be shared by multiple processes
"""
try: #XXX: is multiprocess ever useful here vs multiprocessing?
import multiprocess as mp
except ImportError:
import multiprocessing as mp
self.val = mp.Value('i', value)
def __init__(self, num_of_processes=2):
super(MultiprocessTool, self).__init__()
global global_last_id
global_last_id = mp.Value('i', 0)
self.num_of_processes = num_of_processes
lock = mp.Lock()
self.lock = lock
def test_crossover_prob_estimation(self):
"""Test that crossover probabilities are updated as expected when changing or not changing parameter locations and giving points that give a greater jump distance."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
dream = Dream(model=model, save_history=False)
starting_crossover = dream.CR_probabilities
crossover_probabilities = mp.Array('d', starting_crossover)
n = mp.Value('i', 0)
nCR = dream.nCR
CR_vals = dream.CR_values
ncrossover_updates = mp.Array('d', [0] * nCR)
current_position_arr = mp.Array(
'd', [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4])
dream.nchains = 5
delta_m = mp.Array('d', [0] * nCR)
dream.chain_n = 0
pydream.Dream_shared_vars.cross_probs = crossover_probabilities
pydream.Dream_shared_vars.count = n
pydream.Dream_shared_vars.ncr_updates = ncrossover_updates
pydream.Dream_shared_vars.current_positions = current_position_arr
pydream.Dream_shared_vars.delta_m = delta_m
q0 = np.array([1, 2, 3, 4])
q_new = np.array([1, 2, 3, 4])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[0])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
True)
q_new = np.array([1.2, 2.2, 3.3, 4.4])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[0])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
True)
q_new = np.array([2, 3, 4, 5])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[1])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
True)
q_new = np.array([11, -15, 20, 9])
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[2])
self.assertEqual(np.array_equal(new_cr_probs, starting_crossover),
False)
self.assertGreater(new_cr_probs[2], starting_crossover[2])
self.assertAlmostEqual(np.sum(new_cr_probs), 1.0, places=1)
old_cr_probs = new_cr_probs
for i, q_new in zip(list(range(5)), [
np.array([15]),
np.array([17]),
np.array([19]),
np.array([21]),
np.array([23])
]):
new_cr_probs = dream.estimate_crossover_probabilities(
dream.total_var_dimension, q0, q_new, CR_vals[1])
self.assertEqual(np.array_equal(new_cr_probs, old_cr_probs), False)
def __init__(self, db, username, password, host):
super(CampaignRuleConstraintMigrator, self).__init__()
self.db = db
self.username = username
self.password = password
self.host = host
self.conn = psycopg2.connect(database=self.db,
user=self.username,
password=self.password,
host=self.host)
self.cur = self.conn.cursor()
global global_last_id
global_last_id = mp.Value('i', 0)
def test_sharedvalues():
values = [('i', 10), ('h', -2), ('d', 1.25)]
arrays = [('i', range(100)), ('d', [0.25 * i for i in range(100)]),
('H', range(1000))]
shared_values = [processing.Value(id, v) for id, v in values]
shared_arrays = [processing.Array(id, a) for id, a in arrays]
p = processing.Process(target=sharedvalues_func,
args=(values, arrays, shared_values, shared_arrays))
p.start()
p.join()
assert p.exitcode == 0
def test_semaphore():
sema = processing.Semaphore(3)
mutex = processing.RLock()
running = processing.Value('i', 0)
processes = [
processing.Process(target=semaphore_func, args=(sema, mutex, running))
for i in range(10)
]
for p in processes:
p.start()
for p in processes:
p.join()
def main():
enable = mp.Value('i', 0)
while True:
#判断是否开始
sign = input("请输入控制命令：")
print('sign = {}'.format(sign))
if sign == '1': #bytes([0x01])
print('connect')
enable.value = 1
mp.Process(target=motionControl, args=(enable, )).start()
mp.Process(target=displayImage, args=(enable, )).start()
elif sign == '2':
print('disconnect')
enable.value = 0
def __init__(self, db, username, password, host):
super(GeosFiller, self).__init__()
self.db = db
self.username = username
self.password = username
self.host = host
self.conn = psycopg2.connect(
database=self.db,
user=self.username,
password=self.password,
host=self.host
)
self.cur = self.conn.cursor()
global global_last_id
global_last_id = mp.Value('i', 0)
def test_value():
TASKS = 10
running = processing.Value('i', TASKS)
mutex = processing.Lock()
for i in range(TASKS):
processing.Process(target=value_func, args=(running, mutex)).start()
while running.value > 0:
time.sleep(0.08)
mutex.acquire()
print(running.value, end=' ')
sys.stdout.flush()
mutex.release()
print()
print('No more running processes')
def main():
enable = mp.Value('i', 0)
imgQueue = mp.Queue(0)
imgQueueBin = mp.Queue(0)
while True:
sign = receive()
if sign == 1:
print('connect')
enable.value = 1
mp.Process(target=motionControl,
args=(enable, imgQueue, imgQueueBin)).start()
mp.Process(target=displayImage,
args=(enable, imgQueue, imgQueueBin)).start()
elif sign == 2:
print('disconnect')
enable.value = 0
def test_proposal_generation_nosnooker_CR66(self):
"""Test proposal generation without a snooker update with a single or multiple proposed points and a crossover value of 2/3 gives 2/3 of all dimensions changed on average as expected."""
self.param, self.like = multidmodel()
model = Model(self.like, self.param)
step = Dream(model=model)
history_arr = mp.Array('d', list(range(120)))
n = mp.Value('i', 0)
pydream.Dream_shared_vars.history = history_arr
pydream.Dream_shared_vars.count = n
step.nseedchains = 20
q0 = np.array([2, 3, 4, 5])
dims_kept = 0
for iteration in range(100000):
proposed_pt = step.generate_proposal_points(n_proposed_pts=1,
q0=q0,
CR=.66,
DEpairs=1,
gamma_level=1,
snooker=False)
if iteration == 1:
self.assertEqual(len(proposed_pt), 1)
dims_change_vec = np.squeeze(q0 == proposed_pt)
for dim in dims_change_vec:
if dim:
dims_kept += 1
frac_kept = dims_kept / (step.total_var_dimension * 100000.0)
self.assertAlmostEqual(frac_kept, 1 - .66, places=1)
dims_kept = 0
for iteration in range(10000):
proposed_pts = step.generate_proposal_points(n_proposed_pts=5,
q0=q0,
CR=.66,
DEpairs=1,
gamma_level=1,
snooker=False)
if iteration == 1:
self.assertEqual(len(proposed_pts), 5)
for pt in proposed_pts:
dims_change_vec = (q0 == pt)
for dim in dims_change_vec:
if dim:
dims_kept += 1
frac_kept = dims_kept / (step.total_var_dimension * 10000.0 * 5)
self.assertAlmostEqual(frac_kept, 1 - .66, places=1)
ORDER = neopixel.GRB
pixels = neopixel.NeoPixel(pixel_pin,
num_pixels,
brightness=0.2,
auto_write=False,
pixel_order=ORDER)
app = Flask(__name__)
#clock globals
totsec = 86400
#ctr = 0
loc_now = True
#manager = Manager()
ctr = multiprocess.Value('i', 0)
tf = multiprocess.Value('i', 1)
@app.route("/search/<query>")
def coord_query(query):
"""
Web API server query
"""
global tf
if query is None: return jsonify(success=False)
query = query.replace("&&", " ")
lat, long = google_coords(query)
if (lat is not None and long is not None):
res = coords_request(lat, long)
def __call__(self, queue, results):
""" Run worker.
Parameters
----------
queue : multiprocessing.Queue
queue of jobs for worker
results : multiprocessing.Queue
queue for feedback
"""
_devices = [item['device'] for item in self.devices]
self.logger.info('Start {} [id:{}] (devices: {})'.format(
self.worker_name, os.getpid(), _devices))
try:
job = queue.get()
except Exception as exception: #pylint:disable=broad-except
self.logger.error(exception)
else:
while job is not None:
try:
finished = False
self.logger.info(self.worker_name +
' is creating process for Job ' +
str(job[0]))
for trial in range(self.trials):
one_job_queue = mp.JoinableQueue()
one_job_queue.put(job)
feedback_queue = mp.JoinableQueue()
last_update_time = mp.Value('d', time.time())
task = mp.Process(target=self._run_task,
args=(one_job_queue, feedback_queue,
trial, last_update_time))
task.start()
pid = feedback_queue.get()
final_signal = Signal(worker=self.worker_name,
job=job[0],
iteration=0,
n_iters=job[1].n_iters,
trial=trial,
done=False,
exception=None)
while True:
try:
signal = feedback_queue.get(timeout=1)
except EmptyException:
signal = None
if signal is None:
execution_time = (time.time() -
last_update_time.value) / 60
if self.timeout is not None and execution_time > self.timeout:
p = psutil.Process(pid)
p.terminate()
message = f'Job {job[0]} [{pid}] failed in {self.worker_name} because of timeout'
self.logger.info(message)
final_signal.exception = TimeoutError(
message)
results.put(copy(final_signal))
break
if signal is not None and signal.done:
finished = True
final_signal = signal
break
if signal is not None:
final_signal = signal
results.put(copy(final_signal))
if finished:
break
except Exception as exception: #pylint:disable=broad-except
self.logger.error(exception)
final_signal.exception = exception
results.put(copy(final_signal))
if final_signal.done:
results.put(copy(final_signal))
else:
final_signal.exception = RuntimeError(
'Job {} [{}] failed {} times in {}'.format(
job[0], pid, self.trials, self.worker_name))
final_signal.done = True
results.put(copy(final_signal))
queue.task_done()
job = queue.get()
queue.task_done()
def __init__(self, val=0):
self.val = multiprocess.Value('i', val)
def _setup_mp_dream_pool(nchains, niterations, step_instance, start_pt=None):
min_njobs = (2 * len(step_instance.DEpairs)) + 1
if nchains < min_njobs:
raise Exception(
'Dream should be run with at least (2*DEpairs)+1 number of chains. For current algorithmic settings, set njobs>=%s.'
% str(min_njobs))
if step_instance.history_file != False:
old_history = np.load(step_instance.history_file)
len_old_history = len(old_history.flatten())
nold_history_records = len_old_history / step_instance.total_var_dimension
step_instance.nseedchains = nold_history_records
if niterations < step_instance.history_thin:
arr_dim = (
(np.floor(nchains * niterations / step_instance.history_thin) +
nchains) *
step_instance.total_var_dimension) + len_old_history
else:
arr_dim = np.floor(((
(nchains * niterations) * step_instance.total_var_dimension) /
step_instance.history_thin)) + len_old_history
else:
if niterations < step_instance.history_thin:
arr_dim = (
(np.floor(nchains * niterations / step_instance.history_thin) +
nchains) * step_instance.total_var_dimension
) + (step_instance.nseedchains * step_instance.total_var_dimension)
else:
arr_dim = np.floor(
((nchains * niterations / step_instance.history_thin) *
step_instance.total_var_dimension)
) + (step_instance.nseedchains * step_instance.total_var_dimension)
min_nseedchains = 2 * len(step_instance.DEpairs) * nchains
if step_instance.nseedchains < min_nseedchains:
raise Exception(
'The size of the seeded starting history is insufficient. Increase nseedchains>=%s.'
% str(min_nseedchains))
current_position_dim = nchains * step_instance.total_var_dimension
history_arr = mp.Array('d', [0] * int(arr_dim))
if step_instance.history_file != False:
history_arr[0:len_old_history] = old_history.flatten()
nCR = step_instance.nCR
ngamma = step_instance.ngamma
crossover_setting = step_instance.CR_probabilities
crossover_probabilities = mp.Array('d', crossover_setting)
ncrossover_updates = mp.Array('d', [0] * nCR)
delta_m = mp.Array('d', [0] * nCR)
gamma_level_setting = step_instance.gamma_probabilities
gamma_probabilities = mp.Array('d', gamma_level_setting)
ngamma_updates = mp.Array('d', [0] * ngamma)
delta_m_gamma = mp.Array('d', [0] * ngamma)
current_position_arr = mp.Array('d', [0] * current_position_dim)
shared_nchains = mp.Value('i', nchains)
n = mp.Value('i', 0)
tf = mp.Value('c', b'F')
if step_instance.crossover_burnin == None:
step_instance.crossover_burnin = int(np.floor(niterations / 10))
if start_pt != None:
if step_instance.start_random:
print(
'Warning: start position provided but random_start set to True. Overrode random_start value and starting walk at provided start position.'
)
step_instance.start_random = False
p = DreamPool(nchains,
initializer=_mp_dream_init,
initargs=(
history_arr,
current_position_arr,
shared_nchains,
crossover_probabilities,
ncrossover_updates,
delta_m,
gamma_probabilities,
ngamma_updates,
delta_m_gamma,
n,
tf,
))
#p = mp.pool.ThreadPool(nchains, initializer=_mp_dream_init, initargs=(history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, ))
#p = mp.Pool(nchains, initializer=_mp_dream_init, initargs=(history_arr, current_position_arr, shared_nchains, crossover_probabilities, ncrossover_updates, delta_m, gamma_probabilities, ngamma_updates, delta_m_gamma, n, tf, ))
return p
