def run(self):
try:
self.run_setup()
self.active = True
while self.active:
if not self.poll_control():
self.process()
except:
import traceback
import sys
exc_type, exc_value, exc_traceback = sys.exc_info()
print("*** print_tb:")
traceback.print_tb(exc_traceback)
print("*** print_exception:")
traceback.print_exception(exc_type, exc_value, exc_traceback)
self.qout['control'].put(message('terminate'))
finally:
self.cleanup()
self.qout['control'].put(
message('terminated',
pid=self.pid,
obj_class=self.__class__.__name__))
def handle_current_chunk(self):
period_idx = self.cur_chunk['period_idx']
# if period_idx != self.cur_period_idx:
# self._set_cur_period(period_idx)
chunk_idx = self.cur_chunk['chunk_idx']
bgroup = self.map_buffer(self.cur_chunk['buffer'])
bdata = bgroup.map_dims(self.cur_chunk['dims'])
coords = gen_coordset(self.periods[period_idx], self.chunks[chunk_idx])
self.exe_graph.set_data(coords, bdata, self.chunks[chunk_idx].mask)
self.reclaim_buffer(self.cur_chunk['buffer'])
# in_data = self.map_buffer(shm_buf,data['shape'])
#
# pwrite_idx = self.time_indices[v]
# chunk = self.chunks[chunk_idx]
#
# write_idx = (pwrite_idx,chunk.x,chunk.y)
#
# # ENFORCE_MASK +++
# if self.enforce_mask:
# subex = self.extents[chunk_idx]
# in_data[:,subex.mask==True] = FILL_VALUE
#
# self._write_slice(write_var,in_data,write_idx)
self.cur_chunk = None
self.cur_chunk_count += 1
completed = self.cur_chunk_count / len(self.chunks) * 100.
if completed - self.completed > 5:
self._send_log(message("completed %.2f%%" % completed))
# logger.info("completed %.2f%%",completed)
self.completed = completed
if self.cur_chunk_count == len(self.chunks):
# logger.info("Completed period %s - %d of %d",dt.pretty_print_period(self.periods[self.cur_period_count]),self.cur_period_count+1,len(self.periods))
self._send_log(
message("Completed period %s - %d of %d" %
(dt.pretty_print_period(
self.periods[self.cur_period_count]),
self.cur_period_count + 1, len(self.periods))))
self.completed = 0
self.cur_chunk_count = 0
self.cur_period_count += 1
self.exe_graph.sync_all()
if self.cur_period_count == len(self.periods):
self._send_log(message("terminate"))
self.terminate()
def read_active_chunk(self):
chunk_msg = message('chunk')
content = chunk_msg['content']
content['chunk_idx'] = self.cur_chunk_idx
content['period_idx'] = self.cur_period_idx
data = content['data'] = {}
chunk = self.chunks[self.cur_chunk_idx]
for variable in self.variables:
if self.cur_ds[variable] is not None:
#+++
# Need to ensure timeout/polling
# on get_buffer; this should probably
# be made more general...
buf = None
while buf is None:
try:
buf, arr = self.get_buffer()
except Empty:
if self.poll_control():
raise ControlInterrupt
read_complete = False
cur_retries = 0
time_idx = self.time_indices[variable]
if isinstance(time_idx, slice):
time_shape = time_idx.stop - time_idx.start
elif isinstance(time_idx, int):
time_shape = 1
else:
time_shape = len(time_idx)
out_shape = (time_shape, chunk.shape[0], chunk.shape[1])
write_idx = [slice(0, time_shape)] + shape_idx(chunk.shape)
while not read_complete:
try:
read_var = self.cur_ds[variable][
self.nc_vars[variable]]
arr[write_idx] = read_var[time_idx, chunk.x, chunk.y]
read_complete = True
except:
if cur_retries < MAX_RETRIES:
print("Retrying read of %s (%s)" %
(variable, self.cur_f[variable]))
time.sleep(1)
self.cur_ds[variable].close()
self.cur_ds[variable] = db_opener(
self.cur_f[variable], 'r')
cur_retries += 1
else:
raise
data[variable] = dict(buffer=buf, shape=out_shape)
return chunk_msg
def test_fanin():
from time import sleep
from multiprocessing import Queue as MPQueue
from awrams.utils.messaging.brokers import FanInChunkBroker
from awrams.utils.messaging.general import message
NWORKERS = 4
m2b, b2m, outq = MPQueue(), MPQueue(), MPQueue(1)
qin = dict(control=m2b)
for i in range(NWORKERS):
qin[i] = MPQueue()
qout = dict(control=b2m, out=outq, workers=MPQueue())
broker = FanInChunkBroker(qin, qout, NWORKERS)
broker.start()
sleep(1)
# Send some input
i = 0
for x in range(100):
#sleep(0.0001)
print(x)
qin['control'].put(message(i))
for y in range(4):
print(y)
# outpipes[i%16].send(i)
qin[y].put(message(i))
print("i", i)
i += 1
print("WORKERS", qout['workers'].get())
print("OUTQ", outq.get())
#for x in range(1600*16):
# outq.get()
m2b.put(message('terminate'))
broker.join()
resp = b2m.get()
print(resp)
assert resp['content']['obj_class'] == 'FanInChunkBroker'
assert resp['subject'] == 'terminated'
def terminate(self, wait_for_results=True):
self.comms_manager.terminate()
iterations = self.optimizer.evaluations if wait_for_results else -1
self.logger_q.put(
message('terminate', iterations=self.optimizer.evaluations))
self.logger.join()
self.optimizer.terminate_children()
def _submit_evaluation(self, params, job_tag='default'):
job_meta = dict(source=self.eval_id, job_tag=job_tag)
job_msg = message('evaluate', params=dict(params), job_meta=job_meta)
self.job_q.put(job_msg)
self.params[job_tag] = dict(params)
self.results[job_tag] = pd.DataFrame(index=self.subtask_ids,
columns=self.local_schema,
dtype=float)
self.result_counts[job_tag] = 0
def chunk_message(chunk_idx, period_idx, data=None):
chunk_msg = message('chunk')
content = chunk_msg['content']
content['chunk_idx'] = chunk_idx
content['period_idx'] = period_idx
if data is None:
data = {}
content['data'] = data
return chunk_msg
def send(self):
try:
msg = message('terminate')
for q in self.qs:
q.put(msg,timeout=1)
self.active = False
if self.exception_raised:
self.statusq.put({'content': {}, 'subject': 'exception_raised'})
else:
self.statusq.put({'content': {}, 'subject': 'finished'})
except Full:
raise
def evolve(self,
population,
num_offspring,
num_evolutions,
alpha=1.0,
beta=0.5,
spread=None):
results = ComplexEvolver.evolve(self,
population,
num_offspring,
num_evolutions,
alpha=1.0,
beta=0.5,
spread=spread)
self._send_msg(message('handle_results', **results))
def run(self):
self.pr.begin_profiling()
self.active = True
try:
self.run_setup()
while self.active:
msg = self._recv_msg() # (['request',variable,row_idx])
self._result = self._handle_message(msg)
#actions = msg.get('actions')
#if actions is not None:
# for action in actions:
# self._send_msg(message(subject=action['subject'], content=_result),target=action['target'])
#self._send_msg('ack',acknowledgement=ack['message'],id=ack['id'])
except Exception as e:
self._handle_exception(e)
raise
finally:
self.cleanup()
self.pr.end_profiling()
self._send_msg(
message('child_profile_results',
pid=self.pid,
profiler=self.pr))
self._send_msg(message('child_terminated', pid=self.pid))
def terminate_children(self):
'''
Terminate the simulation; close any open subprocesses
'''
#+++
#Input_bridge still using special case ZMQ code
#should be converted to regular python mp process
#self.input_bridge.terminate()
for process in list(self.child_procs.values()):
process['msg_q'].put(message('terminate'))
while len(self.child_procs) > 0:
msg = self.control_q.get()
self._handle_message(msg)
def setup_nodes(self, node_settings):
msg = message('settings', node_settings=node_settings)
self.comms_manager.broadcast_msg(msg)
def new_fn(self, fname, argnames, *args, **kwargs):
a = {}
for i in range(len(args)):
a[argnames[i]] = args[i]
a.update(**kwargs)
self._send(msg=message(subject=fname, **a))
def _terminate_nodes(self):
for v in list(self.out_queues.values()):
v.put(message('terminate'))
def resample_data(in_path,
in_pattern,
variable,
period,
out_path,
to_freq,
method,
mode='w',
enforce_mask=True,
extent=None,
use_weights=False):
'''
method is 'sum' or 'mean'
if no extent is supplied then the full (unmasked) input will be used
'use_weights' should be set for unequally binned conversions (monthly->annual means, for example)
'''
from glob import glob
import time
import numpy as np
from awrams.utils.messaging import reader as nr
from awrams.utils.messaging import writer as nw
from awrams.utils.messaging.brokers import OrderedFanInChunkBroker, FanOutChunkBroker
from awrams.utils.messaging.general import message
from awrams.utils.messaging.buffers import create_managed_buffers
from awrams.utils.processing.chunk_resampler import ChunkedTimeResampler
from awrams.utils.catchments import subdivide_extent
from awrams.utils import datetools as dt
from awrams.utils import mapping_types as mt
from awrams.utils.io import data_mapping as dm
start = time.time()
NWORKERS = 2
read_ahead = 3
writemax = 3
BLOCKSIZE = 128
nbuffers = (NWORKERS * 2) + read_ahead + writemax
# Receives all messages from clients
'''
Build the 'standard queues'
This should be wrapped up somewhere else for
various topologies...
'''
control_master = mp.Queue()
worker_q = mp.Queue()
for i in range(NWORKERS):
worker_q.put(i)
#Reader Queues
chunk_out_r = mp.Queue(read_ahead)
reader_in = dict(control=mp.Queue())
reader_out = dict(control=control_master, chunks=chunk_out_r)
#Writer Queues
chunk_in_w = mp.Queue(writemax)
writer_in = dict(control=mp.Queue(), chunks=chunk_in_w)
writer_out = dict(control=control_master)
#FanIn queues
fanout_in = dict(control=mp.Queue(), chunks=chunk_out_r, workers=worker_q)
fanout_out = dict(control=control_master)
fanin_in = dict(control=mp.Queue())
fanin_out = dict(control=control_master, out=chunk_in_w, workers=worker_q)
#Worker Queues
work_inq = []
work_outq = []
for i in range(NWORKERS):
work_inq.append(mp.Queue())
fanout_out[i] = work_inq[-1]
work_outq.append(mp.Queue())
fanin_in[i] = work_outq[-1]
'''
End standard queues...
'''
infiles = glob(in_path + '/' + in_pattern)
if len(infiles) > 1:
ff = dm.filter_years(period)
else:
ff = None
sfm = dm.SplitFileManager.open_existing(in_path,
in_pattern,
variable,
ff=ff)
in_freq = sfm.get_frequency()
split_periods = [period]
if hasattr(in_freq, 'freqstr'):
if in_freq.freqstr == 'D':
#Force splitting so that flat files don't end up getting loaded entirely into memory!
#Also a bit of a hack to deal with PeriodIndex/DTI issues...
split_periods = dt.split_period(
dt.resample_dti(period, 'd', as_period=False), 'a')
in_periods = [dt.resample_dti(p, in_freq) for p in split_periods]
in_pmap = sfm.get_period_map_multi(in_periods)
out_periods = []
for p in in_periods:
out_periods.append(dt.resample_dti(p, to_freq))
if extent is None:
extent = sfm.ref_ds.get_extent(True)
if extent.mask.size == 1:
extent.mask = (np.ones(extent.shape) * extent.mask).astype(np.bool)
sub_extents = subdivide_extent(extent, BLOCKSIZE)
chunks = [nr.Chunk(*s.indices()) for s in sub_extents]
out_period = dt.resample_dti(period, to_freq)
out_cs = mt.gen_coordset(out_period, extent)
v = mt.Variable.from_ncvar(sfm.ref_ds.awra_var)
in_dtype = sfm.ref_ds.awra_var.dtype
sfm.close_all()
use_weights = False
if method == 'mean':
if dt.validate_timeframe(in_freq) == 'MONTHLY':
use_weights = True
'''
Need a way of formalising multiple buffer pools for different classes of
work..
'''
max_inplen = max([len(p) for p in in_periods])
bufshape = (max_inplen, BLOCKSIZE, BLOCKSIZE)
shared_buffers = {}
shared_buffers['main'] = create_managed_buffers(nbuffers,
bufshape,
build=False)
mvar = mt.MappedVariable(v, out_cs, in_dtype)
sfm = dm.FlatFileManager(out_path, mvar)
CLOBBER = mode == 'w'
sfm.create_files(False, CLOBBER, chunksize=(1, BLOCKSIZE, BLOCKSIZE))
outfile_maps = {
v.name:
dict(nc_var=v.name, period_map=sfm.get_period_map_multi(out_periods))
}
infile_maps = {v.name: dict(nc_var=v.name, period_map=in_pmap)}
reader = nr.StreamingReader(reader_in, reader_out, shared_buffers,
infile_maps, chunks, in_periods)
writer = nw.MultifileChunkWriter(writer_in,
writer_out,
shared_buffers,
outfile_maps,
sub_extents,
out_periods,
enforce_mask=enforce_mask)
fanout = FanOutChunkBroker(fanout_in, fanout_out)
fanin = OrderedFanInChunkBroker(fanin_in, fanin_out, NWORKERS, len(chunks))
fanout.start()
fanin.start()
workers = []
w_control = []
for i in range(NWORKERS):
w_in = dict(control=mp.Queue(), chunks=work_inq[i])
w_out = dict(control=control_master, chunks=work_outq[i])
w = ChunkedTimeResampler(w_in,
w_out,
shared_buffers,
sub_extents,
in_periods,
to_freq,
method,
enforce_mask=enforce_mask,
use_weights=use_weights)
workers.append(w)
w_control.append(w_in['control'])
w.start()
writer.start()
reader.start()
writer.join()
fanout_in['control'].put(message('terminate'))
fanin_in['control'].put(message('terminate'))
for i in range(NWORKERS):
w_control[i].put(message('terminate'))
for x in range(4):
control_master.get()
for i in range(NWORKERS):
workers[i].join()
control_master.get()
reader.join()
fanout.join()
fanin.join()
end = time.time()
logger.info("elapsed time: %ss", end - start)
def terminate(self):
self.submission_q.put(message('terminate'))
def _send_result(wrapped, self, *args, **kwargs):
result = wrapped(self, *args, **kwargs)
self._send_msg(message(subject, **result), target=target)
def submit_evaluation(self, params, job_meta):
job_msg = message('evaluate', params=params, job_meta=job_meta)
self.add_job(job_msg)
def log_results(self, parameters, global_score, local_scores):
self.log_q.put(
message('log_results',
parameters=parameters,
global_score=global_score,
local_scores=local_scores))
def _handle_exception(self, e):
m = message('child_exception',
pid=self.pid,
exception=e,
traceback=get_traceback())
self._send_msg(m)
def read_active_chunk(self):
chunk_msg = message('chunk')
content = chunk_msg['content']
content['chunk_idx'] = self.cur_chunk_idx
content['period_idx'] = self.cur_period_idx
data = content['data'] = {}
chunk = self.chunks[self.cur_chunk_idx]
for variable in self.variables:
if self.cur_ds[variable] is not None:
#+++
# Need to ensure timeout/polling
# on get_buffer; this should probably
# be made more general...
buf = None
while buf is None:
try:
buf, arr = self.get_buffer()
except Empty:
if self.poll_control():
raise ControlInterrupt
read_complete = False
cur_retries = 0
time_idx = self.time_indices[variable]
season_shape = self.season_indices[variable]
out_shape = (season_shape, chunk.shape[0], chunk.shape[1])
write_idx = [slice(0, season_shape)] + shape_idx(chunk.shape)
while not read_complete:
try:
read_var = self.cur_ds[variable][
self.nc_vars[variable]]
### for non-overlapping windows...window size must be <=12 months
# arr[write_idx] = read_var[time_idx,chunk.x,chunk.y].reshape((self.num_agg,-1,out_shape[1],out_shape[2]),order='F').mean(axis=0)
### for moving window...ie kernel moves 1 month steps
in_data = read_var[time_idx, chunk.x, chunk.y]
kernel = np.ones(
(self.num_agg, )) / float(self.num_agg)
arr[write_idx] = (np.apply_along_axis(
lambda m: np.convolve(m, kernel, mode='valid'),
axis=0,
arr=in_data)[-1::-12, :])[::-1, :]
read_complete = True
except AttributeError:
if cur_retries < MAX_RETRIES:
print("Retrying read of %s (%s)" %
(variable, self.cur_f[variable]))
time.sleep(1)
self.cur_ds[variable].close()
# h5py_cleanup_nc_mess()
self.cur_ds[variable] = db_opener(
self.cur_f[variable], 'r')
cur_retries += 1
else:
raise
data[variable] = dict(buffer=buf, shape=out_shape)
return chunk_msg
def terminate(self):
self._send(message('terminate'))
