def _default_labels_single_grid(from_dict):
labels = _od()
labels.title = "%s, (%s)" % (from_dict['variable'].name,
from_dict['variable'].source.name)
labels.ylabel = "%s" % (datetools.pretty_print_period(from_dict['period']))
labels.cmap = 'RdYlBu'
labels.units = from_dict['variable'].units
return labels
def handle_current_chunk(self):
period_idx = self.cur_chunk['period_idx']
# if period_idx != self.cur_period_idx:
# self._set_cur_period(period_idx)
chunk_idx = self.cur_chunk['chunk_idx']
bgroup = self.map_buffer(self.cur_chunk['buffer'])
bdata = bgroup.map_dims(self.cur_chunk['dims'])
coords = gen_coordset(self.periods[period_idx], self.chunks[chunk_idx])
self.exe_graph.set_data(coords, bdata, self.chunks[chunk_idx].mask)
self.reclaim_buffer(self.cur_chunk['buffer'])
# in_data = self.map_buffer(shm_buf,data['shape'])
#
# pwrite_idx = self.time_indices[v]
# chunk = self.chunks[chunk_idx]
#
# write_idx = (pwrite_idx,chunk.x,chunk.y)
#
# # ENFORCE_MASK +++
# if self.enforce_mask:
# subex = self.extents[chunk_idx]
# in_data[:,subex.mask==True] = FILL_VALUE
#
# self._write_slice(write_var,in_data,write_idx)
self.cur_chunk = None
self.cur_chunk_count += 1
completed = self.cur_chunk_count / len(self.chunks) * 100.
if completed - self.completed > 5:
self._send_log(message("completed %.2f%%" % completed))
# logger.info("completed %.2f%%",completed)
self.completed = completed
if self.cur_chunk_count == len(self.chunks):
# logger.info("Completed period %s - %d of %d",dt.pretty_print_period(self.periods[self.cur_period_count]),self.cur_period_count+1,len(self.periods))
self._send_log(
message("Completed period %s - %d of %d" %
(dt.pretty_print_period(
self.periods[self.cur_period_count]),
self.cur_period_count + 1, len(self.periods))))
self.completed = 0
self.cur_chunk_count = 0
self.cur_period_count += 1
self.exe_graph.sync_all()
if self.cur_period_count == len(self.periods):
self._send_log(message("terminate"))
self.terminate()
def gen_labels(self, datum):
dd = {
"title": datum[self.col_field],
"ylabel": "%s" % (dt.pretty_print_period(datum['period'])),
"units": datum['units'],
"cmap": 'RdYlBu'
}
if datum.source.name is not None:
dd['title'] += " " + datum.source.name
dd.update(self.kwds)
dd.update(datum)
for k in 'data', 'extent':
if k in dd:
del dd[k]
return dd
def run(self):
logger.info("Running")
progress_cells = 2000
try:
#gc.disable()
#gc.enable()
total_cell_days = float(len(self.period) * self.extent.cell_count)
cell_days_done = 0
cells = self.extent.cell_list()
#for procs in self.processor_map.values():
# for processor in procs:
# processor.set_extent(self.extent)
#processor.set_extent(self.input_mapper.input_bridge.geo_ref)
for cur_period in self.split_periods:
logger.info("Running period: %s: ",
dt.pretty_print_period(cur_period))
pct_done = (cell_days_done / total_cell_days) * 100.0
logger.info("%.2f%% complete", pct_done)
self.input_mapper.set_active_period(cur_period)
for procs in list(self.processor_map.values()):
for processor in procs:
processor.set_active_period(cur_period)
for cell in cells:
try:
msg = self.control.get_nowait()
if (msg):
if msg['message'] == 'error':
logger.error("Error running cell (%d,%d): %s",
cell[0], cell[1], msg)
raise Exception("%s", msg)
except mp.queues.Empty:
pass
cell_days_done += len(cur_period)
data = self.input_mapper.get_cell_dict(cell)
for k, procs in list(self.processor_map.items()):
for processor in procs:
# +++ assuming mask value is -999.
processor.process_cell(
np.ma.masked_values(data[k], -999.), cell)
for procs in list(self.processor_map.values()):
for processor in procs:
processor.finalise()
except:
raise
finally:
self.input_mapper.terminate()
logger.info("Finished")
while not self.control.empty():
msg = self.control.get()
if msg['message'] == 'occupancy':
logger.info("%s: %.2f%% occupancy" %
(msg['process'], msg['value'] * 100.))
else:
logger.info("%s" % msg)