def __init__(self, socket, message_data, *args, **kwargs):
self.socket = socket
self.message_data = message_data
threading.Thread.__init__(self, *args, **kwargs)
# a flag to notify the thread that it should finish up and exit
self.kill_received = False
self.reporter = StateReporter()
class Listener(threading.Thread):
def __init__(self, socket, message_data, *args, **kwargs):
self.socket = socket
self.message_data = message_data
threading.Thread.__init__(self, *args, **kwargs)
# a flag to notify the thread that it should finish up and exit
self.kill_received = False
self.reporter = StateReporter()
def reset_grid_data(self):
logger.debug(
'Resetting grid data...',
extra={'subgrid_id': self.reporter.redis_key})
for k in self.message_data.grid.keys():
del self.message_data.grid[k] # try to save memory
self.message_data.interpolation_ready = False
self.message_data.pandas = {}
def _run(self):
"""run the thread"""
message_data = self.message_data
socket = self.socket
while not self.kill_received:
logger.debug(
'(a) number of busy workers: %s' %
self.reporter.get_busy_workers(),
extra={'subgrid_id': self.reporter.redis_key})
arr, metadata = recv_array(socket)
# now it is busy
self.reporter.set_busy()
# N.B.: to simulate the wms_busy state uncomment the following
# line, but do not commit it, never!
# time.sleep(random.uniform(0.0, 0.5))
logger.debug(
'(b) number of busy workers: %s' %
self.reporter.get_busy_workers(),
extra={'subgrid_id': self.reporter.redis_key})
logger.debug(
'time in seconds wms is considered busy: %s' %
str(self.reporter.busy_duration),
extra={'subgrid_id': self.reporter.redis_key})
if metadata['action'] == 'reset':
self.reset_grid_data()
elif metadata['action'] == 'update':
self.reporter.set_timestep(metadata['sim_time_seconds'])
logger.debug(
'Updating grid data [%s]' % metadata['name'],
extra={'subgrid_id': self.reporter.redis_key})
if 'model' in metadata:
restarted = metadata['name'] == 't1' and \
metadata['sim_time_seconds'] < 0.1
if metadata['model'] != message_data.loaded_model or \
restarted:
# Since working with 'reset', this part probably never
# occur anymore.
# New model detected
logger.info(
'New model detected: %r (old=%r)' % (
metadata['model'], message_data.loaded_model),
extra={'subgrid_id': self.reporter.redis_key})
# Double reset algorithm.
message_data.grid = {}
message_data.interpolation_ready = False
message_data.loaded_model = metadata['model']
# Update new grid
if arr.dtype.kind == 'S':
# String, for wkt
message_data.grid[metadata['name']] = ''.join(arr)
else:
if 'bbox' in metadata:
logger.debug("BBOXED update")
x0, x1, y0, y1 = metadata['bbox']
message_data.grid[metadata['name']][y0:y1, x0:x1] = arr
else:
# normal case
message_data.grid[metadata['name']] = arr.copy()
# Receive one of the DEPTH_VARS and all DEPTH_VARS are complete
if (all([v in message_data.grid for v in DEPTH_VARS]) and
metadata['name'] in DEPTH_VARS):
if 'bbox' in metadata:
logger.debug(
'Update grids using bbox after receiving '
'dps or quad_grid...',
extra={'subgrid_id': self.reporter.redis_key})
message_data.update_grids_bbox(metadata['bbox'])
else:
logger.debug(
'Update grids after receiving dps or quad_grid...',
extra={'subgrid_id': self.reporter.redis_key})
message_data.update_grids()
logger.debug(
'Update grids finished.',
extra={'subgrid_id': self.reporter.redis_key})
# check update indices
if (all([v in message_data.grid for v in UPDATE_INDICES_VARS])
and metadata['name'] in UPDATE_INDICES_VARS):
logger.debug(
'Update indices...',
extra={'subgrid_id': self.reporter.redis_key})
message_data.X, message_data.Y, message_data.L = \
message_data.calc_indices()
logger.debug(
'Update indices finished.',
extra={'subgrid_id': self.reporter.redis_key})
elif metadata['action'] == 'update-pandas':
logger.debug(
'Update pandas data [%s]...', metadata['name'],
extra={'subgrid_id': self.reporter.redis_key})
# TODO: in case of weir, delete unused variables.
message_data.pandas[metadata['name']] = json.loads(
metadata['pandas_json'])
elif metadata['action'] == 'dump':
output_filename = metadata['output_filename']
logger.debug(
'Dump: checking other threads...',
extra={'subgrid_id': self.reporter.redis_key})
filename_failed = output_filename + '.failed'
if os.path.exists(filename_failed):
os.remove(filename_failed)
if i_am_the_boss(output_filename):
# TODO: with statement
nc_dump = NCDump(output_filename, message_data)
nc_dump.dump_nc('wkt', 'S1', ('i', ), '-',
list(message_data.grid['wkt']))
nc_dump.dump_nc('x0p', 'f8', (), '-')
nc_dump.dump_nc('y0p', 'f8', (), '-')
nc_dump.dump_nc('x1p', 'f8', (), '-')
nc_dump.dump_nc('y1p', 'f8', (), '-')
nc_dump.dump_nc('dxp', 'f8', (), '-')
nc_dump.dump_nc('dyp', 'f8', (), '-')
nc_dump.dump_nc('imax', 'i4', (), '-')
nc_dump.dump_nc('jmax', 'i4', (), '-')
nc_dump.dump_nc('imaxk', 'i4', ('k', ), '-') #
nc_dump.dump_nc('jmaxk', 'i4', ('k', ), '-')
nc_dump.dump_nc('nodm', 'i4', ('nFlowElem2', ), '-')
nc_dump.dump_nc('nodn', 'i4', ('nFlowElem2', ), '-')
nc_dump.dump_nc('nodk', 'i4', ('nFlowElem2', ), '-')
nc_dump.dump_nc('nod_type', 'i4', ('nFlowElem2', ), '-')
nc_dump.dump_nc('dsnop', 'f4', (), '-')
nc_dump.dump_nc('dps', 'f4', ('x', 'y', ), '-')
try:
nc_dump.close()
except:
# I don't know when nc_dump will fail, but if it fails,
# it is probably here.
with file(filename_failed, 'w') as f:
f.write('I failed...')
# So others can see we are finished.
os.remove(output_filename + '.busy')
else:
logger.debug(
'Got an unknown message: %r' % metadata,
extra={'subgrid_id': self.reporter.redis_key})
# set this worker to not busy
self.reporter.set_not_busy()
self.reporter.handle_busy_flag()
def run(self):
"""Run the thread fail-safe"""
while not self.kill_received:
try:
self._run()
except:
logger.exception(
'An unknown exception occured, subgrid_id: %s' %
self.reporter.redis_key)
# Throw away existing data: can be corrupt.
# you have to restart a model or 'beam to wms'.
self.reset_grid_data()
