def __init__(self, num_workers=1, pidantic_dir=None, working_dir=None):
self.guid = create_guid()
self.prep_queue = queue.Queue()
self.work_queue = queue.Queue()
self._pending_work = {}
self._stashed_work = {}
self._active_work = {}
self._do_stop = False
self._count = 0
self._active_work_lock = coros.RLock()
self._pending_work_lock = coros.RLock()
self._shutdown = False
self.context = zmq.Context(1)
self.prov_sock = self.context.socket(zmq.REP)
self.prov_port = self._get_port(self.prov_sock)
log.info('Provisioning url: tcp://*:{0}'.format(self.prov_port))
self.resp_sock = self.context.socket(zmq.SUB)
self.resp_port = self._get_port(self.resp_sock)
self.resp_sock.setsockopt(zmq.SUBSCRIBE, '')
log.info('Response url: tcp://*:{0}'.format(self.resp_port))
self.num_workers = num_workers if num_workers > 0 else 1
self.is_single_worker = self.num_workers == 1
self.working_dir = working_dir or '.'
self.pidantic_dir = pidantic_dir or './pid_dir'
self.workers = []
self._configure_workers()
def _write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD):
"""
Creates a virtual brick in the PersistenceLayer by updating the HDF5 master file's
brick list, rtree and ExternalLink to where the HDF5 file will be saved in the future (lazy create)
@param rtree_extents Total extents of brick's domain in rtree format
@param brick_extents Size of brick
@param brick_active_size Size of brick (same rank as parameter)
@param origin Domain origin offset
@param bD Slice-friendly size of brick's domain
@return N/A
"""
log.debug('Writing virtual brick...')
# Set HDF5 file and group
# Create a GUID for the brick
brick_guid = create_guid()
brick_file_name = '{0}.hdf5'.format(brick_guid)
#TODO: Inclusion of external links only used for external viewing of master file, remove if non-performant
for parameter_name in self.parameter_metadata.keys():
self._add_brick_link(parameter_name, brick_guid, brick_file_name)
# Update the brick listing
log.debug('Updating brick list[%s] with (%s, %s, %s, %s)', brick_guid, brick_extents, origin, tuple(bD), brick_active_size)
brick_count = len(self.master_manager.brick_list)
self.master_manager.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size]
log.debug('Brick count is %s', brick_count)
# Insert into Rtree
log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid)
self.master_manager.update_rtree(brick_count, rtree_extents, obj=brick_guid)
def _write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD):
"""
Creates a virtual brick in the PersistenceLayer by updating the HDF5 master file's
brick list, rtree and ExternalLink to where the HDF5 file will be saved in the future (lazy create)
@param rtree_extents Total extents of brick's domain in rtree format
@param brick_extents Size of brick
@param brick_active_size Size of brick (same rank as parameter)
@param origin Domain origin offset
@param bD Slice-friendly size of brick's domain
@return N/A
"""
log.debug('Writing virtual brick...')
# Set HDF5 file and group
# Create a GUID for the brick
brick_guid = create_guid()
brick_file_name = '{0}.hdf5'.format(brick_guid)
#TODO: Inclusion of external links only used for external viewing of master file, remove if non-performant
for parameter_name in self.parameter_metadata.keys():
self._add_brick_link(parameter_name, brick_guid, brick_file_name)
# Update the brick listing
log.debug('Updating brick list[%s] with (%s, %s, %s, %s)', brick_guid, brick_extents, origin, tuple(bD), brick_active_size)
brick_count = len(self.master_manager.brick_list)
self.master_manager.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size]
log.debug('Brick count is %s', brick_count)
# Insert into Rtree
log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid)
self.master_manager.update_rtree(brick_count, rtree_extents, obj=brick_guid)
def __init__(self, req_port, resp_port, name=None):
self.name=name or create_guid()
self.context = zmq.Context(1)
# Socket to get work from provisioner
self.req_sock = self.context.socket(zmq.REQ)
self.req_port = req_port
self.req_sock.connect('tcp://localhost:{0}'.format(self.req_port))
# Socket to respond to responder
self.resp_sock = self.context.socket(zmq.PUB)
self.resp_port = resp_port
self.resp_sock.connect('tcp://localhost:{0}'.format(self.resp_port))
self._do_stop = False
def _write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD, parameter_name):
"""
Creates a virtual brick in the PersistenceLayer by updating the HDF5 master file's
brick list, rtree and ExternalLink to where the HDF5 file will be saved in the future (lazy create)
@param rtree_extents Total extents of brick's domain in rtree format
@param brick_extents Size of brick
@param brick_active_size Size of brick (same rank as parameter)
@param origin Domain origin offset
@param bD Slice-friendly size of brick's domain
@param parameter_name Parameter name as string
@return N/A
"""
pm = self.parameter_metadata[parameter_name]
# rtree_extents, brick_extents, brick_active_size = self.calculate_extents(origin, bD, parameter_name)
#
# do_write, bguid = self._brick_exists(parameter_name, brick_extents)
# if not do_write:
# log.debug('Brick already exists! Updating brick metadata...')
# pm.brick_list[bguid] = [brick_extents, origin, tuple(bD), brick_active_size]
# else:
log.debug('Writing virtual brick for parameter %s', parameter_name)
# Set HDF5 file and group
# Create a GUID for the brick
brick_guid = create_guid()
brick_file_name = '{0}.hdf5'.format(brick_guid)
brick_rel_path = os.path.join(pm.root_dir.replace(self.root_dir,'.'), brick_file_name)
link_path = '/{0}/{1}'.format(parameter_name, brick_guid)
# Add brick to Master HDF file
self.master_manager.add_external_link(link_path, brick_rel_path, brick_guid)
# Update the brick listing
log.debug('Updating brick list[%s] with (%s, %s)', parameter_name, brick_guid, brick_extents)
brick_count = self.parameter_brick_count(parameter_name)
pm.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size]
log.debug('Brick count for %s is %s', parameter_name, brick_count)
# Insert into Rtree
log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid)
pm.update_rtree(brick_count, rtree_extents, obj=brick_guid)
def write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD, parameter_name):
pm = self.parameter_metadata[parameter_name]
# rtree_extents, brick_extents, brick_active_size = self.calculate_extents(origin, bD, parameter_name)
#
# do_write, bguid = self._brick_exists(parameter_name, brick_extents)
# if not do_write:
# log.debug('Brick already exists! Updating brick metadata...')
# pm.brick_list[bguid] = [brick_extents, origin, tuple(bD), brick_active_size]
# else:
log.debug('Writing virtual brick for parameter %s', parameter_name)
# Set HDF5 file and group
# Create a GUID for the brick
brick_guid = create_guid()
brick_file_name = '{0}.hdf5'.format(brick_guid)
brick_rel_path = os.path.join(pm.root_dir.replace(self.root_dir,'.'), brick_file_name)
link_path = '/{0}/{1}'.format(parameter_name, brick_guid)
# Add brick to Master HDF file
self.master_manager.add_external_link(link_path, brick_rel_path, brick_guid)
# Update the brick listing
log.debug('Updating brick list[%s] with (%s, %s)', parameter_name, brick_guid, brick_extents)
brick_count = self.parameter_brick_count(parameter_name)
pm.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size]
log.debug('Brick count for %s is %s', parameter_name, brick_count)
# Insert into Rtree
log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid)
pm.update_rtree(brick_count, rtree_extents, obj=brick_guid)
# Flush the parameter_metadata
if pm.is_dirty():
pm.flush()
if self.master_manager.is_dirty():
self.master_manager.flush()
