def __init__(self, num_workers=1, pidantic_dir=None, working_dir=None): self.guid = create_guid() self.prep_queue = queue.Queue() self.work_queue = queue.Queue() self._pending_work = {} self._stashed_work = {} self._active_work = {} self._do_stop = False self._count = 0 self._active_work_lock = coros.RLock() self._pending_work_lock = coros.RLock() self._shutdown = False self.context = zmq.Context(1) self.prov_sock = self.context.socket(zmq.REP) self.prov_port = self._get_port(self.prov_sock) log.info('Provisioning url: tcp://*:{0}'.format(self.prov_port)) self.resp_sock = self.context.socket(zmq.SUB) self.resp_port = self._get_port(self.resp_sock) self.resp_sock.setsockopt(zmq.SUBSCRIBE, '') log.info('Response url: tcp://*:{0}'.format(self.resp_port)) self.num_workers = num_workers if num_workers > 0 else 1 self.is_single_worker = self.num_workers == 1 self.working_dir = working_dir or '.' self.pidantic_dir = pidantic_dir or './pid_dir' self.workers = [] self._configure_workers()
def _write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD): """ Creates a virtual brick in the PersistenceLayer by updating the HDF5 master file's brick list, rtree and ExternalLink to where the HDF5 file will be saved in the future (lazy create) @param rtree_extents Total extents of brick's domain in rtree format @param brick_extents Size of brick @param brick_active_size Size of brick (same rank as parameter) @param origin Domain origin offset @param bD Slice-friendly size of brick's domain @return N/A """ log.debug('Writing virtual brick...') # Set HDF5 file and group # Create a GUID for the brick brick_guid = create_guid() brick_file_name = '{0}.hdf5'.format(brick_guid) #TODO: Inclusion of external links only used for external viewing of master file, remove if non-performant for parameter_name in self.parameter_metadata.keys(): self._add_brick_link(parameter_name, brick_guid, brick_file_name) # Update the brick listing log.debug('Updating brick list[%s] with (%s, %s, %s, %s)', brick_guid, brick_extents, origin, tuple(bD), brick_active_size) brick_count = len(self.master_manager.brick_list) self.master_manager.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size] log.debug('Brick count is %s', brick_count) # Insert into Rtree log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid) self.master_manager.update_rtree(brick_count, rtree_extents, obj=brick_guid)
def __init__(self, req_port, resp_port, name=None): self.name=name or create_guid() self.context = zmq.Context(1) # Socket to get work from provisioner self.req_sock = self.context.socket(zmq.REQ) self.req_port = req_port self.req_sock.connect('tcp://localhost:{0}'.format(self.req_port)) # Socket to respond to responder self.resp_sock = self.context.socket(zmq.PUB) self.resp_port = resp_port self.resp_sock.connect('tcp://localhost:{0}'.format(self.resp_port)) self._do_stop = False
def _write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD, parameter_name): """ Creates a virtual brick in the PersistenceLayer by updating the HDF5 master file's brick list, rtree and ExternalLink to where the HDF5 file will be saved in the future (lazy create) @param rtree_extents Total extents of brick's domain in rtree format @param brick_extents Size of brick @param brick_active_size Size of brick (same rank as parameter) @param origin Domain origin offset @param bD Slice-friendly size of brick's domain @param parameter_name Parameter name as string @return N/A """ pm = self.parameter_metadata[parameter_name] # rtree_extents, brick_extents, brick_active_size = self.calculate_extents(origin, bD, parameter_name) # # do_write, bguid = self._brick_exists(parameter_name, brick_extents) # if not do_write: # log.debug('Brick already exists! Updating brick metadata...') # pm.brick_list[bguid] = [brick_extents, origin, tuple(bD), brick_active_size] # else: log.debug('Writing virtual brick for parameter %s', parameter_name) # Set HDF5 file and group # Create a GUID for the brick brick_guid = create_guid() brick_file_name = '{0}.hdf5'.format(brick_guid) brick_rel_path = os.path.join(pm.root_dir.replace(self.root_dir,'.'), brick_file_name) link_path = '/{0}/{1}'.format(parameter_name, brick_guid) # Add brick to Master HDF file self.master_manager.add_external_link(link_path, brick_rel_path, brick_guid) # Update the brick listing log.debug('Updating brick list[%s] with (%s, %s)', parameter_name, brick_guid, brick_extents) brick_count = self.parameter_brick_count(parameter_name) pm.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size] log.debug('Brick count for %s is %s', parameter_name, brick_count) # Insert into Rtree log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid) pm.update_rtree(brick_count, rtree_extents, obj=brick_guid)
def write_brick(self, rtree_extents, brick_extents, brick_active_size, origin, bD, parameter_name): pm = self.parameter_metadata[parameter_name] # rtree_extents, brick_extents, brick_active_size = self.calculate_extents(origin, bD, parameter_name) # # do_write, bguid = self._brick_exists(parameter_name, brick_extents) # if not do_write: # log.debug('Brick already exists! Updating brick metadata...') # pm.brick_list[bguid] = [brick_extents, origin, tuple(bD), brick_active_size] # else: log.debug('Writing virtual brick for parameter %s', parameter_name) # Set HDF5 file and group # Create a GUID for the brick brick_guid = create_guid() brick_file_name = '{0}.hdf5'.format(brick_guid) brick_rel_path = os.path.join(pm.root_dir.replace(self.root_dir,'.'), brick_file_name) link_path = '/{0}/{1}'.format(parameter_name, brick_guid) # Add brick to Master HDF file self.master_manager.add_external_link(link_path, brick_rel_path, brick_guid) # Update the brick listing log.debug('Updating brick list[%s] with (%s, %s)', parameter_name, brick_guid, brick_extents) brick_count = self.parameter_brick_count(parameter_name) pm.brick_list[brick_guid] = [brick_extents, origin, tuple(bD), brick_active_size] log.debug('Brick count for %s is %s', parameter_name, brick_count) # Insert into Rtree log.debug('Inserting into Rtree %s:%s:%s', brick_count, rtree_extents, brick_guid) pm.update_rtree(brick_count, rtree_extents, obj=brick_guid) # Flush the parameter_metadata if pm.is_dirty(): pm.flush() if self.master_manager.is_dirty(): self.master_manager.flush()