def __init__(self, failure_callback, num_workers=1, pidantic_dir=None, working_dir=None): self.guid = create_guid() self.prep_queue = queue.Queue() self.work_queue = queue.Queue() self._pending_work = {} self._stashed_work = {} self._active_work = {} self._failures = {} self._do_stop = False self._count = -1 self._shutdown = False self._failure_callback = failure_callback self.context = zmq.Context(1) self.prov_sock = self.context.socket(zmq.REP) self.prov_port = self._get_port(self.prov_sock) log.info('Provisioning url: tcp://*:{0}'.format(self.prov_port)) self.resp_sock = self.context.socket(zmq.SUB) self.resp_port = self._get_port(self.resp_sock) self.resp_sock.setsockopt(zmq.SUBSCRIBE, '') log.info('Response url: tcp://*:{0}'.format(self.resp_port)) self.num_workers = num_workers if num_workers > 0 else 1 self.is_single_worker = self.num_workers == 1 self.working_dir = working_dir or '.' self.pidantic_dir = pidantic_dir or './pid_dir' self.workers = [] self._configure_workers()
def test_force_unlock(self): guid = utils.create_guid() path = os.path.join(self.working_dir, '%s.h5' % guid) f1 = HDFLockingFile(path, 'a') HDFLockingFile.force_unlock(path) f2 = HDFLockingFile(path, 'a')
def identifier(self): """ The UUID of the object. Generated on first request. """ if self._identifier is None: self._identifier = create_guid() return self._identifier
def test_create_guid(self): guid = utils.create_guid() # Ensure the guid is a str self.assertIsInstance(guid, basestring) # Make sure it's properly formatted - this also tests is_guid self.assertTrue(utils.is_guid(guid)) # Test that is_guid fails when appropriate self.assertFalse(utils.is_guid(guid[:-1]))
def test_locking_contention(self): guid = utils.create_guid() path = os.path.join(self.working_dir, '%s.h5' % guid) f1 = HDFLockingFile(path, 'a') try: f2 = HDFLockingFile(path,'a') raise AssertionError('Failed to raise IOError for exclusion lock') except IOError as e: if 'Resource temporarily unavailable' not in e.message: raise f1.close()
def __init__(self, req_port, resp_port, name=None): self.name = name or create_guid() self.context = zmq.Context(1) # Socket to get work from provisioner self.req_sock = self.context.socket(zmq.REQ) self.req_port = req_port self.req_sock.connect('tcp://localhost:{0}'.format(self.req_port)) # Socket to respond to responder self.resp_sock = self.context.socket(zmq.PUB) self.resp_port = resp_port self.resp_sock.connect('tcp://localhost:{0}'.format(self.resp_port)) self._do_stop = False
def __init__(self, req_port, resp_port, name=None): self.name=name or create_guid() self.context = zmq.Context(1) # Socket to get work from provisioner self.req_sock = self.context.socket(zmq.REQ) self.req_port = req_port self.req_sock.connect('tcp://localhost:{0}'.format(self.req_port)) # Socket to respond to responder self.resp_sock = self.context.socket(zmq.PUB) self.resp_port = resp_port self.resp_sock.connect('tcp://localhost:{0}'.format(self.resp_port)) self._do_stop = False
def test_multiprocess_lock(self): guid = utils.create_guid() path = os.path.join(self.working_dir, '%s.h5' % guid) main_read, child_write = mp.Pipe() child_read, main_write = mp.Pipe() p = mp.Process(target=child_lock, args=(path, child_read, child_write)) p.start() wait_read(main_read.fileno()) print main_read.recv() try: f1 = HDFLockingFile(path, 'a') raise AssertionError('Failed to raise IOError on locking') except IOError: pass wait_write(main_write.fileno()) main_write.send('All done, thanks for the help') p.join() # if I lock it again even though the child is gone, it will fail on the cache f1 = HDFLockingFile(path, 'a')
def _make_coverage(path): tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT]) tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) cat = {0:'lemon',1:'apple',2:'banana',99:'None'} cat_ctxt = ParameterContext('category', param_type=CategoryType(categories=cat)) cat_ctxt.long_name = "example of category" pdict.add_context(cat_ctxt) dens_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) const_ctxt = ParameterContext('constant', param_type=ConstantType()) const_ctxt.long_name = 'example of a parameter of type ConstantType' pdict.add_context(const_ctxt) rec_ctxt = ParameterContext('boolean', param_type=BooleanType()) rec_ctxt.long_name = 'example of a parameter of type BooleanType' pdict.add_context(rec_ctxt) rec_ctxt = ParameterContext('range', param_type=ConstantRangeType()) rec_ctxt.long_name = 'Range example' rec_ctxt.fill_value = 0x0 pdict.add_context(rec_ctxt) rec_ctxt = ParameterContext('record', param_type=RecordType()) rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries' rec_ctxt.fill_value = 0x0 pdict.add_context(rec_ctxt) serial_ctxt = ParameterContext('array', param_type=ArrayType()) serial_ctxt.uom = 'unknown' serial_ctxt.fill_value = 0x0 pdict.add_context(serial_ctxt) guid = create_guid() guid = guid.replace("-", "") cov = SimplexCoverage(path, guid, name="sample_cov", parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) return (cov,path+os.sep+guid)