def __init__(self,
failure_callback,
num_workers=1,
pidantic_dir=None,
working_dir=None):
self.guid = create_guid()
self.prep_queue = queue.Queue()
self.work_queue = queue.Queue()
self._pending_work = {}
self._stashed_work = {}
self._active_work = {}
self._failures = {}
self._do_stop = False
self._count = -1
self._shutdown = False
self._failure_callback = failure_callback
self.context = zmq.Context(1)
self.prov_sock = self.context.socket(zmq.REP)
self.prov_port = self._get_port(self.prov_sock)
log.info('Provisioning url: tcp://*:{0}'.format(self.prov_port))
self.resp_sock = self.context.socket(zmq.SUB)
self.resp_port = self._get_port(self.resp_sock)
self.resp_sock.setsockopt(zmq.SUBSCRIBE, '')
log.info('Response url: tcp://*:{0}'.format(self.resp_port))
self.num_workers = num_workers if num_workers > 0 else 1
self.is_single_worker = self.num_workers == 1
self.working_dir = working_dir or '.'
self.pidantic_dir = pidantic_dir or './pid_dir'
self.workers = []
self._configure_workers()
def __init__(self, failure_callback, num_workers=1, pidantic_dir=None, working_dir=None):
self.guid = create_guid()
self.prep_queue = queue.Queue()
self.work_queue = queue.Queue()
self._pending_work = {}
self._stashed_work = {}
self._active_work = {}
self._failures = {}
self._do_stop = False
self._count = -1
self._shutdown = False
self._failure_callback = failure_callback
self.context = zmq.Context(1)
self.prov_sock = self.context.socket(zmq.REP)
self.prov_port = self._get_port(self.prov_sock)
log.info('Provisioning url: tcp://*:{0}'.format(self.prov_port))
self.resp_sock = self.context.socket(zmq.SUB)
self.resp_port = self._get_port(self.resp_sock)
self.resp_sock.setsockopt(zmq.SUBSCRIBE, '')
log.info('Response url: tcp://*:{0}'.format(self.resp_port))
self.num_workers = num_workers if num_workers > 0 else 1
self.is_single_worker = self.num_workers == 1
self.working_dir = working_dir or '.'
self.pidantic_dir = pidantic_dir or './pid_dir'
self.workers = []
self._configure_workers()
def test_force_unlock(self):
guid = utils.create_guid()
path = os.path.join(self.working_dir, '%s.h5' % guid)
f1 = HDFLockingFile(path, 'a')
HDFLockingFile.force_unlock(path)
f2 = HDFLockingFile(path, 'a')
def identifier(self):
"""
The UUID of the object. Generated on first request.
"""
if self._identifier is None:
self._identifier = create_guid()
return self._identifier
def identifier(self):
"""
The UUID of the object. Generated on first request.
"""
if self._identifier is None:
self._identifier = create_guid()
return self._identifier
def test_create_guid(self):
guid = utils.create_guid()
# Ensure the guid is a str
self.assertIsInstance(guid, basestring)
# Make sure it's properly formatted - this also tests is_guid
self.assertTrue(utils.is_guid(guid))
# Test that is_guid fails when appropriate
self.assertFalse(utils.is_guid(guid[:-1]))
def test_locking_contention(self):
guid = utils.create_guid()
path = os.path.join(self.working_dir, '%s.h5' % guid)
f1 = HDFLockingFile(path, 'a')
try:
f2 = HDFLockingFile(path,'a')
raise AssertionError('Failed to raise IOError for exclusion lock')
except IOError as e:
if 'Resource temporarily unavailable' not in e.message:
raise
f1.close()
def test_create_guid(self):
guid = utils.create_guid()
# Ensure the guid is a str
self.assertIsInstance(guid, basestring)
# Make sure it's properly formatted - this also tests is_guid
self.assertTrue(utils.is_guid(guid))
# Test that is_guid fails when appropriate
self.assertFalse(utils.is_guid(guid[:-1]))
def __init__(self, req_port, resp_port, name=None):
self.name = name or create_guid()
self.context = zmq.Context(1)
# Socket to get work from provisioner
self.req_sock = self.context.socket(zmq.REQ)
self.req_port = req_port
self.req_sock.connect('tcp://localhost:{0}'.format(self.req_port))
# Socket to respond to responder
self.resp_sock = self.context.socket(zmq.PUB)
self.resp_port = resp_port
self.resp_sock.connect('tcp://localhost:{0}'.format(self.resp_port))
self._do_stop = False
def __init__(self, req_port, resp_port, name=None):
self.name=name or create_guid()
self.context = zmq.Context(1)
# Socket to get work from provisioner
self.req_sock = self.context.socket(zmq.REQ)
self.req_port = req_port
self.req_sock.connect('tcp://localhost:{0}'.format(self.req_port))
# Socket to respond to responder
self.resp_sock = self.context.socket(zmq.PUB)
self.resp_port = resp_port
self.resp_sock.connect('tcp://localhost:{0}'.format(self.resp_port))
self._do_stop = False
def test_multiprocess_lock(self):
guid = utils.create_guid()
path = os.path.join(self.working_dir, '%s.h5' % guid)
main_read, child_write = mp.Pipe()
child_read, main_write = mp.Pipe()
p = mp.Process(target=child_lock, args=(path, child_read, child_write))
p.start()
wait_read(main_read.fileno())
print main_read.recv()
try:
f1 = HDFLockingFile(path, 'a')
raise AssertionError('Failed to raise IOError on locking')
except IOError:
pass
wait_write(main_write.fileno())
main_write.send('All done, thanks for the help')
p.join()
# if I lock it again even though the child is gone, it will fail on the cache
f1 = HDFLockingFile(path, 'a')
def _make_coverage(path):
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT])
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
cat = {0:'lemon',1:'apple',2:'banana',99:'None'}
cat_ctxt = ParameterContext('category', param_type=CategoryType(categories=cat))
cat_ctxt.long_name = "example of category"
pdict.add_context(cat_ctxt)
dens_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.float32))
dens_ctxt.uom = 'unknown'
dens_ctxt.fill_value = 0x0
pdict.add_context(dens_ctxt)
const_ctxt = ParameterContext('constant', param_type=ConstantType())
const_ctxt.long_name = 'example of a parameter of type ConstantType'
pdict.add_context(const_ctxt)
rec_ctxt = ParameterContext('boolean', param_type=BooleanType())
rec_ctxt.long_name = 'example of a parameter of type BooleanType'
pdict.add_context(rec_ctxt)
rec_ctxt = ParameterContext('range', param_type=ConstantRangeType())
rec_ctxt.long_name = 'Range example'
rec_ctxt.fill_value = 0x0
pdict.add_context(rec_ctxt)
rec_ctxt = ParameterContext('record', param_type=RecordType())
rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries'
rec_ctxt.fill_value = 0x0
pdict.add_context(rec_ctxt)
serial_ctxt = ParameterContext('array', param_type=ArrayType())
serial_ctxt.uom = 'unknown'
serial_ctxt.fill_value = 0x0
pdict.add_context(serial_ctxt)
guid = create_guid()
guid = guid.replace("-", "")
cov = SimplexCoverage(path, guid, name="sample_cov", parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
return (cov,path+os.sep+guid)
