def run_test_dispatcher(work_count, num_workers=1):
# Set up temporary directories to save data
import shutil
BASE_DIR = tempfile.mkdtemp()
PIDANTIC_DIR = tempfile.mkdtemp()
WORK_KEYS = ['a', 'b', 'c', 'd', 'e']
for x in [x for x in os.listdir(BASE_DIR) if x.endswith('.h5')]:
os.remove(os.path.join(BASE_DIR, x))
fps = {}
for k in WORK_KEYS:
fps[k] = os.path.join(BASE_DIR, '{0}.h5'.format(k))
# with h5py.File(fps[k], 'a'):
# pass
bD = (50, )
cD = (5, )
fv = -9999
dtype = 'f'
def fcb(message, work):
log.error('WORK DISCARDED!!!; %s: %s', message, work)
disp = BrickWriterDispatcher(fcb,
num_workers=num_workers,
pidantic_dir=PIDANTIC_DIR)
disp.run()
def make_work():
for x in xrange(work_count):
bk = random.choice(WORK_KEYS)
brick_metrics = (fps[bk], bD, cD, dtype, fv)
if np.random.random_sample(1)[0] > 0.5:
sl = int(np.random.randint(0, 10, 1)[0])
w = np.random.random_sample(1)[0]
else:
strt = int(np.random.randint(0, bD[0] - 2, 1)[0])
stp = int(np.random.randint(strt + 1, bD[0], 1)[0])
sl = slice(strt, stp)
w = np.random.random_sample(stp - strt)
disp.put_work(work_key=bk,
work_metrics=brick_metrics,
work=([sl], w))
time.sleep(0.1)
spawn(make_work)
# Remove temporary directories
shutil.rmtree(BASE_DIR)
shutil.rmtree(PIDANTIC_DIR)
return disp
def _es_call(es, *args, **kwargs):
res = AsyncResult()
def async_call(es, *args, **kwargs):
res.set(es(*args,**kwargs))
spawn(async_call,es,*args,**kwargs)
try:
retval = res.get(timeout=CFG.get_safe('server.elasticsearch.timeout', 10))
except Timeout:
raise exceptions.Timeout("Call to ElasticSearch timed out.")
return retval
def _es_call(es, *args, **kwargs):
res = AsyncResult()
def async_call(es, *args, **kwargs):
res.set(es(*args,**kwargs))
spawn(async_call,es,*args,**kwargs)
try:
retval = res.get(timeout=10)
except Timeout:
raise exceptions.Timeout("Call to ElasticSearch timed out.")
return retval
def get_ready_event(self):
"""
Returns an Event that is set when all the listeners in this Process are running.
"""
ev = Event()
def allready(ev):
waitall([x.get_ready_event() for x in self.listeners])
ev.set()
spawn(allready, ev)
return ev
def get_ready_event(self):
"""
Returns an Event that is set when all the listeners in this Process are running.
"""
ev = Event()
def allready(ev):
waitall([x.get_ready_event() for x in self.listeners])
ev.set()
spawn(allready, ev)
return ev
def get_dirty_values_async_result(self):
dirty_async_res = AsyncResult()
def dirty_check(self, res):
while True:
if self.is_dirty():
time.sleep(0.1)
else:
res.set(True)
break
spawn(dirty_check, self, dirty_async_res)
return dirty_async_res
def run_test_dispatcher(work_count, num_workers=1):
# Set up temporary directories to save data
import shutil
BASE_DIR = tempfile.mkdtemp()
PIDANTIC_DIR = tempfile.mkdtemp()
WORK_KEYS = ['a','b','c','d','e']
for x in [x for x in os.listdir(BASE_DIR) if x.endswith('.h5')]:
os.remove(os.path.join(BASE_DIR,x))
fps = {}
for k in WORK_KEYS:
fps[k] = os.path.join(BASE_DIR, '{0}.h5'.format(k))
# with h5py.File(fps[k], 'a'):
# pass
bD = (50,)
cD = (5,)
fv = -9999
dtype = 'f'
def fcb(message, work):
log.error('WORK DISCARDED!!!; %s: %s', message, work)
disp = BrickWriterDispatcher(fcb, num_workers=num_workers, pidantic_dir=PIDANTIC_DIR)
disp.run()
def make_work():
for x in xrange(work_count):
bk = random.choice(WORK_KEYS)
brick_metrics = (fps[bk], bD, cD, dtype, fv)
if np.random.random_sample(1)[0] > 0.5:
sl = int(np.random.randint(0,10,1)[0])
w = np.random.random_sample(1)[0]
else:
strt = int(np.random.randint(0,bD[0] - 2,1)[0])
stp = int(np.random.randint(strt+1,bD[0],1)[0])
sl = slice(strt, stp)
w = np.random.random_sample(stp-strt)
disp.put_work(work_key=bk, work_metrics=brick_metrics, work=([sl], w))
time.sleep(0.1)
spawn(make_work)
# Remove temporary directories
shutil.rmtree(BASE_DIR)
shutil.rmtree(PIDANTIC_DIR)
return disp
def get_dirty_values_async_result(self):
dirty_async_res = AsyncResult()
def dirty_check(self, res):
while True:
if self.is_dirty():
time.sleep(0.1)
else:
res.set(True)
break
spawn(dirty_check, self, dirty_async_res)
return dirty_async_res
def on_start(self):
# Persister thread
self._persist_greenlet = spawn(self._persister_loop, self.persist_interval)
log.debug('EventPersister persist greenlet started in "%s" (interval %s)', self.__class__.__name__, self.persist_interval)
# View trigger thread
self._refresh_greenlet = spawn(self._refresher_loop, self.refresh_interval)
log.debug('EventPersister view refresher greenlet started in "%s" (interval %s)', self.__class__.__name__, self.refresh_interval)
# Event subscription
self.event_sub = EventSubscriber(pattern=EventSubscriber.ALL_EVENTS,
callback=self._on_event,
queue_name="event_persister")
self.event_sub.start()
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event)
self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
def start(self):
self._leader_quit = Event()
if self.process:
self._leader_thread = self.process._process.thread_manager.spawn(
self._leader_loop)
else:
self._leader_thread = spawn(self._leader_loop)
def test_rpc_speed(self):
hsc = HelloServiceClient()
print >> sys.stderr, ""
self.counter = 0
self.alive = True
def sendem():
while self.alive:
hsc.noop('data')
self.counter += 1
start_time = time.time()
sendgl = spawn(sendem)
time.sleep(5)
end_time = time.time()
self.alive = False
sendgl.join(timeout=2)
sendgl.kill()
diff = end_time - start_time
mps = float(self.counter) / diff
print >> sys.stderr, "Requests per second (RPC):", mps, "(", self.counter, "messages in", diff, "seconds)"
def test_known_error(self):
# IonExceptions and TypeErrors get forwarded back intact
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def proc_call():
raise NotFound("didn't find it")
def client_call(p=None, ar=None):
try:
ca = p._routing_call(proc_call, None)
ca.get(timeout=5)
except IonException as e:
ar.set(e)
ar = AsyncResult()
gl_call = spawn(client_call, p=p, ar=ar)
e = ar.get(timeout=5)
self.assertIsInstance(e, NotFound)
def test_pub_speed(self):
pub = Publisher(node=self.container.node, name="i_no_exist")
print >>sys.stderr, ""
self.counter = 0
self.alive = True
def sendem():
while self.alive:
self.counter += 1
pub.publish('meh')
start_time = time.time()
sendgl = spawn(sendem)
time.sleep(5)
end_time = time.time()
self.alive = False
sendgl.join(timeout=2)
sendgl.kill()
diff = end_time - start_time
mps = float(self.counter) / diff
print >>sys.stderr, "Published messages per second:", mps, "(", self.counter, "messages in", diff, "seconds)"
def test_pub_speed(self):
pub = Publisher(node=self.container.node, name="i_no_exist")
print >> sys.stderr, ""
self.counter = 0
self.alive = True
def sendem():
while self.alive:
self.counter += 1
pub.publish('meh')
start_time = time.time()
sendgl = spawn(sendem)
time.sleep(5)
end_time = time.time()
self.alive = False
sendgl.join(timeout=2)
sendgl.kill()
diff = end_time - start_time
mps = float(self.counter) / diff
print >> sys.stderr, "Published messages per second:", mps, "(", self.counter, "messages in", diff, "seconds)"
def test_known_error(self):
# IonExceptions and TypeErrors get forwarded back intact
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def proc_call():
raise NotFound("didn't find it")
def client_call(p=None, ar=None):
try:
ca = p._routing_call(proc_call, None)
ca.get(timeout=5)
except IonException as e:
ar.set(e)
ar = AsyncResult()
gl_call = spawn(client_call, p=p, ar=ar)
e = ar.get(timeout=5)
self.assertIsInstance(e, NotFound)
def test_rpc_with_xn(self):
# get an xn to use for send/recv
xn = self.container.ex_manager.create_xn_service('hello')
self.addCleanup(xn.delete)
# create an RPCServer for a hello service
hs = HelloService()
rpcs = RPCServer(from_name=xn, service=hs)
# spawn the listener, kill on test exit (success/fail/error should cover?)
gl_listen = spawn(rpcs.listen)
def cleanup():
rpcs.close()
gl_listen.join(timeout=2)
gl_listen.kill()
self.addCleanup(cleanup)
# wait for listen to be ready
rpcs.get_ready_event().wait(timeout=5)
# ok, now create a client using same xn
hsc = HelloServiceClient(to_name=xn)
# try to message it!
ret = hsc.hello('hi there')
# did we get back what we expected?
self.assertEquals(ret, 'BACK:hi there')
def test_rpc_speed(self):
hsc = HelloServiceClient()
print >>sys.stderr, ""
self.counter = 0
self.alive = True
def sendem():
while self.alive:
hsc.noop('data')
self.counter += 1
start_time = time.time()
sendgl = spawn(sendem)
time.sleep(5)
end_time = time.time()
self.alive = False
sendgl.join(timeout=2)
sendgl.kill()
diff = end_time - start_time
mps = float(self.counter) / diff
print >>sys.stderr, "Requests per second (RPC):", mps, "(", self.counter, "messages in", diff, "seconds)"
def start(self):
queue_name = get_safe(self._pd_core.pd_cfg, "command_queue") or "pd_command"
self.sub_cont = Subscriber(binding=queue_name, from_name=queue_name, callback=self._receive_command)
self.sub_cont_gl = spawn(self.sub_cont.listen, activate=False)
self.sub_cont.get_ready_event().wait()
self.pub_result = Publisher()
def test_rpc_with_xn(self):
# get an xn to use for send/recv
xn = self.container.ex_manager.create_xn_service('hello')
self.addCleanup(xn.delete)
# create an RPCServer for a hello service
hs = HelloService()
rpcs = RPCServer(from_name=xn, service=hs)
# spawn the listener, kill on test exit (success/fail/error should cover?)
gl_listen = spawn(rpcs.listen)
def cleanup():
rpcs.close()
gl_listen.join(timeout=2)
gl_listen.kill()
self.addCleanup(cleanup)
# wait for listen to be ready
rpcs.get_ready_event().wait(timeout=5)
# ok, now create a client using same xn
hsc = HelloServiceClient(to_name=xn)
# try to message it!
ret = hsc.hello('hi there')
# did we get back what we expected?
self.assertEquals(ret, 'BACK:hi there')
def execute_acquire_data(self, *args):
"""
Spawns a greenlet to perform a data acquisition
Calls BaseDataHandler._acquire_data
Disallows multiple "new data" (unconstrained) requests using BaseDataHandler._semaphore lock
Called from:
InstrumentAgent._handler_observatory_execute_resource
|--> ExternalDataAgent._handler_streaming_execute_resource
@parameter args First argument should be a config dictionary
"""
try:
config = args[0]
except IndexError:
raise ParameterError('\'acquire_data\' command requires a config dict.')
if not isinstance(config, dict):
raise TypeError('args[0] of \'acquire_data\' is not a dict.')
else:
if get_safe(config,'constraints') is None and not self._semaphore.acquire(blocking=False):
log.warn('Already acquiring new data - action not duplicated')
return
g = spawn(self._acquire_data, config, self._unlock_new_data_callback)
log.debug('** Spawned {0}'.format(g))
self._glet_queue.append(g)
def test_unknown_error(self):
# Unhandled exceptions get handled and then converted to ContainerErrors
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def proc_call():
raise self.ExpectedError("didn't find it")
def client_call(p=None, ar=None):
try:
ca = p._routing_call(proc_call, None)
ca.get(timeout=5)
except IonException as e:
ar.set(e)
ar = AsyncResult()
gl_call = spawn(client_call, p=p, ar=ar)
e = ar.get(timeout=5)
self.assertIsInstance(e, ContainerError)
self.assertEquals(len(p._errors), 1)
def on_start(self):
super(IngestionWorker,self).on_start()
#----------------------------------------------
# Start up couch
#----------------------------------------------
self.couch_config = self.CFG.get('couch_storage')
self.hdf_storage = self.CFG.get('hdf_storage')
self.number_of_workers = self.CFG.get('number_of_workers')
self.description = self.CFG.get('description')
self.ingest_config_id = self.CFG.get('configuration_id')
self.datastore_name = self.couch_config.get('datastore_name',None) or 'dm_datastore'
try:
self.datastore_profile = getattr(DataStore.DS_PROFILE, self.couch_config.get('datastore_profile','SCIDATA'))
except AttributeError:
log.exception('Invalid datastore profile passed to ingestion worker. Defaulting to SCIDATA')
self.datastore_profile = DataStore.DS_PROFILE.SCIDATA
log.debug('datastore_profile %s' % self.datastore_profile)
self.db = self.container.datastore_manager.get_datastore(ds_name=self.datastore_name, profile = self.datastore_profile, config = self.CFG)
self.resource_reg_client = ResourceRegistryServiceClient(node = self.container.node)
self.dataset_configs = {}
# update the policy
def receive_dataset_config_event(event_msg, headers):
log.info('Updating dataset config in ingestion worker: %s', event_msg)
if event_msg.type != DatasetIngestionTypeEnum.DATASETINGESTIONBYSTREAM:
raise IngestionWorkerException('Received invalid type in dataset config event.')
stream_id = event_msg.configuration.stream_id
if event_msg.deleted:
try:
del self.dataset_configs[stream_id]
except KeyError:
log.info('Tried to remove dataset config that does not exist!')
else:
self.dataset_configs[stream_id] = event_msg
# Hook to override just before processing is complete
self.dataset_configs_event_test_hook(event_msg, headers)
#Start the event subscriber - really - what a mess!
self.event_subscriber = EventSubscriber(
event_type="DatasetIngestionConfigurationEvent",
origin=self.ingest_config_id,
callback=receive_dataset_config_event
)
self.gl = spawn(self.event_subscriber.listen)
self.event_subscriber._ready_event.wait(timeout=5)
log.info(str(self.db))
def _spawn(self):
""" Spawn a gevent greenlet using defined target method and args.
"""
gl = spawn(self.target, *self.spawn_args, **self.spawn_kwargs)
gl.link(lambda _: self.ev_exit.set()) # Set exit event when we terminate
gl._glname = "ION Thread %s" % str(self.target)
return gl
def test_unknown_error(self):
# Unhandled exceptions get handled and then converted to ContainerErrors
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
def proc_call():
raise self.ExpectedError("didn't find it")
def client_call(p=None, ar=None):
try:
ca = p._routing_call(proc_call, None)
ca.get(timeout=5)
except IonException as e:
ar.set(e)
ar = AsyncResult()
gl_call = spawn(client_call, p=p, ar=ar)
e = ar.get(timeout=5)
self.assertIsInstance(e, ContainerError)
self.assertEquals(len(p._errors), 1)
def execute_acquire_data(self, *args):
"""
Creates a copy of self._dh_config, creates a publisher, and spawns a greenlet to perform a data acquisition cycle
If the args[0] is a dict, any entries keyed with one of the 'PATCHABLE_CONFIG_KEYS' are used to patch the config
Greenlet binds to BaseDataHandler._acquire_data and passes the publisher and config
Disallows multiple "new data" (unconstrained) requests using BaseDataHandler._semaphore lock
Called from:
InstrumentAgent._handler_observatory_execute_resource
|--> ExternalDataAgent._handler_streaming_execute_resource
@parameter args First argument can be a config dictionary
"""
log.debug('Executing acquire_data: args = {0}'.format(args))
# Make a copy of the config to ensure no cross-pollution
config = self._dh_config.copy()
# Patch the config if mods are passed in
try:
config_mods = args[0]
if not isinstance(config_mods, dict):
raise IndexError()
log.debug('Configuration modifications provided: {0}'.format(config_mods))
for k in self._params['PATCHABLE_CONFIG_KEYS']:
p=get_safe(config_mods, k)
if not p is None:
config[k] = p
except IndexError:
log.info('No configuration modifications were provided')
# Verify that there is a stream_id member in the config
stream_id = get_safe(config, 'stream_id')
if not stream_id:
raise ConfigurationError('Configuration does not contain required \'stream_id\' member')
isNew = get_safe(config, 'constraints') is None
if isNew and not self._semaphore.acquire(blocking=False):
log.warn('Already acquiring new data - action not duplicated')
return
ndc = None
if isNew:
# Get the NewDataCheck attachment and add it's content to the config
ext_ds_id = get_safe(config,'external_dataset_res_id')
if ext_ds_id:
ndc = self._find_new_data_check_attachment(ext_ds_id)
config['new_data_check'] = ndc
# Create a publisher to pass into the greenlet
publisher = self._stream_registrar.create_publisher(stream_id=stream_id)
# Spawn a greenlet to do the data acquisition and publishing
g = spawn(self._acquire_data, config, publisher, self._unlock_new_data_callback, self._update_new_data_check_attachment)
log.debug('** Spawned {0}'.format(g))
self._glet_queue.append(g)
def on_start(self):
# Persister thread
self._persist_greenlet = spawn(self._trigger_func, self.persist_interval)
log.debug('Publisher Greenlet started in "%s"' % self.__class__.__name__)
# Event subscription
self.event_sub = EventSubscriber(pattern=EventSubscriber.ALL_EVENTS, callback=self._on_event)
self.event_sub.start()
def on_start_streaming(self, streaming_args=None):
self.sampling_gl_quit = Event()
self.sampling_interval = self.agent_config.get("sampling_interval", 5)
self.sampling_gl = spawn(self._sample_data_loop,
self.sampling_interval)
if self.agent_plugin and hasattr(self.agent_plugin,
'on_start_streaming'):
self.agent_plugin.on_start_streaming(streaming_args)
def start(self):
self._do_stop = False
self._g = spawn(self._run, self.name)
log.info(
'Brick writer worker \'%s\' started: req_port=%s, resp_port=%s',
self.name, 'tcp://localhost:{0}'.format(self.req_port),
'tcp://localhost:{0}'.format(self.resp_port))
return self._g
def _spawn(self):
""" Spawn a gevent greenlet using defined target method and args.
"""
gl = spawn(self.target, *self.spawn_args, **self.spawn_kwargs)
gl.link(
lambda _: self.ev_exit.set()) # Set exit event when we terminate
gl._glname = "ION Thread %s" % str(self.target)
return gl
def start(self):
log.debug("SFlowManager.start")
if self._counter_interval > 0:
self._gl_counter = spawn(self._counter)
else:
log.debug("Counter interval is 0, not spawning counter greenlet")
self._udp_socket = socket(AF_INET, SOCK_DGRAM)
def start(self):
"""
Starts all internal greenlets of this router device.
"""
self._queue_incoming = Queue()
self._gl_msgs = self._gl_pool.spawn(self._run_gl_msgs)
self._gl_msgs.link_exception(self._child_failed)
self.gl_ioloop = spawn(self._run_ioloop)
def activate(self):
if self.activated:
raise BadRequest("Already active")
self.listen_gl = spawn(self.wait_sub.listen
) # This initializes and activates the listener
self.wait_sub.get_ready_event().wait(timeout=1)
self.activated = True
return self.wait_name
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._listens.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def start(self):
log.debug("SFlowManager.start")
if self._counter_interval > 0:
self._gl_counter = spawn(self._counter)
else:
log.debug("Counter interval is 0, not spawning counter greenlet")
self._udp_socket = socket(AF_INET, SOCK_DGRAM)
def start(self):
"""
Starts all internal greenlets of this router device.
"""
self._queue_incoming = Queue()
self._gl_msgs = self._gl_pool.spawn(self._run_gl_msgs)
self._gl_msgs.link_exception(self._child_failed)
self.gl_ioloop = spawn(self._run_ioloop)
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._listens.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def insert_timesteps(self, count, origin=None, oob=True):
"""
Insert count # of timesteps beginning at the origin
The specified # of timesteps are inserted into the temporal value array at the indicated origin. This also
expands the temporal dimension of the AbstractParameterValue for each parameters
@param count The number of timesteps to insert
@param origin The starting location, from which to begin the insertion
@param oob Out of band operations, True will use greenlets, False will be in-band.
"""
if self.closed:
raise IOError('I/O operation on closed file')
if self.mode == 'r':
raise IOError('Coverage not open for writing: mode == \'{0}\''.format(self.mode))
# Get the current shape of the temporal_dimension
shp = self.temporal_domain.shape
# If not provided, set the origin to the end of the array
if origin is None or not isinstance(origin, int):
origin = shp.extents[0]
# Expand the shape of the temporal_domain - following works if extents is a list or tuple
shp.extents = (shp.extents[0]+count,)+tuple(shp.extents[1:])
# Expand the temporal dimension of each of the parameters - the parameter determines how to apply the change
for n in self._range_dictionary:
pc = self._range_dictionary.get_context(n)
# Update the dom of the parameter_context
if pc.dom.tdom is not None:
pc.dom.tdom = self.temporal_domain.shape.extents
self._persistence_layer.expand_domain(pc)
self._range_value[n].expand_content(VariabilityEnum.TEMPORAL, origin, count)
# Update the temporal_domain in the master_manager, do NOT flush!!
self._persistence_layer.update_domain(tdom=self.temporal_domain, do_flush=False)
# Flush the master_manager & parameter_managers in a separate greenlet
if oob:
spawn(self._persistence_layer.flush)
else:
self._persistence_layer.flush()
def start(self):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
assert not self._cbthread, "start called twice on ConversationSubscriber"
gl = spawn(self.listen)
self._cbthread = gl
self._ready_event.wait(timeout=5)
log.info("ConversationSubscriber started; pattern=%s" % self.binding)
return gl
def activate(self):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
assert not self._cbthread, "activate called twice on EventSubscriber"
gl = spawn(self.listen)
self._cbthread = gl
self._ready_event.wait(timeout=5)
log.info("EventSubscriber activated. Event pattern=%s" % self.binding)
return gl
def activate(self):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
assert not self._cbthread, "activate called twice on EventSubscriber"
gl = spawn(self.listen)
self._cbthread = gl
self._ready_event.wait(timeout=5)
log.info("EventSubscriber activated. Event pattern=%s" % self.binding)
return gl
def on_start(self):
# Persister thread
self._persist_greenlet = spawn(self._trigger_func, self.persist_interval)
log.debug('EventPersister timer greenlet started in "%s" (interval %s)', self.__class__.__name__, self.persist_interval)
# Event subscription
self.event_sub = EventSubscriber(pattern=EventSubscriber.ALL_EVENTS,
callback=self._on_event,
queue_name="event_persister")
self.event_sub.start()
def start(self):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
assert not self._cbthread, "start called twice on EventSubscriber"
gl = spawn(self.listen)
self._cbthread = gl
if not self._ready_event.wait(timeout=5):
log.warning('EventSubscriber start timed out.')
log.info("EventSubscriber started. Event pattern=%s" % self.binding)
return gl
def on_start(self):
# Persister thread
self._persist_greenlet = spawn(self._trigger_func, self.persist_interval)
log.debug('Publisher Greenlet started in "%s"' % self.__class__.__name__)
# Conv subscription to as many as it takes
self.conv_sub = ConvSubscriber(callback=self._on_message)
self.conv_sub.start()
# Open repository
self.conv_repository = ConvRepository()
def start(self):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
assert not self._cbthread, "start called twice on EventSubscriber"
gl = spawn(self.listen)
self._cbthread = gl
if not self._ready_event.wait(timeout=5):
log.warning('EventSubscriber start timed out.')
log.info("EventSubscriber started. Event pattern=%s" % self.binding)
return gl
def start(self):
"""
Start consuming from the queue
"""
if self._chan is not None:
try:
self._chan.start_consume()
except ChannelError:
log.info('Subscriber is already started')
else:
self.gl = spawn(self.listen)
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg,
"aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (
topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic,
from_name=queue_name,
auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent,
callback=self._receive_event)
self.evt_sub.add_event_subscription(
event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
def on_start(self):
TransformDataProcess.on_start(self)
# set up subscriber to *
self._bt_sub = Subscriber(callback=lambda m, h: self.call_process(m),
from_name=NameTrio(get_sys_name(), 'bench_queue', '*'))
# spawn listener
self._sub_gl = spawn(self._bt_sub.listen)
# set up publisher to anything!
self._bt_pub = Publisher(to_name=NameTrio(get_sys_name(), str(uuid.uuid4())[0:6]))
def start(self):
"""
Start consuming from the queue
"""
if hasattr(self, '_chan'):
try:
self._chan.start_consume()
except ChannelError:
log.info('Subscriber is already started')
else:
self.gl = spawn(self.listen)
def start(self):
from pyon.net.endpoint import Publisher
from pyon.util.async import spawn
self.heartbeat_quit = Event()
self.heartbeat_interval = float(self.heartbeat_cfg.get("publish_interval", 60))
self.heartbeat_topic = self.heartbeat_cfg.get("topic", "heartbeat")
self.heartbeat_pub = Publisher(to_name=self.heartbeat_topic)
# Directly spawn a greenlet - we don't want this to be a supervised IonProcessThread
self.heartbeat_gl = spawn(self.heartbeat_loop)
self.started = True
log.info("Started container heartbeat (interval=%s, topic=%s)", self.heartbeat_interval, self.heartbeat_topic)
def run_test_dispatcher(work_count, num_workers=1):
BASE_DIR = 'test_data/masonry'
WORK_KEYS = ['a','b','c','d','e']
for x in [x for x in os.listdir(BASE_DIR) if x.endswith('.h5')]:
os.remove(os.path.join(BASE_DIR,x))
fps = {}
for k in WORK_KEYS:
fps[k] = os.path.join(BASE_DIR, '{0}.h5'.format(k))
# with h5py.File(fps[k], 'a'):
# pass
bD = (50,)
cD = (5,)
fv = -9999
dtype = 'f'
disp = BrickWriterDispatcher(num_workers=num_workers, pidantic_dir='test_data/pid')
disp.run()
def make_work():
for x in xrange(work_count):
bk = random.choice(WORK_KEYS)
brick_metrics = (fps[bk], bD, cD, dtype, fv)
if np.random.random_sample(1)[0] > 0.5:
sl = int(np.random.randint(0,10,1)[0])
w = np.random.random_sample(1)[0]
else:
strt = int(np.random.randint(0,bD[0] - 2,1)[0])
stp = int(np.random.randint(strt+1,bD[0],1)[0])
sl = slice(strt, stp)
w = np.random.random_sample(stp-strt)
disp.put_work(work_key=bk, work_metrics=brick_metrics, work=([sl], w))
time.sleep(0.1)
spawn(make_work)
return disp
def start_listeners(self):
"""
Starts all listeners in managed greenlets.
This must be called after starting this IonProcess. Currently, the Container's ProcManager
will handle this for you, but if using an IonProcess manually, you must remember to call
this method or no attached listeners will run.
"""
# spawn all listeners in startup listeners (from initializer, or added later)
for listener in self._startup_listeners:
self.add_endpoint(listener)
ev = Event()
def allready(ev):
waitall([x.get_ready_event() for x in self.listeners])
ev.set()
spawn(allready, ev)
ev.wait(timeout=10)
def on_start(self):
# Persister thread
self._persist_greenlet = spawn(self._trigger_func,
self.persist_interval)
log.debug('Publisher Greenlet started in "%s"' %
self.__class__.__name__)
# Conv subscription to as many as it takes
self.conv_sub = ConvSubscriber(callback=self._on_message)
self.conv_sub.start()
# Open repository
self.conv_repository = ConvRepository()
def start(self):
from pyon.net.endpoint import Publisher
from pyon.util. async import spawn
self.heartbeat_quit = Event()
self.heartbeat_interval = float(
self.heartbeat_cfg.get("publish_interval", 60))
self.heartbeat_topic = self.heartbeat_cfg.get("topic", "heartbeat")
self.heartbeat_pub = Publisher(to_name=self.heartbeat_topic)
# Directly spawn a greenlet - we don't want this to be a supervised IonProcessThread
self.heartbeat_gl = spawn(self.heartbeat_loop)
self.started = True
log.info("Started container heartbeat (interval=%s, topic=%s)",
self.heartbeat_interval, self.heartbeat_topic)
def on_start(self):
TransformDataProcess.on_start(self)
# set up subscriber to *
self._bt_sub = Subscriber(callback=lambda m, h: self.call_process(m),
from_name=NameTrio(get_sys_name(),
'bench_queue', '*'))
# spawn listener
self._sub_gl = spawn(self._bt_sub.listen)
# set up publisher to anything!
self._bt_pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
def spawn_listener(self):
def client_recv():
while True:
try:
log.debug("client_recv waiting for a message")
msg, headers, delivery_tag = self.channel.recv()
log.debug("client_recv got a message")
log_message(self.channel._send_name , msg, headers, delivery_tag)
try:
self._message_received(msg, headers)
finally:
# always ack a listener response
self.channel.ack(delivery_tag)
except ChannelClosedError:
log.debug('Channel was closed during client_recv listen loop')
break
# @TODO: spawn should be configurable to maybe the proc_sup in the container?
self._recv_greenlet = spawn(client_recv)
def execute_start_autosample(self, *args, **kwargs):
#TODO: Add documentation
#TODO: Fix raises statements
"""
Put the DataHandler into streaming mode and start polling for new data
Called from:
InstrumentAgent._handler_observatory_go_streaming
@raises InstrumentTimeoutException:
@raises InstrumentProtocolException:
@raises NotImplementedException:
@raises InstrumentParameterException:
@retval Next ResourceAgentState (STREAMING)
"""
log.debug('Entered execute_start_autosample with args={0} & kwargs={1}'.format(args, kwargs))
if not self._polling and self._polling_glet is None:
self._polling_glet = spawn(self._poll)
return ResourceAgentState.STREAMING, None
def _leader_callback(self, leader_info):
if leader_info["action"] == "acquire_leader":
def start_sub():
if not self.registry.preconditions_true.is_set():
log.info("PD is leader - awaiting PD preconditions")
# Await preconditions
await_timeout = get_safe(self._pd_core.pd_cfg, "engine.await_preconditions.await_timeout")
precond_true = self.registry.preconditions_true.wait(timeout=await_timeout)
if not precond_true:
log.warn("PD preconditions not satisfied after timeout - continuing")
if self._pd_core.is_leader() and self.sub_cont is not None and not self.sub_active:
# Are we still leader? Not activated?
num_msg, num_cons = self.sub_cont.get_stats()
log.info("PD is leader - starting to consume (%s pending commands, %s consumers)", num_msg, num_cons)
self.sub_cont.activate()
self.sub_active = True
start_sub_gl = spawn(start_sub)
elif leader_info["action"] == "release_leader":
if self.sub_cont is not None and self.sub_active:
self.sub_cont.deactivate()
self.sub_active = False