def test_xp_durable_send(self):
xp = self.container.ex_manager.create_xp('an_xp')
#self.addCleanup(xp.delete)
xq = self.container.ex_manager.create_xn_queue('no_matter', xp)
self.addCleanup(xq.delete)
xq.bind('one')
pub = Publisher(to_name=xp.create_route('one'))
pub.publish('test')
pub.close()
url = self.container.ex_manager._get_management_url(
"queues", "%2f", xq.queue, "get")
res = self.container.ex_manager._make_management_call(url,
use_ems=False,
method='post',
data=json.dumps({
'count':
1,
'requeue':
True,
'encoding':
'auto'
}))
self.assertEquals(len(res), 1)
self.assertIn('properties', res[0])
self.assertIn('delivery_mode', res[0]['properties'])
self.assertEquals(2, res[0]['properties']['delivery_mode'])
def launch_benchmark(transform_number=1, primer=1, message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt = cc.proc_manager._create_service_instance(
str(pids), 'tbt', 'prototype.transforms.linear',
'TransformInPlace',
DotDict({
'process': {
'name': 'tbt%d' % pids,
'transform_id': pids
}
}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
pub.publish(list(xrange(msg_len)))
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
class TestPublisher(PyonTestCase):
def setUp(self):
self._node = Mock(spec=NodeB)
self._pub = Publisher(node=self._node, to_name="testpub")
self._ch = Mock(spec=SendChannel)
self._node.channel.return_value = self._ch
def test_publish(self):
self.assertEquals(self._node.channel.call_count, 0)
self._pub.publish("pub")
self._node.channel.assert_called_once_with(self._pub.channel_type)
self.assertEquals(self._ch.send.call_count, 1)
self._pub.publish("pub2")
self._node.channel.assert_called_once_with(self._pub.channel_type)
self.assertEquals(self._ch.send.call_count, 2)
def test_publish_with_new_name(self):
self.assertEquals(self._node.channel.call_count, 0)
self._pub.publish(sentinel.msg, to_name=sentinel.to_name)
self.assertEquals(self._ch.send.call_count, 1)
self._pub.publish(sentinel.msg, to_name=sentinel.to_name)
self.assertEquals(self._ch.send.call_count, 2)
def test_close(self):
self._pub.publish(sentinel.msg)
self._pub._pub_ep.close = Mock()
self._pub.close()
self._pub._pub_ep.close.assert_called_once_with()
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
def test_sub(self):
#start interaction observer
io = InteractionObserver()
io.start()
#publish an event
ev_pub = EventPublisher(event_type="ResourceEvent")
ev_pub.publish_event(origin="specific", description="event")
# publish a message
msg_pub = Publisher()
msg_pub.publish(to_name='anyone', msg="msg")
# give 2 seconds for the messages to arrive
time.sleep(2)
#verify that two messages (an event and a message) are seen
self.assertEquals(len(io.msg_log), 2)
#iterate through the messages observed
for item in io.msg_log:
# if event
if item[2]:
#verify that the origin is what we sent
self.assertEquals(item[1]['origin'], 'specific')
dump = io._get_data(io.msg_log,{})
sump = dump
class AsyncResultPublisher(object):
"""
Class that helps sending async results.
"""
def __init__(self, process=None, wait_name=None):
self.process = process
if not wait_name.startswith("asyncresult_"):
raise BadRequest("Not a valid wait_name")
self.wait_name = wait_name
if self.process:
self.pub = ProcessPublisher(process=self.process,
to_name=wait_name)
else:
self.pub = Publisher(to_name=wait_name)
def publish_result(self, request_id, result):
async_res = AsyncResultMsg(result=result,
request_id=request_id,
ts=get_ion_ts())
self.pub.publish(async_res)
self.pub.close()
def publish_error(self, request_id, error, error_code):
async_res = AsyncResultMsg(result=error,
request_id=request_id,
ts=get_ion_ts(),
status=error_code)
self.pub.publish(async_res)
self.pub.close()
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
def test_sub(self):
#start interaction observer
io = InteractionObserver()
io.start()
#publish an event
ev_pub = EventPublisher(event_type="ResourceEvent")
ev_pub.publish_event(origin="specific", description="event")
# publish a message
msg_pub = Publisher()
msg_pub.publish(to_name='anyone', msg="msg")
# give 2 seconds for the messages to arrive
time.sleep(2)
#verify that two messages (an event and a message) are seen
self.assertEquals(len(io.msg_log), 2)
#iterate through the messages observed
for item in io.msg_log:
# if event
if item[2]:
#verify that the origin is what we sent
self.assertEquals(item[1]['origin'], 'specific')
dump = io._get_data(io.msg_log,{})
sump = dump
def launch_benchmark(transform_number=1, primer=1,message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt=cc.proc_manager._create_service_instance(str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlace', DotDict({'process':{'name':'tbt%d' % pids, 'transform_id':pids}}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
pub.publish(list(xrange(msg_len)))
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def test_xp_durable_send(self):
xp = self.container.ex_manager.create_xp('an_xp')
#self.addCleanup(xp.delete)
xq = self.container.ex_manager.create_xn_queue('no_matter', xp)
self.addCleanup(xq.delete)
xq.bind('one')
pub = Publisher(to_name=xp.create_route('one'))
pub.publish('test')
pub.close()
try:
url = self.container.ex_manager._get_management_url("queues", "%2f", xq.queue, "get")
res = self.container.ex_manager._make_management_call(url,
use_ems=False,
method='post',
data=json.dumps({'count':1, 'requeue':True,'encoding':'auto'}))
self.assertEquals(len(res), 1)
self.assertIn('properties', res[0])
self.assertIn('delivery_mode', res[0]['properties'])
self.assertEquals(2, res[0]['properties']['delivery_mode'])
except Exception, e:
# Rabbit 3.x does not support this command anymore apparently.
self.assertIn('Method Not Allowed', e.message)
class TestPublisher(PyonTestCase):
def setUp(self):
self._node = Mock(spec=NodeB)
self._pub = Publisher(node=self._node, to_name="testpub")
self._ch = Mock(spec=SendChannel)
self._node.channel.return_value = self._ch
def test_publish(self):
self.assertEquals(self._node.channel.call_count, 0)
self._pub.publish("pub")
self._node.channel.assert_called_once_with(self._pub.channel_type)
self.assertEquals(self._ch.send.call_count, 1)
self._pub.publish("pub2")
self._node.channel.assert_called_once_with(self._pub.channel_type)
self.assertEquals(self._ch.send.call_count, 2)
def test_publish_with_new_name(self):
self.assertEquals(self._node.channel.call_count, 0)
self._pub.publish(sentinel.msg, to_name=sentinel.to_name)
self.assertEquals(self._ch.send.call_count, 1)
self._pub.publish(sentinel.msg, to_name=sentinel.to_name)
self.assertEquals(self._ch.send.call_count, 2)
def test_close(self):
self._pub.publish(sentinel.msg)
self._pub._pub_ep.close = Mock()
self._pub.close()
self._pub._pub_ep.close.assert_called_once_with()
def test_sub(self):
ar = event.AsyncResult()
def cb(*args, **kwargs):
ar.set(args)
sub = ConvSubscriber(callback=cb)
pub = Publisher()
self._listen(sub)
pub.publish(to_name='anyone', msg="hello")
evmsg, evheaders = ar.get(timeout=5)
self.assertEquals(evmsg, "hello")
self.assertAlmostEquals(int(evheaders['ts']), int(get_ion_ts()), delta=5000)
def launch_benchmark(transform_number=1, primer=1, message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import numpy
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
tt = TaxyTool()
tt.add_taxonomy_set('a')
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt = cc.proc_manager._create_service_instance(
str(pids), 'tbt', 'prototype.transforms.linear',
'TransformInPlaceNewGranule',
DotDict({
'process': {
'name': 'tbt%d' % pids,
'transform_id': pids
}
}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
rd = RecordDictionaryTool(tt, message_length)
rd['a'] = numpy.arange(message_length)
gran = build_granule(data_producer_id='dp_id',
taxonomy=tt,
record_dictionary=rd)
pub.publish(gran)
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def test_async_result(self):
request_id = "request_foo"
waiter = AsyncResultWaiter()
self.assertFalse(waiter.async_res.ready())
token = waiter.activate()
self.assertFalse(waiter.async_res.ready())
log.info("Wait token: %s", token)
pub = Publisher(to_name=token)
async_msg = AsyncResultMsg(request_id=request_id)
pub.publish(async_msg)
res = waiter.await(timeout=1, request_id=request_id)
self.assertTrue(waiter.async_res.ready())
self.assertIsInstance(res, AsyncResultMsg)
self.assertEqual(res.__dict__, async_msg.__dict__)
class ContainerHeartbeater(object):
""" Utility class that implements the container heartbeat publishing mechanism """
def __init__(self, container, cfg):
self.container = container
self.heartbeat_cfg = cfg
self.started = False
def start(self):
from pyon.net.endpoint import Publisher
from pyon.util. async import spawn
self.heartbeat_quit = Event()
self.heartbeat_interval = float(
self.heartbeat_cfg.get("publish_interval", 60))
self.heartbeat_topic = self.heartbeat_cfg.get("topic", "heartbeat")
self.heartbeat_pub = Publisher(to_name=self.heartbeat_topic)
# Directly spawn a greenlet - we don't want this to be a supervised IonProcessThread
self.heartbeat_gl = spawn(self.heartbeat_loop)
self.started = True
log.info("Started container heartbeat (interval=%s, topic=%s)",
self.heartbeat_interval, self.heartbeat_topic)
def stop(self):
if self.started:
self.heartbeat_quit.set()
self.heartbeat_gl.join(timeout=1)
self.started = False
def heartbeat_loop(self):
self.publish_heartbeat()
while not self.heartbeat_quit.wait(timeout=self.heartbeat_interval):
self.publish_heartbeat()
def publish_heartbeat(self):
try:
hb_msg = self.get_heartbeat_message()
headers = dict(expiration=60000)
self.heartbeat_pub.publish(hb_msg, headers=headers)
except Exception:
log.exception("Error publishing heatbeat")
def get_heartbeat_message(self):
from interface.objects import ContainerHeartbeat
hb_msg = ContainerHeartbeat(container_id=self.container.id,
ts=get_ion_ts())
return hb_msg
class HeartBeater(object):
def __init__(self, CFG, factory, log=logging):
self._log = log
self._log.log(logging.DEBUG, "Starting the heartbeat thread")
self._CFG = CFG
self._res = None
self._interval = int(CFG.eeagent.heartbeat)
self._res = None
self._done = False
self._factory = factory
self._publisher = Publisher()
self._pd_name = CFG.eeagent.get('heartbeat_queue', 'heartbeat_queue')
self._factory.set_state_change_callback(self._state_change_callback,
None)
self._first_beat()
def _first_beat(self):
self._beat_time = datetime.datetime.now()
def _next_beat(self, now):
self._beat_time = now + datetime.timedelta(seconds=self._interval)
def _state_change_callback(self, user_arg):
# on state change set the beat time to now
self._beat_time = datetime.datetime.now()
def poll(self):
now = datetime.datetime.now()
if now > self._beat_time:
self._next_beat(now)
self.beat()
def beat(self):
try:
beat = make_beat_msg(self._factory, self._CFG)
message = dict(beat=beat, resource_id=self._CFG.agent.resource_id)
to_name = self._pd_name
self._log.debug("Send heartbeat: %s to %s", message, self._pd_name)
self._publisher.publish(message, to_name=to_name)
except:
self._log.exception("beat failed")
class ContainerHeartbeater(object):
""" Utility class that implements the container heartbeat publishing mechanism """
def __init__(self, container, cfg):
self.container = container
self.heartbeat_cfg = cfg
self.started = False
def start(self):
from pyon.net.endpoint import Publisher
from pyon.util.async import spawn
self.heartbeat_quit = Event()
self.heartbeat_interval = float(self.heartbeat_cfg.get("publish_interval", 60))
self.heartbeat_topic = self.heartbeat_cfg.get("topic", "heartbeat")
self.heartbeat_pub = Publisher(to_name=self.heartbeat_topic)
# Directly spawn a greenlet - we don't want this to be a supervised IonProcessThread
self.heartbeat_gl = spawn(self.heartbeat_loop)
self.started = True
log.info("Started container heartbeat (interval=%s, topic=%s)", self.heartbeat_interval, self.heartbeat_topic)
def stop(self):
if self.started:
self.heartbeat_quit.set()
self.heartbeat_gl.join(timeout=1)
self.started = False
def heartbeat_loop(self):
self.publish_heartbeat()
while not self.heartbeat_quit.wait(timeout=self.heartbeat_interval):
self.publish_heartbeat()
def publish_heartbeat(self):
try:
hb_msg = self.get_heartbeat_message()
headers = dict(expiration=60000)
self.heartbeat_pub.publish(hb_msg, headers=headers)
except Exception:
log.exception("Error publishing heatbeat")
def get_heartbeat_message(self):
from interface.objects import ContainerHeartbeat
hb_msg = ContainerHeartbeat(container_id=self.container.id, ts=get_ion_ts())
return hb_msg
class HeartBeater(object):
def __init__(self, CFG, factory, log=logging):
self._log = log
self._log.log(logging.DEBUG, "Starting the heartbeat thread")
self._CFG = CFG
self._res = None
self._interval = int(CFG.eeagent.heartbeat)
self._res = None
self._done = False
self._factory = factory
self._next_beat(datetime.datetime.now())
self._publisher = Publisher()
self._pd_name = CFG.eeagent.get('heartbeat_queue', 'heartbeat_queue')
self._factory.set_state_change_callback(self._state_change_callback, None)
def _next_beat(self, now):
self._beat_time = now + datetime.timedelta(seconds=self._interval)
def _state_change_callback(self, user_arg):
# on state change set the beat time to now
self._beat_time = datetime.datetime.now()
def poll(self):
now = datetime.datetime.now()
if now > self._beat_time:
self._next_beat(now)
self.beat()
def beat(self):
try:
beat = make_beat_msg(self._factory, self._CFG)
message = dict(beat=beat, resource_id=self._CFG.agent.resource_id)
to_name = self._pd_name
self._log.debug("Send heartbeat: %s to %s", message, self._pd_name)
self._publisher.publish(message, to_name=to_name)
except:
self._log.exception("beat failed")
def test_sub(self):
# publish 2 messages
pub = Publisher()
pub.publish(to_name='anyone', msg="hello1")
pub.publish(to_name='anyone', msg="hello2")
dsm = self.container.datastore_manager
ds = dsm.get_datastore("conversations")
# give at least 2 seconds for the persister to save in the repository
# test may fail if it does not wait long enough for the persister
no_of_conv = 0
retried = 0
while (no_of_conv != 2 and retried < 5):
time.sleep(2)
# assert that the 2 messages have been persisted
no_of_conv = len(ds.list_objects())
retried = retried + 1
self.assertEquals(no_of_conv, 2)
def test_sub(self):
# publish 2 messages
pub = Publisher()
pub.publish(to_name='anyone', msg="hello1")
pub.publish(to_name='anyone', msg="hello2")
dsm = self.container.datastore_manager
ds = dsm.get_datastore("conversations")
# give at least 2 seconds for the persister to save in the repository
# test may fail if it does not wait long enough for the persister
no_of_conv = 0
retried = 0
while (no_of_conv != 2 and retried < 5):
time.sleep(2)
# assert that the 2 messages have been persisted
no_of_conv = len(ds.list_objects())
retried = retried + 1
self.assertEquals(no_of_conv, 2)
def launch_benchmark(transform_number=1, primer=1,message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import numpy
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
tt = TaxyTool()
tt.add_taxonomy_set('a')
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt=cc.proc_manager._create_service_instance(str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlaceNewGranule', DotDict({'process':{'name':'tbt%d' % pids, 'transform_id':pids}}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
rd = RecordDictionaryTool(tt, message_length)
rd['a'] = numpy.arange(message_length)
gran = build_granule(data_producer_id='dp_id',taxonomy=tt, record_dictionary=rd)
pub.publish(gran)
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
class HeartBeater(object):
def __init__(self, CFG, factory, process_id, process, log=logging):
self._log = log
self._log.log(logging.DEBUG, "Starting the heartbeat thread")
self._CFG = CFG
self._res = None
self._interval = float(CFG.eeagent.heartbeat)
self._res = None
self._done = False
self._started = False
self._factory = factory
self.process = process
self.process_id = process_id
self._pd_name = CFG.eeagent.get('heartbeat_queue', 'heartbeat_queue')
self._publisher = Publisher(to_name=self._pd_name)
self._factory.set_state_change_callback(self._state_change_callback,
None)
self._first_beat()
def _first_beat(self):
self._beat_time = datetime.datetime.now()
def _next_beat(self, now):
self._beat_time = now + datetime.timedelta(seconds=self._interval)
def _state_change_callback(self, user_arg):
# on state change set the beat time to now
self._beat_time = datetime.datetime.now()
@property
def _eea_started(self):
"""_eea_started
We must ensure that the eea is listening before heartbeating to the PD.
If the eea isn't listening, the PD's reply will be lost.
So we must ensure that the Pyon process's listeners are created, and are ready
"""
if self._started:
return True
if len(self.process._process.listeners) > 0 and all(
self.process._process.heartbeat()):
self._log.debug(
"eeagent heartbeat started because len(self.process._process.listeners) > 0 (%s) "
"and all(self.process._process.heartbeat()) == True (%s)" %
(len(self.process._process.listeners),
str(self.process._process.heartbeat())))
self._started = True
return True
else:
return False
def poll(self):
if not self._eea_started:
return
now = datetime.datetime.now()
if now > self._beat_time:
self._next_beat(now)
self.beat()
def beat(self):
try:
beat = make_beat_msg(self._factory, self._CFG)
message = dict(beat=beat,
eeagent_id=self.process_id,
resource_id=self._CFG.agent.resource_id)
if self._log.isEnabledFor(logging.DEBUG):
processes = beat.get('processes')
if processes is not None:
processes_str = "processes=%d" % len(processes)
else:
processes_str = ""
self._log.debug("Sending heartbeat to %s %s", self._pd_name,
processes_str)
self._publisher.publish(message)
except Exception:
self._log.exception("beat failed")
class TransformBenchTesting(TransformDataProcess):
"""
Easiest way to run:
from pyon.util.containers import DotDict
tbt=cc.proc_manager._create_service_instance('55', 'tbt', 'pyon.ion.transform', 'TransformBenchTesting', DotDict({'process':{'name':'tbt', 'transform_id':'55'}}))
tbt.init()
tbt.start()
"""
transform_number = 0
message_length = 0
def __init__(self):
super(TransformBenchTesting,self).__init__()
self.count = 0
TransformBenchTesting.transform_number += 1
def perf(self):
with open('/tmp/pyon_performance.dat','a') as f:
then = time.time()
ocount = self.count
while True:
gevent.sleep(2.)
now = time.time()
count = self.count
delta_t = now - then
delta_c = count - ocount
f.write('%s|%s\t%s\t%s\t%3.3f\n' % (get_sys_name(),time.strftime("%H:%M:%S", time.gmtime()),TransformBenchTesting.message_length,TransformBenchTesting.transform_number, float(delta_c) / delta_t))
then = now
ocount = count
f.flush()
@staticmethod
def launch_benchmark(transform_number=1, primer=1,message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt=cc.proc_manager._create_service_instance(str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlace', DotDict({'process':{'name':'tbt%d' % pids, 'transform_id':pids}}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
pub.publish(list(xrange(msg_len)))
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def on_start(self):
TransformDataProcess.on_start(self)
# set up subscriber to *
self._bt_sub = Subscriber(callback=lambda m, h: self.call_process(m),
from_name=NameTrio(get_sys_name(), 'bench_queue', '*'))
# spawn listener
self._sub_gl = spawn(self._bt_sub.listen)
# set up publisher to anything!
self._bt_pub = Publisher(to_name=NameTrio(get_sys_name(), str(uuid.uuid4())[0:6]))
def publish(self, msg):
self._bt_pub.publish(msg)
self.count+=1
def _stop_listener(self):
self._bt_sub.close()
self._sub_gl.join(timeout=2)
self._sub_gl.kill()
def on_stop(self):
TransformDataProcess.on_stop(self)
self._stop_listener()
def on_quit(self):
TransformDataProcess.on_quit(self)
self._stop_listener()
def test_consume_one_message_at_a_time(self):
# see also pyon.net.test.test_channel:TestChannelInt.test_consume_one_message_at_a_time
pub3 = Publisher(to_name=(self.container.ex_manager.default_xs.exchange, 'routed.3'))
pub5 = Publisher(to_name=(self.container.ex_manager.default_xs.exchange, 'routed.5'))
#
# SETUP COMPLETE, BEGIN TESTING OF EXCHANGE OBJECTS
#
xq = self.container.ex_manager.create_xn_queue('random_queue')
self.addCleanup(xq.delete)
# recv'd messages from the subscriber
self.recv_queue = Queue()
def cb(m, h):
raise StandardError("Subscriber callback never gets called back!")
sub = Subscriber(from_name=xq, callback=cb)
sub.initialize()
# publish 10 messages - we're not bound yet, so they'll just dissapear
for x in xrange(10):
pub3.publish("3,%s" % str(x))
# allow time for routing
time.sleep(2)
# no messages yet
self.assertRaises(Timeout, sub.get_one_msg, timeout=0)
# now, we'll bind the xq
xq.bind('routed.3')
# even tho we are consuming, there are no messages - the previously published ones all dissapeared
self.assertRaises(Timeout, sub.get_one_msg, timeout=0)
# publish those messages again
for x in xrange(10):
pub3.publish("3,%s" % str(x))
# allow time for routing
time.sleep(2)
# NOW we have messages!
for x in xrange(10):
mo = sub.get_one_msg(timeout=10)
self.assertEquals(mo.body, "3,%s" % str(x))
mo.ack()
# we've cleared it all
self.assertRaises(Timeout, sub.get_one_msg, timeout=0)
# bind a wildcard and publish on both
xq.bind('routed.*')
for x in xrange(10):
time.sleep(0.3)
pub3.publish("3,%s" % str(x))
time.sleep(0.3)
pub5.publish("5,%s" % str(x))
# allow time for routing
time.sleep(2)
# should get all 20, interleaved
for x in xrange(10):
mo = sub.get_one_msg(timeout=1)
self.assertEquals(mo.body, "3,%s" % str(x))
mo.ack()
mo = sub.get_one_msg(timeout=1)
self.assertEquals(mo.body, "5,%s" % str(x))
mo.ack()
# add 5 binding, remove all other bindings
xq.bind('routed.5')
xq.unbind('routed.3')
xq.unbind('routed.*')
# try publishing to 3, shouldn't arrive anymore
pub3.publish("3")
self.assertRaises(Timeout, sub.get_one_msg, timeout=0)
# let's turn off the consumer and let things build up a bit
sub._chan.stop_consume()
for x in xrange(10):
pub5.publish("5,%s" % str(x))
# allow time for routing
time.sleep(2)
# 10 messages in the queue, no consumers
self.assertTupleEqual((10, 0), sub._chan.get_stats())
# drain queue
sub._chan.start_consume()
for x in xrange(10):
mo = sub.get_one_msg(timeout=1)
mo.ack()
sub.close()
class HeartBeater(object):
def __init__(self, CFG, factory, process_id, process, log=logging):
self._log = log
self._log.log(logging.DEBUG, "Starting the heartbeat thread")
self._CFG = CFG
self._res = None
self._interval = int(CFG.eeagent.heartbeat)
self._res = None
self._done = False
self._started = False
self._factory = factory
self.process = process
self.process_id = process_id
self._publisher = Publisher()
self._pd_name = CFG.eeagent.get('heartbeat_queue', 'heartbeat_queue')
self._factory.set_state_change_callback(self._state_change_callback,
None)
self._first_beat()
def _first_beat(self):
self._beat_time = datetime.datetime.now()
def _next_beat(self, now):
self._beat_time = now + datetime.timedelta(seconds=self._interval)
def _state_change_callback(self, user_arg):
# on state change set the beat time to now
self._beat_time = datetime.datetime.now()
@property
def _eea_started(self):
if self._started:
return True
if all(self.process._process.heartbeat()):
self._started = True
return True
else:
return False
def poll(self):
if not self._eea_started:
return
now = datetime.datetime.now()
if now > self._beat_time:
self._next_beat(now)
self.beat()
def beat(self):
try:
beat = make_beat_msg(self._factory, self._CFG)
message = dict(beat=beat,
eeagent_id=self.process_id,
resource_id=self._CFG.agent.resource_id)
to_name = self._pd_name
if self._log.isEnabledFor(logging.DEBUG):
processes = beat.get('processes')
if processes is not None:
processes_str = "processes=%d" % len(processes)
else:
processes_str = ""
self._log.debug("Sending heartbeat to %s %s", self._pd_name,
processes_str)
self._publisher.publish(message, to_name=to_name)
except Exception:
self._log.exception("beat failed")
def test_replay_integration(self):
'''
test_replay_integration
'''
import numpy as np
# Keep the import it's used in the vector comparison below even though pycharm says its unused.
cc = self.container
XP = self.XP
assertions = self.assertTrue
### Every thing below here can be run as a script:
log.debug('Got it')
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
datastore_name = 'dm_test_replay_integration'
producer = Publisher(name=(XP, 'stream producer'))
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id=XP,
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
hdf_storage=HdfStorage(),
number_of_workers=1)
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
definition = SBE37_CDM_stream_definition()
data_stream_id = definition.data_stream_id
encoding_id = definition.identifiables[data_stream_id].encoding_id
element_count_id = definition.identifiables[
data_stream_id].element_count_id
stream_def_id = pubsub_management_service.create_stream_definition(
container=definition)
stream_id = pubsub_management_service.create_stream(
stream_definition_id=stream_def_id)
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/dataset_by_id')
ingestion_management_service.create_dataset_configuration(
dataset_id=dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=ingestion_configuration_id)
definition.stream_resource_id = stream_id
packet = _create_packet(definition)
input_file = FileSystem.mktemp()
input_file.write(packet.identifiables[data_stream_id].values)
input_file_path = input_file.name
input_file.close()
fields = [
'conductivity', 'height', 'latitude', 'longitude', 'pressure',
'temperature', 'time'
]
input_vectors = acquire_data([input_file_path], fields, 2).next()
producer.publish(msg=packet, to_name=(XP, '%s.data' % stream_id))
replay_id, replay_stream_id = data_retriever_service.define_replay(
dataset_id)
ar = gevent.event.AsyncResult()
def sub_listen(msg, headers):
assertions(isinstance(msg, StreamGranuleContainer),
'replayed message is not a granule.')
hdf_string = msg.identifiables[data_stream_id].values
sha1 = hashlib.sha1(hdf_string).hexdigest().upper()
assertions(sha1 == msg.identifiables[encoding_id].sha1,
'Checksum failed.')
assertions(
msg.identifiables[element_count_id].value == 1,
'record replay count is incorrect %d.' %
msg.identifiables[element_count_id].value)
output_file = FileSystem.mktemp()
output_file.write(msg.identifiables[data_stream_id].values)
output_file_path = output_file.name
output_file.close()
output_vectors = acquire_data([output_file_path], fields, 2).next()
for field in fields:
comparison = (input_vectors[field]['values'] ==
output_vectors[field]['values'])
assertions(
comparison.all(), 'vector mismatch: %s vs %s' %
(input_vectors[field]['values'],
output_vectors[field]['values']))
FileSystem.unlink(output_file_path)
ar.set(True)
subscriber = Subscriber(name=(XP, 'replay listener'),
callback=sub_listen)
g = gevent.Greenlet(subscriber.listen,
binding='%s.data' % replay_stream_id)
g.start()
data_retriever_service.start_replay(replay_id)
ar.get(timeout=10)
FileSystem.unlink(input_file_path)
def test_replay_integration(self):
'''
test_replay_integration
'''
import numpy as np
# Keep the import it's used in the vector comparison below even though pycharm says its unused.
cc = self.container
XP = self.XP
assertions = self.assertTrue
### Every thing below here can be run as a script:
log.debug('Got it')
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
datastore_name = 'dm_test_replay_integration'
producer = Publisher(name=(XP,'stream producer'))
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id=XP,
couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'),
hdf_storage=HdfStorage(),
number_of_workers=1
)
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id
)
definition = SBE37_CDM_stream_definition()
data_stream_id = definition.data_stream_id
encoding_id = definition.identifiables[data_stream_id].encoding_id
element_count_id = definition.identifiables[data_stream_id].element_count_id
stream_def_id = pubsub_management_service.create_stream_definition(
container=definition
)
stream_id = pubsub_management_service.create_stream(
stream_definition_id=stream_def_id
)
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/dataset_by_id'
)
ingestion_management_service.create_dataset_configuration(
dataset_id=dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id = ingestion_configuration_id
)
definition.stream_resource_id = stream_id
packet = _create_packet(definition)
input_file = FileSystem.mktemp()
input_file.write(packet.identifiables[data_stream_id].values)
input_file_path = input_file.name
input_file.close()
fields=[
'conductivity',
'height',
'latitude',
'longitude',
'pressure',
'temperature',
'time'
]
input_vectors = acquire_data([input_file_path],fields , 2).next()
producer.publish(msg=packet, to_name=(XP,'%s.data' % stream_id))
replay_id, replay_stream_id = data_retriever_service.define_replay(dataset_id)
ar = gevent.event.AsyncResult()
def sub_listen(msg, headers):
assertions(isinstance(msg,StreamGranuleContainer),'replayed message is not a granule.')
hdf_string = msg.identifiables[data_stream_id].values
sha1 = hashlib.sha1(hdf_string).hexdigest().upper()
assertions(sha1 == msg.identifiables[encoding_id].sha1,'Checksum failed.')
assertions(msg.identifiables[element_count_id].value==1, 'record replay count is incorrect %d.' % msg.identifiables[element_count_id].value)
output_file = FileSystem.mktemp()
output_file.write(msg.identifiables[data_stream_id].values)
output_file_path = output_file.name
output_file.close()
output_vectors = acquire_data([output_file_path],fields,2).next()
for field in fields:
comparison = (input_vectors[field]['values']==output_vectors[field]['values'])
assertions(comparison.all(), 'vector mismatch: %s vs %s' %
(input_vectors[field]['values'],output_vectors[field]['values']))
FileSystem.unlink(output_file_path)
ar.set(True)
subscriber = Subscriber(name=(XP,'replay listener'),callback=sub_listen)
g = gevent.Greenlet(subscriber.listen, binding='%s.data' % replay_stream_id)
g.start()
data_retriever_service.start_replay(replay_id)
ar.get(timeout=10)
FileSystem.unlink(input_file_path)
class HeartBeater(object):
def __init__(self, CFG, factory, process_id, process, log=logging):
self._log = log
self._log.log(logging.DEBUG, "Starting the heartbeat thread")
self._CFG = CFG
self._res = None
self._interval = int(CFG.eeagent.heartbeat)
self._res = None
self._done = False
self._started = False
self._factory = factory
self.process = process
self.process_id = process_id
self._publisher = Publisher()
self._pd_name = CFG.eeagent.get('heartbeat_queue', 'heartbeat_queue')
self._factory.set_state_change_callback(self._state_change_callback, None)
self._first_beat()
def _first_beat(self):
self._beat_time = datetime.datetime.now()
def _next_beat(self, now):
self._beat_time = now + datetime.timedelta(seconds=self._interval)
def _state_change_callback(self, user_arg):
# on state change set the beat time to now
self._beat_time = datetime.datetime.now()
@property
def _eea_started(self):
if self._started:
return True
try:
_eea_pyon_client = SimpleResourceAgentClient(self.process_id, process=self.process)
eea_client = ExecutionEngineAgentClient(_eea_pyon_client)
eea_client.dump_state()
self._started = True
return True
except NotFound:
return False
except Exception:
self._log.exception("Couldn't get eeagent state. Perhaps it is broken?")
return False
def poll(self):
if not self._eea_started:
return
now = datetime.datetime.now()
if now > self._beat_time:
self._next_beat(now)
self.beat()
def beat(self):
try:
beat = make_beat_msg(self._factory, self._CFG)
message = dict(beat=beat, eeagent_id=self.process_id, resource_id=self._CFG.agent.resource_id)
to_name = self._pd_name
if self._log.isEnabledFor(logging.DEBUG):
processes = beat.get('processes')
if processes is not None:
processes_str = "processes=%d" % len(processes)
else:
processes_str = ""
self._log.debug("Sending heartbeat to %s %s", self._pd_name, processes_str)
self._publisher.publish(message, to_name=to_name)
except Exception:
self._log.exception("beat failed")
class HeartBeater(object):
def __init__(self, CFG, factory, process_id, process, log=logging):
self._log = log
self._log.log(logging.DEBUG, "Starting the heartbeat thread")
self._CFG = CFG
self._res = None
self._interval = float(CFG.eeagent.heartbeat)
self._res = None
self._done = False
self._started = False
self._factory = factory
self.process = process
self.process_id = process_id
self._pd_name = CFG.eeagent.get('heartbeat_queue', 'heartbeat_queue')
self._publisher = Publisher(to_name=self._pd_name)
self._factory.set_state_change_callback(
self._state_change_callback, None)
self._first_beat()
def _first_beat(self):
self._beat_time = datetime.datetime.now()
def _next_beat(self, now):
self._beat_time = now + datetime.timedelta(seconds=self._interval)
def _state_change_callback(self, user_arg):
# on state change set the beat time to now
self._beat_time = datetime.datetime.now()
@property
def _eea_started(self):
"""_eea_started
We must ensure that the eea is listening before heartbeating to the PD.
If the eea isn't listening, the PD's reply will be lost.
So we must ensure that the Pyon process's listeners are created, and are ready
"""
if self._started:
return True
if len(self.process._process.listeners) > 0 and all(self.process._process.heartbeat()):
self._log.debug(
"eeagent heartbeat started because len(self.process._process.listeners) > 0 (%s) "
"and all(self.process._process.heartbeat()) == True (%s)" % (
len(self.process._process.listeners), str(self.process._process.heartbeat())))
self._started = True
return True
else:
return False
def poll(self):
if not self._eea_started:
return
now = datetime.datetime.now()
if now > self._beat_time:
self._next_beat(now)
self.beat()
def beat(self):
try:
beat = make_beat_msg(self._factory, self._CFG)
message = dict(
beat=beat, eeagent_id=self.process_id,
resource_id=self._CFG.agent.resource_id)
if self._log.isEnabledFor(logging.DEBUG):
processes = beat.get('processes')
if processes is not None:
processes_str = "processes=%d" % len(processes)
else:
processes_str = ""
self._log.debug("Sending heartbeat to %s %s",
self._pd_name, processes_str)
self._publisher.publish(message)
except Exception:
self._log.exception("beat failed")
class TransformBenchTesting(TransformDataProcess):
"""
Easiest way to run:
from pyon.util.containers import DotDict
tbt=cc.proc_manager._create_service_instance('55', 'tbt', 'pyon.ion.transform', 'TransformBenchTesting', DotDict({'process':{'name':'tbt', 'transform_id':'55'}}))
tbt.init()
tbt.start()
"""
transform_number = 0
message_length = 0
def __init__(self):
super(TransformBenchTesting, self).__init__()
self.count = 0
TransformBenchTesting.transform_number += 1
def perf(self):
with open('/tmp/pyon_performance.dat', 'a') as f:
then = time.time()
ocount = self.count
while True:
gevent.sleep(2.)
now = time.time()
count = self.count
delta_t = now - then
delta_c = count - ocount
f.write(
'%s|%s\t%s\t%s\t%3.3f\n' %
(get_sys_name(), time.strftime("%H:%M:%S", time.gmtime()),
TransformBenchTesting.message_length,
TransformBenchTesting.transform_number,
float(delta_c) / delta_t))
then = now
ocount = count
f.flush()
@staticmethod
def launch_benchmark(transform_number=1, primer=1, message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt = cc.proc_manager._create_service_instance(
str(pids), 'tbt', 'prototype.transforms.linear',
'TransformInPlace',
DotDict({
'process': {
'name': 'tbt%d' % pids,
'transform_id': pids
}
}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
pub.publish(list(xrange(msg_len)))
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def on_start(self):
TransformDataProcess.on_start(self)
# set up subscriber to *
self._bt_sub = Subscriber(callback=lambda m, h: self.call_process(m),
from_name=NameTrio(get_sys_name(),
'bench_queue', '*'))
# spawn listener
self._sub_gl = spawn(self._bt_sub.listen)
# set up publisher to anything!
self._bt_pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
def publish(self, msg):
self._bt_pub.publish(msg)
self.count += 1
def _stop_listener(self):
self._bt_sub.close()
self._sub_gl.join(timeout=2)
self._sub_gl.kill()
def on_stop(self):
TransformDataProcess.on_stop(self)
self._stop_listener()
def on_quit(self):
TransformDataProcess.on_quit(self)
self._stop_listener()
def test_consume_one_message_at_a_time(self):
# see also pyon.net.test.test_channel:TestChannelInt.test_consume_one_message_at_a_time
pub3 = Publisher(to_name=(self.container.ex_manager.default_xs.exchange, 'routed.3'))
pub5 = Publisher(to_name=(self.container.ex_manager.default_xs.exchange, 'routed.5'))
#
# SETUP COMPLETE, BEGIN TESTING OF EXCHANGE OBJECTS
#
xq = self.container.ex_manager.create_xn_queue('random_queue')
self.addCleanup(xq.delete)
# recv'd messages from the subscriber
self.recv_queue = Queue()
sub = Subscriber(from_name=xq, callback=lambda m,h: self.recv_queue.put((m, h)))
sub.prepare_listener()
# publish 10 messages - we're not bound yet, so they'll just dissapear
for x in xrange(10):
pub3.publish("3,%s" % str(x))
# no messages yet
self.assertFalse(sub.get_one_msg(timeout=0))
# now, we'll bind the xq
xq.bind('routed.3')
# even tho we are consuming, there are no messages - the previously published ones all dissapeared
self.assertFalse(sub.get_one_msg(timeout=0))
# publish those messages again
for x in xrange(10):
pub3.publish("3,%s" % str(x))
# NOW we have messages!
for x in xrange(10):
self.assertTrue(sub.get_one_msg(timeout=0))
m,h = self.recv_queue.get(timeout=0)
self.assertEquals(m, "3,%s" % str(x))
# we've cleared it all
self.assertFalse(sub.get_one_msg(timeout=0))
# bind a wildcard and publish on both
xq.bind('routed.*')
for x in xrange(10):
time.sleep(0.3)
pub3.publish("3,%s" % str(x))
time.sleep(0.3)
pub5.publish("5,%s" % str(x))
# should get all 20, interleaved
for x in xrange(10):
self.assertTrue(sub.get_one_msg(timeout=0))
m, h = self.recv_queue.get(timeout=0)
self.assertEquals(m, "3,%s" % str(x))
self.assertTrue(sub.get_one_msg(timeout=0))
m, h = self.recv_queue.get(timeout=0)
self.assertEquals(m, "5,%s" % str(x))
# add 5 binding, remove all other bindings
xq.bind('routed.5')
xq.unbind('routed.3')
xq.unbind('routed.*')
# try publishing to 3, shouldn't arrive anymore
pub3.publish("3")
self.assertFalse(sub.get_one_msg(timeout=0))
# let's turn off the consumer and let things build up a bit
sub._chan.stop_consume()
for x in xrange(10):
pub5.publish("5,%s" % str(x))
# 10 messages in the queue, no consumers
self.assertTupleEqual((10, 0), sub._chan.get_stats())
# drain queue
sub._chan.start_consume()
time.sleep(1) # yield to allow delivery
for x in xrange(10):
self.assertTrue(sub.get_one_msg(timeout=0))
self.recv_queue.get(timeout=0)
sub.close()
