def test_published_with_policy_queue_and_rpc_down(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue'))
publisher.publish_samples(None, self.test_data)
self.assertEqual(0, len(self.published))
self.assertEqual(1, len(publisher.local_queue))
def test_publish_error(self):
with mock.patch('socket.socket',
self._make_broken_socket):
publisher = udp.UDPPublisher(
network_utils.urlsplit('udp://localhost'))
publisher.publish_counters(None,
self.test_data)
def test_published(self):
publisher = rpc.RPCPublisher(network_utils.urlsplit("rpc://"))
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0], self.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]["args"]["data"], list)
self.assertEqual(self.published[0][1]["method"], "record_metering_data")
def test_published_concurrency(self):
"""This test the concurrent access to the local queue
of the rpc publisher
"""
def faux_cast_go(context, topic, msg):
self.published.append((topic, msg))
def faux_cast_wait(context, topic, msg):
self.useFixture(fixtures.MonkeyPatch("ceilometer.openstack.common.rpc.cast", faux_cast_go))
# Sleep to simulate concurrency and allow other threads to work
eventlet.sleep(0)
self.published.append((topic, msg))
self.useFixture(fixtures.MonkeyPatch("ceilometer.openstack.common.rpc.cast", faux_cast_wait))
publisher = rpc.RPCPublisher(network_utils.urlsplit("rpc://"))
job1 = eventlet.spawn(publisher.publish_samples, None, self.test_data)
job2 = eventlet.spawn(publisher.publish_samples, None, self.test_data)
job1.wait()
job2.wait()
self.assertEqual(publisher.policy, "default")
self.assertEqual(len(self.published), 2)
self.assertEqual(len(publisher.local_queue), 0)
def test_publish_target(self):
publisher = rpc.RPCPublisher(network_utils.urlsplit("rpc://?target=custom_procedure_call"))
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0], self.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]["args"]["data"], list)
self.assertEqual(self.published[0][1]["method"], "custom_procedure_call")
def test_published_no_mock(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://'))
endpoint = mock.MagicMock(['record_metering_data'])
collector = messaging.get_rpc_server(
self.transport, self.CONF.publisher_rpc.metering_topic, endpoint)
endpoint.record_metering_data.side_effect = \
lambda *args, **kwds: collector.stop()
collector.start()
eventlet.sleep()
publisher.publish_samples(context.RequestContext(),
self.test_data)
collector.wait()
class Matcher(object):
@staticmethod
def __eq__(data):
for i, sample in enumerate(data):
if sample['counter_name'] != self.test_data[i].name:
return False
return True
endpoint.record_metering_data.assert_called_once_with(
mock.ANY, data=Matcher())
def test_published_concurrency(self):
"""This test the concurrent access to the local queue
of the rpc publisher
"""
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
cast_context = mock.MagicMock()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
def fake_prepare_go(topic):
return cast_context
def fake_prepare_wait(topic):
prepare.side_effect = fake_prepare_go
# Sleep to simulate concurrency and allow other threads to work
eventlet.sleep(0)
return cast_context
prepare.side_effect = fake_prepare_wait
job1 = eventlet.spawn(publisher.publish_samples,
mock.MagicMock(), self.test_data)
job2 = eventlet.spawn(publisher.publish_samples,
mock.MagicMock(), self.test_data)
job1.wait()
job2.wait()
self.assertEqual('default', publisher.policy)
self.assertEqual(2, len(cast_context.cast.mock_calls))
self.assertEqual(0, len(publisher.local_queue))
def test_published_with_per_meter_topic(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?per_meter_topic=1'))
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
publisher.publish_samples(mock.MagicMock(),
self.test_data)
class MeterGroupMatcher(object):
def __eq__(self, meters):
return len(set(meter['counter_name']
for meter in meters)) == 1
topic = self.CONF.publisher_rpc.metering_topic
expected = [mock.call(topic=topic),
mock.call().cast(mock.ANY, 'record_metering_data',
data=mock.ANY),
mock.call(topic=topic + '.test'),
mock.call().cast(mock.ANY, 'record_metering_data',
data=MeterGroupMatcher()),
mock.call(topic=topic + '.test2'),
mock.call().cast(mock.ANY, 'record_metering_data',
data=MeterGroupMatcher()),
mock.call(topic=topic + '.test3'),
mock.call().cast(mock.ANY, 'record_metering_data',
data=MeterGroupMatcher())]
self.assertEqual(expected, prepare.mock_calls)
def test_published(self):
self.data_sent = []
with mock.patch('socket.socket',
self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(
network_utils.urlsplit('udp://somehost'))
publisher.publish_samples(None,
self.test_data)
self.assertEqual(5, len(self.data_sent))
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data)
sent_counters.append(counter)
# Check destination
self.assertEqual(('somehost',
self.CONF.collector.udp_port), dest)
# Check that counters are equal
self.assertEqual(sorted(
[utils.meter_message_from_counter(d, "not-so-secret")
for d in self.test_data]), sorted(sent_counters))
def test_published_with_policy_sized_queue_and_rpc_down(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue&max_queue_length=3'))
side_effect = oslo.messaging._drivers.common.RPCException()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.side_effect = side_effect
for i in range(0, 5):
for s in self.test_data:
s.source = 'test-%d' % i
publisher.publish_samples(mock.MagicMock(),
self.test_data)
self.assertEqual(3, len(publisher.local_queue))
self.assertEqual(
'test-2',
publisher.local_queue[0][2][0]['source']
)
self.assertEqual(
'test-3',
publisher.local_queue[1][2][0]['source']
)
self.assertEqual(
'test-4',
publisher.local_queue[2][2][0]['source']
)
def test_publish_error(self):
with mock.patch('socket.socket',
self._make_broken_socket):
publisher = udp.UDPPublisher(
network_utils.urlsplit('udp://localhost'))
publisher.publish_samples(None,
self.test_data)
def test_published_with_policy_block(self, mylog):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(network_utils.urlsplit("rpc://?policy=default"))
self.assertTrue(mylog.info.called)
self.assertRaises(SystemExit, publisher.publish_samples, None, self.test_data)
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 0)
def _handle_action(self, action, alarm_id, previous, current, reason, reason_data):
try:
action = network_utils.urlsplit(action)
except Exception:
LOG.error(
_("Unable to parse action %(action)s for alarm %(alarm_id)s"), {"action": action, "alarm_id": alarm_id}
)
return
try:
notifier = self.notifiers[action.scheme].obj
except KeyError:
scheme = action.scheme
LOG.error(
_("Action %(scheme)s for alarm %(alarm_id)s is unknown, " "cannot notify"),
{"scheme": scheme, "alarm_id": alarm_id},
)
return
try:
LOG.debug(_("Notifying alarm %(id)s with action %(act)s") % ({"id": alarm_id, "act": action}))
notifier.notify(action, alarm_id, previous, current, reason, reason_data)
except Exception:
LOG.exception(_("Unable to notify alarm %s"), alarm_id)
return
def test_published_concurrency(self):
"""This test the concurrent access to the local queue
of the rpc publisher
"""
def faux_cast_go(context, topic, msg):
self.published.append((topic, msg))
def faux_cast_wait(context, topic, msg):
self.useFixture(
fixtures.MonkeyPatch("ceilometer.openstack.common.rpc.cast",
faux_cast_go))
# Sleep to simulate concurrency and allow other threads to work
eventlet.sleep(0)
self.published.append((topic, msg))
self.useFixture(
fixtures.MonkeyPatch("ceilometer.openstack.common.rpc.cast",
faux_cast_wait))
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
job1 = eventlet.spawn(publisher.publish_samples, None, self.test_data)
job2 = eventlet.spawn(publisher.publish_samples, None, self.test_data)
job1.wait()
job2.wait()
self.assertEqual('default', publisher.policy)
self.assertEqual(2, len(self.published))
self.assertEqual(0, len(publisher.local_queue))
def setUp(self):
super(TestIPMIInspector, self).setUp()
self.inspector = ipmi.IPMIInspector()
self.host = network_utils.urlsplit("ipmi://*****:*****@160.85.197.44")
self.stubs.Set(self.inspector,
'execute_sensor_ipmi_command',
fake_sensor_list)
def _handle_action(self, action, alarm, state, reason):
try:
action = network_utils.urlsplit(action)
except Exception:
LOG.error(
_("Unable to parse action %(action)s for alarm %(alarm)s"),
locals())
return
try:
notifier = self.notifiers[action.scheme].obj
except KeyError:
scheme = action.scheme
LOG.error(
_("Action %(scheme)s for alarm %(alarm)s is unknown, "
"cannot notify"),
locals())
return
try:
LOG.debug("Notifying alarm %s with action %s",
alarm, action)
notifier.notify(action, alarm, state, reason)
except Exception:
LOG.exception(_("Unable to notify alarm %s"), alarm)
return
def _handle_action(self, action, alarm_id, previous, current, reason):
try:
action = network_utils.urlsplit(action)
except Exception:
LOG.error(
_("Unable to parse action %(action)s for alarm %(alarm_id)s"),
{'action': action, 'alarm_id': alarm_id})
return
try:
notifier = self.notifiers[action.scheme].obj
except KeyError:
scheme = action.scheme
LOG.error(
_("Action %(scheme)s for alarm %(alarm_id)s is unknown, "
"cannot notify"),
{'scheme': scheme, 'alarm_id': alarm_id})
return
try:
LOG.debug("Notifying alarm %s with action %s",
alarm_id, action)
notifier.notify(action, alarm_id, previous, current, reason)
except Exception:
LOG.exception(_("Unable to notify alarm %s"), alarm_id)
return
def test_published(self):
self.data_sent = []
with mock.patch('socket.socket',
self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(
network_utils.urlsplit('udp://somehost'))
publisher.publish_counters(None,
self.test_data,
self.COUNTER_SOURCE)
self.assertEqual(len(self.data_sent), 5)
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data)
self.assertEqual(counter['source'], self.COUNTER_SOURCE)
# Remove source because our test Counters don't have it, so the
# comparison would fail later
del counter['source']
sent_counters.append(counter)
# Check destination
self.assertEqual(dest, ('somehost',
cfg.CONF.collector.udp_port))
# Check that counters are equal
self.assertEqual(sorted(sent_counters),
sorted([dict(d._asdict()) for d in self.test_data]))
def get_samples(self, manager, cache, resources=None):
"""Return an iterable of Sample instances from polling the resources.
:param manager: The service manager invoking the plugin
:param cache: A dictionary for passing data between plugins
:param resources: end point to poll data from
"""
resources = resources or []
h_cache = cache.setdefault(self.CACHE_KEY, {})
sample_iters = []
for res in resources:
parsed_url = network_utils.urlsplit(res)
inspector = self._get_inspector(parsed_url)
func = getattr(inspector, self.INSPECT_METHOD)
try:
# Call hardware inspector to poll for the data
i_cache = h_cache.setdefault(res, {})
if res not in i_cache:
i_cache[res] = list(func(parsed_url))
# Generate samples
if i_cache[res]:
sample_iters.append(self.generate_samples(parsed_url,
i_cache[res]))
except Exception as err:
LOG.exception(_('inspector call %(func)r failed for '
'host %(host)s: %(err)s'),
dict(func=func,
host=parsed_url.hostname,
err=err))
return itertools.chain(*sample_iters)
def test_published_concurrency(self):
"""This test the concurrent access to the local queue
of the rpc publisher
"""
def faux_cast_go(context, topic, msg):
self.published.append((topic, msg))
def faux_cast_wait(context, topic, msg):
self.stubs.Set(oslo_rpc, 'cast', faux_cast_go)
# Sleep to simulate concurrency and allow other threads to work
eventlet.sleep(0)
self.published.append((topic, msg))
self.stubs.Set(oslo_rpc, 'cast', faux_cast_wait)
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
job1 = eventlet.spawn(publisher.publish_samples, None, self.test_data)
job2 = eventlet.spawn(publisher.publish_samples, None, self.test_data)
job1.wait()
job2.wait()
self.assertEqual(publisher.policy, 'default')
self.assertEqual(len(self.published), 2)
self.assertEqual(len(publisher.local_queue), 0)
def test_published_with_policy_queue_and_rpc_down(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue'))
publisher.publish_samples(None,
self.test_data)
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 1)
def test_published_with_policy_block(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=default'))
self.assertRaises(SystemExit, publisher.publish_samples, None,
self.test_data)
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 0)
def setUp(self):
super(TestSNMPInspector, self).setUp()
self.inspector = snmp.SNMPInspector()
self.host = network_utils.urlsplit("snmp://localhost")
self.useFixture(mockpatch.PatchObject(
self.inspector._cmdGen, 'getCmd', new=faux_getCmd))
self.useFixture(mockpatch.PatchObject(
self.inspector._cmdGen, 'nextCmd', new=faux_nextCmd))
def _parse_my_resource(resource):
parse_url = network_utils.urlsplit(resource)
params = urlparse.parse_qs(parse_url.query)
parts = urlparse.ParseResult(parse_url.scheme, parse_url.netloc,
parse_url.path, None, None, None)
return parts, params
def test_file_publisher(self):
# Test valid configurations
parsed_url = urlsplit(
'file:///tmp/log_file?max_bytes=50&backup_count=3')
publisher = file.FilePublisher(parsed_url)
publisher.publish_counters(None,
self.test_data,
self.COUNTER_SOURCE)
handler = publisher.publisher_logger.handlers[0]
self.assertTrue(isinstance(handler,
logging.handlers.RotatingFileHandler))
self.assertEqual([handler.maxBytes, handler.baseFilename,
handler.backupCount],
[50, '/tmp/log_file', 3])
# The rotating file gets created since only allow 50 bytes.
self.assertTrue(os.path.exists('/tmp/log_file.1'))
# Test missing max bytes, backup count configurations
parsed_url = urlsplit(
'file:///tmp/log_file_plain')
publisher = file.FilePublisher(parsed_url)
publisher.publish_counters(None,
self.test_data,
self.COUNTER_SOURCE)
handler = publisher.publisher_logger.handlers[0]
self.assertTrue(isinstance(handler,
logging.handlers.RotatingFileHandler))
self.assertEqual([handler.maxBytes, handler.baseFilename,
handler.backupCount],
[0, '/tmp/log_file_plain', 0])
# The rotating file gets created since only allow 50 bytes.
self.assertTrue(os.path.exists('/tmp/log_file_plain'))
# Test invalid max bytes, backup count configurations
parsed_url = urlsplit(
'file:///tmp/log_file_bad?max_bytes=yus&backup_count=5y')
publisher = file.FilePublisher(parsed_url)
publisher.publish_counters(None,
self.test_data,
self.COUNTER_SOURCE)
self.assertIsNone(publisher.publisher_logger)
def test_published(self):
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0],
self.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]['args']['data'], list)
self.assertEqual(self.published[0][1]['method'],
'record_metering_data')
def get_publisher(url, namespace='ceilometer.publisher'):
"""Get publisher driver and load it.
:param URL: URL for the publisher
:param namespace: Namespace to use to look for drivers.
"""
parse_result = network_utils.urlsplit(url)
loaded_driver = driver.DriverManager(namespace, parse_result.scheme)
return loaded_driver.driver(parse_result)
def get_publisher(url, namespace='ceilometer.publisher'):
"""Get publisher driver and load it.
:param URL: URL for the publisher
:param namespace: Namespace to use to look for drivers.
"""
parse_result = network_utils.urlsplit(url)
loaded_driver = driver.DriverManager(namespace, parse_result.scheme)
return loaded_driver.driver(parse_result)
def test_published_with_policy_drop_and_rpc_down(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=drop'))
publisher.publish_counters(None,
self.test_data,
'test')
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 0)
def test_file_publisher_invalid(self):
# Test invalid max bytes, backup count configurations
tempdir = tempfile.mkdtemp()
parsed_url = utils.urlsplit('file://%s/log_file_bad'
'?max_bytes=yus&backup_count=5y' % tempdir)
publisher = file.FilePublisher(parsed_url)
publisher.publish_samples(None, self.test_data)
self.assertIsNone(publisher.publisher_logger)
def test_published(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://'))
publisher.publish_counters(None,
self.test_data,
'test')
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0],
cfg.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]['args']['data'], list)
def test_published_with_policy_block(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=default'))
self.assertRaises(
SystemExit,
publisher.publish_counters,
None, self.test_data, 'test')
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 0)
def test_publish_target(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?target=custom_procedure_call'))
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0],
self.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]['args']['data'], list)
self.assertEqual(self.published[0][1]['method'],
'custom_procedure_call')
def test_published_with_no_policy(self, mylog):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
self.assertTrue(mylog.info.called)
self.assertRaises(SystemExit, publisher.publish_samples, None,
self.test_data)
self.assertEqual('default', publisher.policy)
self.assertEqual(0, len(self.published))
self.assertEqual(0, len(publisher.local_queue))
def test_published_with_policy_drop_and_rpc_down(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=drop'))
side_effect = oslo.messaging._drivers.common.RPCException()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.side_effect = side_effect
publisher.publish_samples(mock.MagicMock(), self.test_data)
self.assertEqual(0, len(publisher.local_queue))
prepare.assert_called_once_with(
topic=self.CONF.publisher_rpc.metering_topic)
def test_file_publisher_invalid(self):
# Test invalid max bytes, backup count configurations
parsed_url = urlsplit(
'file://%s/log_file_bad'
'?max_bytes=yus&backup_count=5y' % self.tempdir.path)
publisher = file.FilePublisher(parsed_url)
publisher.publish_samples(None,
self.test_data)
self.assertIsNone(publisher.publisher_logger)
def test_published_with_policy_incorrect(self, mylog):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=notexist'))
self.assertRaises(SystemExit, publisher.publish_samples, None,
self.test_data)
self.assertTrue(mylog.warn.called)
self.assertEqual(publisher.policy, 'default')
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 0)
def test_published_with_policy_incorrect(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=notexist'))
self.assertRaises(
SystemExit,
publisher.publish_samples,
None, self.test_data)
self.assertEqual(publisher.policy, 'default')
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 0)
def test_published_with_policy_default_sized_queue_and_rpc_down(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(network_utils.urlsplit("rpc://?policy=queue"))
for i in range(2000):
for s in self.test_data:
s.source = "test-%d" % i
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 1024)
self.assertEqual(publisher.local_queue[0][2]["args"]["data"][0]["source"], "test-976")
self.assertEqual(publisher.local_queue[1023][2]["args"]["data"][0]["source"], "test-1999")
def test_published_with_policy_drop_and_rpc_down(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=drop'))
side_effect = oslo.messaging._drivers.common.RPCException()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.side_effect = side_effect
publisher.publish_samples(mock.MagicMock(),
self.test_data)
self.assertEqual(0, len(publisher.local_queue))
prepare.assert_called_once_with(
topic=self.CONF.publisher_rpc.metering_topic)
def test_publish_target(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?target=custom_procedure_call'))
publisher.publish_samples(None,
self.test_data)
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0],
cfg.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]['args']['data'], list)
self.assertEqual(self.published[0][1]['method'],
'custom_procedure_call')
def test_published(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://'))
publisher.publish_samples(None,
self.test_data)
self.assertEqual(len(self.published), 1)
self.assertEqual(self.published[0][0],
self.CONF.publisher_rpc.metering_topic)
self.assertIsInstance(self.published[0][1]['args']['data'], list)
self.assertEqual(self.published[0][1]['method'],
'record_metering_data')
def test_published(self):
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
cast_context = mock.MagicMock()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.return_value = cast_context
publisher.publish_samples(mock.MagicMock(), self.test_data)
prepare.assert_called_once_with(
topic=self.CONF.publisher_rpc.metering_topic)
cast_context.cast.assert_called_once_with(mock.ANY,
'record_metering_data',
data=mock.ANY)
def setUp(self):
super(TestSNMPInspector, self).setUp()
self.inspector = snmp.SNMPInspector()
self.host = network_utils.urlsplit("snmp://localhost")
self.useFixture(
mockpatch.PatchObject(self.inspector._cmdGen,
'getCmd',
new=faux_getCmd))
self.useFixture(
mockpatch.PatchObject(self.inspector._cmdGen,
'nextCmd',
new=faux_nextCmd))
def test_published_with_policy_block(self, mylog):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=default'))
side_effect = oslo.messaging._drivers.common.RPCException()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.side_effect = side_effect
self.assertRaises(oslo.messaging._drivers.common.RPCException,
publisher.publish_samples, mock.MagicMock(),
self.test_data)
self.assertTrue(mylog.info.called)
self.assertEqual(0, len(publisher.local_queue))
prepare.assert_called_once_with(
topic=self.CONF.publisher_rpc.metering_topic)
def test_file_publisher(self):
# Test valid configurations
parsed_url = urlsplit(
'file:///tmp/log_file?max_bytes=50&backup_count=3')
publisher = file.FilePublisher(parsed_url)
publisher.publish_counters(None, self.test_data)
handler = publisher.publisher_logger.handlers[0]
self.assertTrue(
isinstance(handler, logging.handlers.RotatingFileHandler))
self.assertEqual(
[handler.maxBytes, handler.baseFilename, handler.backupCount],
[50, '/tmp/log_file', 3])
# The rotating file gets created since only allow 50 bytes.
self.assertTrue(os.path.exists('/tmp/log_file.1'))
# Test missing max bytes, backup count configurations
parsed_url = urlsplit('file:///tmp/log_file_plain')
publisher = file.FilePublisher(parsed_url)
publisher.publish_counters(None, self.test_data)
handler = publisher.publisher_logger.handlers[0]
self.assertTrue(
isinstance(handler, logging.handlers.RotatingFileHandler))
self.assertEqual(
[handler.maxBytes, handler.baseFilename, handler.backupCount],
[0, '/tmp/log_file_plain', 0])
# The rotating file gets created since only allow 50 bytes.
self.assertTrue(os.path.exists('/tmp/log_file_plain'))
# Test invalid max bytes, backup count configurations
parsed_url = urlsplit(
'file:///tmp/log_file_bad?max_bytes=yus&backup_count=5y')
publisher = file.FilePublisher(parsed_url)
publisher.publish_counters(None, self.test_data)
self.assertIsNone(publisher.publisher_logger)
def test_published_with_policy_sized_queue_and_rpc_down(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue&max_queue_length=3'))
for i in range(5):
for s in self.test_data:
s.source = 'test-%d' % i
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 3)
self.assertEqual(
publisher.local_queue[0][2]['args']['data'][0]['source'], 'test-2')
self.assertEqual(
publisher.local_queue[1][2]['args']['data'][0]['source'], 'test-3')
self.assertEqual(
publisher.local_queue[2][2]['args']['data'][0]['source'], 'test-4')
def test_file_publisher_maxbytes(self):
# Test valid configurations
tempdir = tempfile.mkdtemp()
name = '%s/log_file' % tempdir
parsed_url = utils.urlsplit('file://%s?max_bytes=50&backup_count=3' %
name)
publisher = file.FilePublisher(parsed_url)
publisher.publish_samples(None, self.test_data)
handler = publisher.publisher_logger.handlers[0]
self.assertIsInstance(handler, logging.handlers.RotatingFileHandler)
self.assertEqual(
[50, name, 3],
[handler.maxBytes, handler.baseFilename, handler.backupCount])
# The rotating file gets created since only allow 50 bytes.
self.assertTrue(os.path.exists('%s.1' % name))
def test_published_with_policy_default_sized_queue_and_rpc_down(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue'))
for i in range(2000):
for s in self.test_data:
s.source = 'test-%d' % i
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 0)
self.assertEqual(len(publisher.local_queue), 1024)
self.assertEqual(
publisher.local_queue[0][2]['args']['data'][0]['source'],
'test-976')
self.assertEqual(
publisher.local_queue[1023][2]['args']['data'][0]['source'],
'test-1999')
def test_published_with_policy_default_sized_queue_and_rpc_down(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue'))
side_effect = oslo.messaging._drivers.common.RPCException()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.side_effect = side_effect
for i in range(0, 2000):
for s in self.test_data:
s.source = 'test-%d' % i
publisher.publish_samples(mock.MagicMock(), self.test_data)
self.assertEqual(1024, len(publisher.local_queue))
self.assertEqual('test-976', publisher.local_queue[0][2][0]['source'])
self.assertEqual('test-1999',
publisher.local_queue[1023][2][0]['source'])
def connect(self, url):
connection_options = pymongo.uri_parser.parse_uri(url)
del connection_options['database']
del connection_options['username']
del connection_options['password']
del connection_options['collection']
pool_key = tuple(connection_options)
if pool_key in self._pool:
client = self._pool.get(pool_key)()
if client:
return client
splitted_url = network_utils.urlsplit(url)
log_data = {'db': splitted_url.scheme,
'nodelist': connection_options['nodelist']}
LOG.info(_('Connecting to %(db)s on %(nodelist)s') % log_data)
client = self._mongo_connect(url)
self._pool[pool_key] = weakref.ref(client)
return client
def _parse_connection_url(url):
"""Parse connection parameters from a database url.
.. note::
HBase Thrift does not support authentication and there is no
database name, so we are not looking for these in the url.
"""
opts = {}
result = network_utils.urlsplit(url)
opts['table_prefix'] = urlparse.parse_qs(result.query).get(
'table_prefix', [None])[0]
opts['dbtype'] = result.scheme
if ':' in result.netloc:
opts['host'], port = result.netloc.split(':')
else:
opts['host'] = result.netloc
port = 9090
opts['port'] = port and int(port) or 9090
return opts
def test_file_publisher(self):
# Test missing max bytes, backup count configurations
tempdir = tempfile.mkdtemp()
name = '%s/log_file_plain' % tempdir
parsed_url = utils.urlsplit('file://%s' % name)
publisher = file.FilePublisher(parsed_url)
publisher.publish_samples(None, self.test_data)
handler = publisher.publisher_logger.handlers[0]
self.assertIsInstance(handler, logging.handlers.RotatingFileHandler)
self.assertEqual(
[0, name, 0],
[handler.maxBytes, handler.baseFilename, handler.backupCount])
# Test the content is corrected saved in the file
self.assertTrue(os.path.exists(name))
with open(name, 'r') as f:
content = f.read()
for sample_item in self.test_data:
self.assertTrue(sample_item.id in content)
self.assertTrue(sample_item.timestamp in content)
def test_published_with_per_meter_topic(self):
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?per_meter_topic=1'))
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.published), 4)
for topic, rpc_call in self.published:
meters = rpc_call['args']['data']
self.assertIsInstance(meters, list)
if topic != self.CONF.publisher_rpc.metering_topic:
self.assertEqual(
len(set(meter['counter_name'] for meter in meters)), 1,
"Meter are published grouped by name")
topics = [topic for topic, meter in self.published]
self.assertIn(self.CONF.publisher_rpc.metering_topic, topics)
self.assertIn(self.CONF.publisher_rpc.metering_topic + '.' + 'test',
topics)
self.assertIn(self.CONF.publisher_rpc.metering_topic + '.' + 'test2',
topics)
self.assertIn(self.CONF.publisher_rpc.metering_topic + '.' + 'test3',
topics)
def test_published(self):
self.data_sent = []
with mock.patch('socket.socket',
self._make_fake_socket(self.data_sent)):
publisher = udp.UDPPublisher(
network_utils.urlsplit('udp://somehost'))
publisher.publish_samples(None, self.test_data)
self.assertEqual(len(self.data_sent), 5)
sent_counters = []
for data, dest in self.data_sent:
counter = msgpack.loads(data)
sent_counters.append(counter)
# Check destination
self.assertEqual(dest, ('somehost', self.CONF.collector.udp_port))
# Check that counters are equal
self.assertEqual(sorted(sent_counters),
sorted([dict(d.as_dict()) for d in self.test_data]))
def _handle_action(self, action, alarm_id, previous, current, reason,
reason_data):
try:
action = network_utils.urlsplit(action)
except Exception:
LOG.error(
_("Unable to parse action %(action)s for alarm %(alarm_id)s"),
{
'action': action,
'alarm_id': alarm_id
})
return
try:
notifier = self.notifiers[action.scheme].obj
except KeyError:
scheme = action.scheme
LOG.error(
_("Action %(scheme)s for alarm %(alarm_id)s is unknown, "
"cannot notify"), {
'scheme': scheme,
'alarm_id': alarm_id
})
return
try:
LOG.debug(
_("Notifying alarm %(id)s with action %(act)s") %
({
'id': alarm_id,
'act': action
}))
notifier.notify(action, alarm_id, previous, current, reason,
reason_data)
except Exception:
LOG.exception(_("Unable to notify alarm %s"), alarm_id)
return
def _handle_action(self, action, alarm_id, previous, current, reason):
try:
action = network_utils.urlsplit(action)
except Exception:
LOG.error(
_("Unable to parse action %(action)s for alarm %(alarm_id)s"),
locals())
return
try:
notifier = self.notifiers[action.scheme].obj
except KeyError:
scheme = action.scheme
LOG.error(
_("Action %(scheme)s for alarm %(alarm_id)s is unknown, "
"cannot notify"), locals())
return
try:
LOG.debug("Notifying alarm %s with action %s", alarm_id, action)
notifier.notify(action, alarm_id, previous, current, reason)
except Exception:
LOG.exception(_("Unable to notify alarm %s"), alarm_id)
return
def test_published_no_mock(self):
publisher = rpc.RPCPublisher(network_utils.urlsplit('rpc://'))
endpoint = mock.MagicMock(['record_metering_data'])
collector = messaging.get_rpc_server(
self.transport, self.CONF.publisher_rpc.metering_topic, endpoint)
endpoint.record_metering_data.side_effect = (
lambda *args, **kwds: collector.stop())
collector.start()
eventlet.sleep()
publisher.publish_samples(context.RequestContext(), self.test_data)
collector.wait()
class Matcher(object):
@staticmethod
def __eq__(data):
for i, sample_item in enumerate(data):
if sample_item['counter_name'] != self.test_data[i].name:
return False
return True
endpoint.record_metering_data.assert_called_once_with(mock.ANY,
data=Matcher())
def test_published_with_policy_queue_and_rpc_down_up(self):
self.rpc_unreachable = True
publisher = rpc.RPCPublisher(
network_utils.urlsplit('rpc://?policy=queue'))
side_effect = oslo.messaging._drivers.common.RPCException()
with mock.patch.object(publisher.rpc_client, 'prepare') as prepare:
prepare.side_effect = side_effect
publisher.publish_samples(mock.MagicMock(), self.test_data)
self.assertEqual(1, len(publisher.local_queue))
prepare.side_effect = mock.MagicMock()
publisher.publish_samples(mock.MagicMock(), self.test_data)
self.assertEqual(0, len(publisher.local_queue))
topic = self.CONF.publisher_rpc.metering_topic
expected = [
mock.call(topic=topic),
mock.call(topic=topic),
mock.call(topic=topic)
]
self.assertEqual(expected, prepare.mock_calls)
def test_get_security_name(self):
self.assertEqual(self.inspector._get_security_name(self.host),
self.inspector._security_name)
host2 = network_utils.urlsplit("snmp://foo:80?security_name=fake")
self.assertEqual(self.inspector._get_security_name(host2), 'fake')
