def test_simple_strict_asserted_by_check_present_unordered(self):
log_capture = LogCapture(ensure_checks_above=ERROR)
root.error('during')
log_capture.uninstall()
log_capture.check_present(("root", "ERROR", "during"),
order_matters=False)
log_capture.ensure_checked()
def _init_log_check(log_out: LogCapture, expected1: str, expected2: str,
expected3: str, expected4: str) -> None:
root = 'root'
log_level = al.logging.getLevelName(al.logging.INFO)
log_out.check_present(
(root, log_level, expected1), (root, log_level, expected2),
(root, log_level, expected3), (root, log_level, expected4))
async def test_register_device(http_client, base_url):
os.environ["TESTER_NUMBERS"] += f",{TEST_PHONE_NUMBER}"
await clean_test_user()
await test_utils.create_test_user(TEST_PHONE_NUMBER)
capture = LogCapture()
# Get the user
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
# Remove all devices associated with user
await graph.dissociate_user_devices(user)
capture.check_present((
"tornado.application",
"INFO",
f"Dissociating devices from {utils.mask_phone(TEST_PHONE_NUMBER)}",
))
with mock.patch.object(corona_backend.onboarding.app.RegisterDeviceHandler,
"get_current_user") as m:
m.return_value = get_test_payload()
response = await http_client.fetch(f"{base_url}/register-device",
**post_request_args())
# Check that `get_current_user` was called
m.assert_any_call()
capture.check_present((
"tornado.application",
"INFO",
"Tester {0} is impersonating test user {0}".format(
TEST_PHONE_NUMBER),
))
assert response.code == 200
body = json.loads(response.body)
# Check phone number
assert body["PhoneNumber"] == TEST_PHONE_NUMBER
# Check keys in connection string
for key in ["HostName=", "DeviceId=", "SharedAccessKey="]:
assert key in body["ConnectionString"]
# Check that we have the correct device_id
# Get all devices associated with user
groups = await graph.graph_request(f"/users/{user['id']}/memberOf")
# For this test user we should only have one device
assert len(groups) == 1
device = groups[0]
assert body["DeviceId"] == device["displayName"]
# Clean up
await graph.dissociate_user_devices(user)
capture.uninstall()
def test_simple_strict_not_asserted_by_check_present(self):
log_capture = LogCapture(ensure_checks_above=ERROR)
root.error('before')
root.error('during')
log_capture.uninstall()
log_capture.check_present(("root", "ERROR", "during"))
with ShouldAssert(
"Not asserted ERROR log(s): [('root', 'ERROR', 'before')]"):
log_capture.ensure_checked()
async def test_find_groups_to_delete_without_timestamp(user):
# Create fake id
device_id = "".join(str(uuid.uuid1()).split("-"))
device = await graph.store_device_id(user, device_id)
group_id = device["id"]
groups = await find_groups_to_delete()
old_device_ids = {group["displayName"] for group in groups}
assert device_id not in old_device_ids
# Mark group for deletion without a timestamp
await graph.graph_request(
f"/groups/{group_id}",
method="PATCH",
body=json.dumps({
graph.extension_attr_name("toDelete"): True,
graph.extension_attr_name("toDeleteDate"): None,
}),
headers={"Content-Type": "application/json"},
)
capture = LogCapture("tornado.application")
# Since there is not timestamp it should not have been deleted
groups = await find_groups_to_delete()
device_ids = {group["displayName"] for group in groups}
assert device_id not in device_ids
capture.check_present(
(
"tornado.application",
"WARNING",
f"Group {device_id} marked toDelete, but no date! Saving for later.",
),
(
"tornado.application",
"INFO",
f"Marking device id group {device_id} for deletion",
),
)
# Calling it again should now delete it
groups = await find_groups_to_delete()
device_ids = {group["displayName"] for group in groups}
assert device_id in device_ids
await graph.delete_device_group(device_id)
capture.uninstall()
async def test_register_device_create_new_device_reach_consecutive_failure_limit_raises_500( # noqa
http_client,
base_url,
app,
):
async def timeout_error(*args, **kwargs):
await asyncio.sleep(
float(corona_backend.onboarding.app.PROVISIONING_TIMEOUT) + 1)
device_future = asyncio.ensure_future(timeout_error())
app.settings["consecutive_failures"] = 0
app.settings["consecutive_failure_limit"] = 1
capture = LogCapture("tornado.application")
with mock.patch.object(corona_backend.onboarding.app.RegisterDeviceHandler,
"get_current_user") as m_auth:
m_auth.return_value = get_test_payload()
with mock.patch.object(devices,
"create_new_device") as m_create_device:
m_create_device.return_value = device_future
with pytest.raises(tornado.httpclient.HTTPClientError) as e:
await http_client.fetch(f"{base_url}/register-device",
**post_request_args())
assert e.value.code == 500
assert app.settings["consecutive_failures"] == 1
# Check log
capture.check_present(
(
"tornado.application",
"ERROR",
"Timeout registering device (1/1 before abort)",
),
(
"tornado.application",
"CRITICAL",
"Aborting due to consecutive failure limit!",
),
)
capture.uninstall()
async def test_get_user_token():
test_phone_number = "+00001234"
# create test user
existing_user = await graph.find_user_by_phone(test_phone_number)
if existing_user is None:
await test_utils.create_test_user(test_phone_number)
# remove devices associated with the user
capture = LogCapture()
user = await graph.find_user_by_phone(test_phone_number)
await graph.dissociate_user_devices(user)
capture.check_present(
(
"tornado.application",
"INFO",
f"Dissociating devices from {utils.mask_phone(test_phone_number)}",
)
)
handler = MagicMock()
key = os.urandom(5)
import jwt
payload = {
"aud": "abc-123",
"scp": "Device.Write",
"signInNames.phoneNumber": test_phone_number,
}
token = jwt.encode(payload, key=key, headers={"kid": "test"}).decode("utf8")
handler.request.headers = {"Authorization": f"Bearer {token}"}
with mock.patch.object(
jwt, "decode", lambda *args, **kwargs: payload
), mock.patch.dict(graph._PUBLIC_KEYS, {"test": key}):
authenticated = graph.get_user_token(handler)
assert authenticated is not None
assert authenticated["_access_token"] == token
assert authenticated["_phonenumber"] == test_phone_number
# clean up
await test_utils.delete_test_user(user["id"])
capture.uninstall()
async def test_revoke_consent_handler_response(http_client, base_url):
await clean_test_user()
await test_utils.create_test_user(TEST_PHONE_NUMBER)
capture = LogCapture("tornado.application")
with mock.patch.object(corona_backend.onboarding.app.RevokeConsentHandler,
"get_current_user") as m_auth:
m_auth.return_value = get_test_payload()
response = await http_client.fetch(f"{base_url}/revoke-consent",
**post_request_args())
assert response.code == 200
body = json.loads(response.body)
assert body["Status"] == "Success"
assert "Your phone number is no longer associated" in body["Message"]
capture.check_present(
(
"tornado.application",
"INFO",
f"Storing revoked consent on user {MASKED_TEST_PHONE_NUMBER}",
),
(
"tornado.application",
"INFO",
f"Dissociating devices from {MASKED_TEST_PHONE_NUMBER}",
),
(
"tornado.application",
"INFO",
f"Deleting user {MASKED_TEST_PHONE_NUMBER}",
),
)
capture.uninstall()
async def test_register_device_with_consent_revoked_resets_consent(
http_client, base_url):
await clean_test_user()
await test_utils.create_test_user(TEST_PHONE_NUMBER)
os.environ["TESTER_NUMBERS"] += f",{TEST_PHONE_NUMBER}"
capture = LogCapture()
# Get the user
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
await graph.store_consent_revoked(user)
capture.check_present((
"tornado.application",
"INFO",
f"Storing revoked consent on user {utils.mask_phone(TEST_PHONE_NUMBER)}",
))
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
assert graph.extension_attr_name("consentRevoked") in user
assert user[graph.extension_attr_name("consentRevoked")] is True
with mock.patch.object(corona_backend.onboarding.app.RegisterDeviceHandler,
"get_current_user") as m:
m.return_value = get_test_payload()
await http_client.fetch(f"{base_url}/register-device",
**post_request_args())
capture.check_present((
"tornado.application",
"WARNING",
f"Clearing revoked consent on user {utils.mask_phone(TEST_PHONE_NUMBER)}",
))
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
assert graph.extension_attr_name("consentRevoked") not in user
capture.uninstall()
async def test_register_device_without_test_number_in_test_enviroment(
http_client, base_url):
phone_number = os.environ.get("TESTER_NUMBERS", "+00").split(",")[0]
if phone_number.startswith("+00"):
return
capture = LogCapture()
with mock.patch.object(corona_backend.onboarding.app.RegisterDeviceHandler,
"get_current_user") as m:
m.return_value = get_test_payload(phone_number)
with pytest.raises(tornado.httpclient.HTTPClientError) as e:
await http_client.fetch(f"{base_url}/register-device",
**post_request_args(phone_number))
assert e.value.code == 403
capture.check_present((
"tornado.application",
"ERROR",
"Tester {0} attempted to impersonate non-test user {0}".format(
phone_number),
))
capture.uninstall()
def test_custom_content_length_bad(self):
request = Request(self.getURL('contentlength'))
actual_content_length = str(len(request.body))
bad_content_length = str(len(request.body) + 1)
request.headers['Content-Length'] = bad_content_length
log = LogCapture()
d = self.download_request(request, Spider('foo'))
d.addCallback(lambda r: r.text)
d.addCallback(self.assertEqual, actual_content_length)
d.addCallback(lambda _: log.check_present((
'scrapy.core.http2.stream',
'WARNING',
f'Ignoring bad Content-Length header '
f'{bad_content_length!r} of request {request}, sending '
f'{actual_content_length!r} instead',
)))
d.addCallback(lambda _: log.uninstall())
return d
class TestPanoptesPluginRunner(unittest.TestCase):
@staticmethod
def extract(record):
message = record.getMessage()
match_obj = re.match(r'(?P<name>.*):\w+(?P<body>.*)', message)
if match_obj:
message = match_obj.group('name') + match_obj.group('body')
match_obj = re.match(
r'(?P<start>.*[R|r]an in\s)\d+\.?\d*.*(?P<end>seconds.*)', message)
if match_obj:
return record.name, record.levelname, match_obj.group(
'start') + match_obj.group('end')
match_obj = re.match(
r'(?P<start>.*took\s*)\d+\.?\d*.*(?P<seconds>seconds.*)\d+\s(?P<end>garbage objects.*)',
message)
if match_obj:
return record.name, record.levelname, match_obj.group('start') + match_obj.group('seconds') + \
match_obj.group('end')
match_obj = re.match(
r'(?P<start>Attempting to get lock for plugin .*with lock path) \".*\".*(?P<id> and identifier).*'
r'(?P<in> in) \d\.?\d*(?P<seconds> seconds)', message)
if match_obj:
return record.name, record.levelname, match_obj.group('start') + match_obj.group('id') + \
match_obj.group('in') + match_obj.group('seconds')
return record.name, record.levelname, message
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
@patch('kazoo.client.KazooClient', panoptes_mock_kazoo_client)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = get_test_conf_file()
self._panoptes_context = PanoptesContext(
self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore],
create_message_producer=False,
async_message_producer=False,
create_zookeeper_client=True)
self._runner_class = PanoptesPluginRunner
self._log_capture = LogCapture(attributes=self.extract)
def tearDown(self):
self._log_capture.uninstall()
def test_logging_methods(self):
runner = self._runner_class("Test Polling Plugin", "polling",
PanoptesPollingPlugin, PanoptesPluginInfo,
None, self._panoptes_context,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
# Ensure logging methods run:
runner.info(PanoptesTestPluginNoLock(), "Test Info log message")
runner.warn(PanoptesTestPluginNoLock(), "Test Warning log message")
runner.error(PanoptesTestPluginNoLock(), "Test Error log message",
Exception)
runner.exception(PanoptesTestPluginNoLock(),
"Test Exception log message")
self._log_capture.check(
('panoptes.tests.test_runner', 'INFO',
'[None] [{}] Test Info log message'),
('panoptes.tests.test_runner', 'WARNING',
'[None] [{}] Test Warning log message'),
('panoptes.tests.test_runner', 'ERROR',
"[None] [{}] Test Error log message: <type 'exceptions.Exception'>"
), ('panoptes.tests.test_runner', 'ERROR',
'[None] [{}] Test Exception log message'))
def test_basic_operations(self):
runner = self._runner_class("Test Polling Plugin", "polling",
PanoptesPollingPlugin, PanoptesPluginInfo,
None, self._panoptes_context,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'Attempting to execute plugin "Test Polling Plugin"'),
('panoptes.tests.test_runner', 'DEBUG',
'''Starting Plugin Manager for "polling" plugins with the following '''
'''configuration: {'polling': <class'''
""" 'yahoo_panoptes.polling.polling_plugin.PanoptesPollingPlugin'>}, """
"""['tests/plugins/polling'], panoptes-plugin"""),
('panoptes.tests.test_runner', 'DEBUG', 'Found 3 plugins'),
('panoptes.tests.test_runner', 'DEBUG', 'Loaded plugin '
'"Test Polling Plugin", version "0.1" of type "polling"'
', category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin 2", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin Second Instance", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'INFO',
'''[Test Polling Plugin] [None] '''
'''Attempting to get lock for plugin "Test Polling Plugin"'''),
('panoptes.tests.test_runner', 'DEBUG',
'Attempting to get lock for plugin "Test Polling Plugin", with lock path and '
'identifier in seconds'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [None] Acquired lock'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [None]'
' Ran in seconds'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [None] Released lock'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [None] Plugin returned'
' a result set with 1 members'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [None]'
' Callback function ran in seconds'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [None] GC took seconds. There are garbage objects.'
), ('panoptes.tests.test_runner', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin_0'),
('panoptes.tests.test_runner', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin_Second_Instance_0'
),
order_matters=False)
def test_nonexistent_plugin(self):
runner = self._runner_class("Non-existent Plugin", "polling",
PanoptesPollingPlugin, PanoptesPluginInfo,
None, self._panoptes_context,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present((
'panoptes.tests.test_runner', 'INFO',
'Attempting to execute plugin "Non-existent Plugin"'
), (
'panoptes.tests.test_runner', 'DEBUG',
'Starting Plugin Manager for "polling" plugins with the following '
"configuration: {'polling': <class 'yahoo_panoptes.polling.polling_plugin."
"PanoptesPollingPlugin'>}, "
"['tests/plugins/polling'], panoptes-plugin"
), ('panoptes.tests.test_runner', 'DEBUG', 'Found 3 plugins'), (
'panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin", version "0.1" of type "polling", '
'category "polling"'
), ('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin Second Instance", version "0.1" of type '
'"polling", category "polling"'), (
'panoptes.tests.test_runner', 'WARNING',
'No plugin named "Non-existent Plugin" found in "'
'''['tests/plugins/polling']"'''))
def test_bad_plugin_type(self):
runner = self._runner_class("Test Polling Plugin", "bad",
PanoptesPollingPlugin, PanoptesPluginInfo,
None, self._panoptes_context,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present((
'panoptes.tests.test_runner', 'ERROR',
'''Error trying to load plugin "Test Polling Plugin": KeyError('bad',)'''
))
def test_execute_now_false(self):
mock_get_plugin_by_name = MagicMock(
return_value=MockPluginExecuteNow())
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginManager.getPluginByName',
mock_get_plugin_by_name):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, None, self._panoptes_context,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'Attempting to execute plugin "Test Polling Plugin"'),
('panoptes.tests.test_runner', 'DEBUG',
'''Starting Plugin Manager for '''
'''"polling" plugins with the '''
'''following configuration: {'polling': '''
"""<class 'yahoo_panoptes.polling.polling_plugin.PanoptesPollingPlugin'"""
""">}, ['tests/plugins/polling'], panoptes-plugin"""),
('panoptes.tests.test_runner', 'DEBUG', 'Found 3 plugins'),
('panoptes.tests.test_runner', 'DEBUG', 'Loaded plugin '
'"Test Polling Plugin", version "0.1" of type "polling"'
', category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin Second Instance", '
'version "0.1" of type "polling", category "polling"'))
def test_callback_failure(self):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, None, self._panoptes_context,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback_with_exception)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'ERROR', '[Test Polling Plugin] '
'[None] Results callback function failed'))
def test_lock_no_lock_object(self):
mock_plugin = MagicMock(return_value=PanoptesTestPluginNoLock)
mock_get_context = MagicMock(return_value=self._panoptes_context)
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginManager.getPluginByName',
mock_plugin):
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginRunner._get_context',
mock_get_context):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, None, self._panoptes_context,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'ERROR',
'[None] [{}] Error in acquiring lock'))
def test_lock_is_none(self):
mock_get_plugin_by_name = MagicMock(return_value=MockPluginLockNone())
mock_get_context = MagicMock(return_value=self._panoptes_context)
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginManager.getPluginByName',
mock_get_plugin_by_name):
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginRunner._get_context',
mock_get_context):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, None, self._panoptes_context,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'[None] [{}] Attempting to get lock for plugin'
' "Test Polling Plugin"'))
def test_lock_is_not_locked(self):
mock_get_plugin_by_name = MagicMock(
return_value=MockPluginLockIsNotLocked())
mock_get_context = MagicMock(return_value=self._panoptes_context)
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginManager.getPluginByName',
mock_get_plugin_by_name):
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginRunner._get_context',
mock_get_context):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, None, self._panoptes_context,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'[None] [{}] Attempting to get lock for plugin'
' "Test Polling Plugin"'))
def test_plugin_failure(self):
mock_plugin = MagicMock(
return_value=PanoptesTestPluginRaisePluginReleaseException)
mock_get_context = MagicMock(return_value=self._panoptes_context)
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginManager.getPluginByName',
mock_plugin):
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesPluginRunner._get_context',
mock_get_context):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, None, self._panoptes_context,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'ERROR',
'[None] [{}] Failed to execute plugin'),
('panoptes.tests.test_runner', 'INFO',
'[None] [{}] Ran in seconds'),
('panoptes.tests.test_runner', 'ERROR',
'[None] [{}] Failed to release lock for plugin'),
('panoptes.tests.test_runner', 'WARNING',
'[None] [{}] Plugin did not return any results'))
def test_plugin_wrong_result_type(self):
runner = self._runner_class("Test Polling Plugin 2", "polling",
PanoptesPollingPlugin, PanoptesPluginInfo,
None, self._panoptes_context,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present((
'panoptes.tests.test_runner', 'WARNING',
'[Test Polling Plugin 2] [None] Plugin returned an unexpected result type: '
'"PanoptesMetricsGroup"'))
class TestPanoptesPluginWithEnrichmentRunner(TestPanoptesPluginRunner):
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
@patch('kazoo.client.KazooClient', panoptes_mock_kazoo_client)
def setUp(self):
super(TestPanoptesPluginWithEnrichmentRunner, self).setUp()
self._panoptes_resource = PanoptesResource(
resource_site="test",
resource_class="test",
resource_subclass="test",
resource_type="test",
resource_id="test",
resource_endpoint="test",
resource_creation_timestamp=_TIMESTAMP,
resource_plugin="test")
self._runner_class = PanoptesPluginWithEnrichmentRunner
def test_basic_operations(self):
# Test where enrichment is None
mock_panoptes_enrichment_cache = Mock(return_value=None)
with patch(
'yahoo_panoptes.framework.plugins.runner.PanoptesEnrichmentCache',
mock_panoptes_enrichment_cache):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, self._panoptes_resource,
self._panoptes_context, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
"plugin_logger", PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present((
'panoptes.tests.test_runner', 'ERROR',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test] '
'Could not set up context for plugin'))
self._log_capture.uninstall()
self._log_capture = LogCapture(attributes=self.extract)
# Test with enrichment
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, self._panoptes_resource,
self._panoptes_context, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
"plugin_logger", PanoptesMetricsGroupSet, _callback)
runner.execute_plugin()
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'Attempting to execute plugin "Test Polling Plugin"'),
('panoptes.tests.test_runner', 'DEBUG',
'''Starting Plugin Manager for "polling" plugins with the following '''
'''configuration: {'polling': <class'''
""" 'yahoo_panoptes.polling.polling_plugin.PanoptesPollingPlugin'>}, """
"""['tests/plugins/polling'], panoptes-plugin"""),
('panoptes.tests.test_runner', 'DEBUG', 'Found 3 plugins'),
('panoptes.tests.test_runner', 'DEBUG', 'Loaded plugin '
'"Test Polling Plugin", version "0.1" of type "polling"'
', category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin 2", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin Second Instance", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test] Attempting to get lock for plugin '
'"Test Polling Plugin"'),
('panoptes.tests.test_runner', 'DEBUG',
'Attempting to get lock for plugin "Test Polling Plugin", with lock path and '
'identifier in seconds'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test] Acquired lock'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test]'
' Ran in seconds'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test] Released lock'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test] Plugin returned'
' a result set with 1 members'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test]'
' Callback function ran in seconds'),
('panoptes.tests.test_runner', 'INFO',
'[Test Polling Plugin] [plugin|test|site|test|class|test|subclass|test|type|'
'test|id|test|endpoint|test] GC took seconds. There are garbage objects.'
),
('panoptes.tests.test_runner', 'ERROR',
'No enrichment data found on KV store for plugin Test Polling Plugin '
'resource test namespace test using key test'),
('panoptes.tests.test_runner', 'DEBUG',
'Successfully created PanoptesEnrichmentCache enrichment_data {} for plugin '
'Test Polling Plugin'),
order_matters=False)
def test_callback_failure(self):
runner = self._runner_class(
"Test Polling Plugin", "polling", PanoptesPollingPlugin,
PanoptesPluginInfo, self._panoptes_resource,
self._panoptes_context, PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, PanoptesTestKeyValueStore,
"plugin_logger", PanoptesMetricsGroupSet, _callback_with_exception)
runner.execute_plugin()
self._log_capture.check_present((
'panoptes.tests.test_runner', 'ERROR', '[Test Polling Plugin] '
'[plugin|test|site|test|class|test|subclass|test|'
'type|test|id|test|endpoint|test] Results callback function failed'
))
# 'pass' is needed for these methods because the only difference in their logging output from
# TestPanoptesPluginRunner is the presence of the PanoptesResource in some log messages.
def test_lock_no_lock_object(self):
pass
def test_lock_is_none(self):
pass
def test_lock_is_not_locked(self):
pass
def test_plugin_failure(self):
pass
def test_plugin_wrong_result_type(self):
runner = self._runner_class("Test Polling Plugin 2", "polling",
PanoptesPollingPlugin, PanoptesPluginInfo,
None, self._panoptes_context,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore,
PanoptesTestKeyValueStore, "plugin_logger",
PanoptesMetric, _callback)
runner.execute_plugin()
self._log_capture.check_present((
'panoptes.tests.test_runner', 'ERROR',
'[Test Polling Plugin 2] [None] Could not set up context for plugin'
))
class TestPanoptesPollingPluginRunner(unittest.TestCase):
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
@patch('kazoo.client.KazooClient', panoptes_mock_kazoo_client)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = get_test_conf_file()
self._panoptes_resource = PanoptesResource(
resource_site="test",
resource_class="test",
resource_subclass="test",
resource_type="test",
resource_id="test",
resource_endpoint="test",
resource_creation_timestamp=_TIMESTAMP,
resource_plugin="test")
self._panoptes_context = PanoptesContext(
self.panoptes_test_conf_file,
key_value_store_class_list=[
PanoptesTestKeyValueStore, PanoptesResourcesKeyValueStore,
PanoptesPollingPluginKeyValueStore, PanoptesSecretsStore,
PanoptesPollingPluginAgentKeyValueStore,
PanoptesDiscoveryPluginAgentKeyValueStore,
PanoptesDiscoveryPluginKeyValueStore
],
create_message_producer=False,
async_message_producer=False,
create_zookeeper_client=True)
self._runner_class = PanoptesPluginWithEnrichmentRunner
self._log_capture = LogCapture(
attributes=TestPanoptesPluginRunner.extract)
def tearDown(self):
self._log_capture.uninstall()
def tearDown(self):
self._log_capture.uninstall()
@patch('yahoo_panoptes.framework.metrics.time')
@patch(
'yahoo_panoptes.framework.context.PanoptesContext._get_message_producer'
)
@patch('yahoo_panoptes.framework.context.PanoptesContext.message_producer',
new_callable=PropertyMock)
@patch(
'yahoo_panoptes.polling.polling_plugin_agent.PanoptesPollingTaskContext'
)
@patch(
'yahoo_panoptes.framework.resources.PanoptesResourceStore.get_resource'
)
def test_polling_plugin_agent(self, resource, panoptes_context,
message_producer, message_producer_property,
time):
producer = MockPanoptesMessageProducer()
time.return_value = 1
message_producer.return_value = producer
message_producer_property.return_value = producer
resource.return_value = self._panoptes_resource
panoptes_context.return_value = self._panoptes_context
polling_plugin_task('Test Polling Plugin', 'polling')
log_prefix = '[Test Polling Plugin] [plugin|test|site|test|class|test|' \
'subclass|test|type|test|id|test|endpoint|test]'
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'Attempting to execute plugin "Test Polling Plugin"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin 2", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'ERROR',
'No enrichment data found on KV store for plugin Test'
' Polling Plugin resource test namespace test using key test'),
('panoptes.tests.test_runner', 'DEBUG',
'Successfully created PanoptesEnrichmentCache enrichment_data '
'{} for plugin Test Polling Plugin'),
('panoptes.tests.test_runner', 'DEBUG',
'Attempting to get lock for plugin "Test Polling Plugin", '
'with lock path and identifier in seconds'),
('panoptes.tests.test_runner', 'INFO',
'{} Acquired lock'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Plugin returned a result set with 1 members'.format(
log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Callback function ran in seconds'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Ran in seconds'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Released lock'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} GC took seconds. There are garbage objects.'.format(
log_prefix)),
('panoptes.tests.test_runner', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin'),
('panoptes.tests.test_runner', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin'),
('panoptes.tests.test_runner', 'DEBUG', 'Deleting module: '
'yapsy_loaded_plugin_Test_Polling_Plugin_Second_Instance'),
order_matters=False)
kafka_push_log = {
"metrics_group_type":
"Test",
"metrics_group_interval":
60,
"metrics_group_creation_timestamp":
1,
"metrics_group_schema_version":
"0.2",
"resource": {
"resource_site": "test",
"resource_class": "test",
"resource_subclass": "test",
"resource_type": "test",
"resource_id": "test",
"resource_endpoint": "test",
"resource_metadata": {
"_resource_ttl": "604800"
},
"resource_creation_timestamp": 1.0,
"resource_plugin": "test"
},
"metrics": [{
"metric_creation_timestamp": 1,
"metric_name": "test",
"metric_value": 0.0,
"metric_type": "gauge"
}],
"dimensions": []
}
# Timestamps need to be removed to check Panoptes Metrics
metric_groups_seen = 0
for line in self._log_capture.actual():
_, _, log = line
if 'resource_creation_timestamp' in log:
log = re.sub(r"resource_creation_timestamp\": \d+\.\d+,",
"resource_creation_timestamp\": 1.0,", log)
resource_match = re.search(r'{.*}', log)
if resource_match is not None:
self.assertEqual(
ordered(json.loads(resource_match.group(0))),
ordered(kafka_push_log))
if log.startswith('Sent metric group'):
metric_groups_seen += 1
if log.startswith('Going to send metric group'):
metric_groups_seen += 1
self.assertEqual(metric_groups_seen, 2)
