def setUpClass(cls):
# The rule to evaluate number as boolean is inspired by C++11 implementation
# All numbers but 0 and True or true will be evaluated as True
cls.logging = re.search(r'^-?[1-9][0-9]*$|^[Tt]rue$',
os.getenv("LOG_TO_STDOUT", "False")) != None
path = os.getenv("CONFIG_YAML", DEFAULT_CONFIG_YAML)
cls.account = read_yaml_file(path)
cls.account["credentials"] = (lookup_values(
cls.account.get("credentials_keys")))
if "resources" in cls.account:
cls.resource_data = cls.account.get("resources")
else:
cls.resource_data = [{
"resource":
cls.account.get("resource"),
"expected_metrics":
cls.account.get("expected_metrics")
}]
cls.runner = Flexer()
cfg = load_config(cfg_file=CONFIG_FILE)["regions"]["default"]
client = CmpClient(url=cfg["cmp_url"],
auth=(cfg['cmp_api_key'], cfg['cmp_api_secret']))
cls.context = FlexerContext(cmp_client=client)
secrets = (lookup_values(cls.account.get("secrets_keys")))
cls.context.secrets = secrets
def setUpClass(cls):
path = os.getenv("CONFIG_YAML", DEFAULT_CONFIG_YAML)
cls.account = read_yaml_file(path)
cls.account["credentials"] = (lookup_credentials(
cls.account.get("credentials_keys")))
cls.runner = Flexer()
cfg = load_config(cfg_file=CONFIG_FILE)["regions"]["default"]
client = CmpClient(url=cfg["cmp_url"],
auth=(cfg['cmp_api_key'], cfg['cmp_api_secret']))
cls.context = FlexerContext(cmp_client=client)
def run(handler, event, config, secrets, cmp_client):
event = json.loads(event)
handler = "main.%s" % handler
context = FlexerContext(cmp_client=cmp_client)
if config is not None:
context.config = json.loads(config)
if secrets is not None:
context.secrets = json.loads(secrets)
result = Flexer().run(event=event,
context=context,
handler=handler,
debug=True)
return json.loads(result)
def setUp(self):
self.maxDiff = None
self.runner = Flexer()
class TestFlexer(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.runner = Flexer()
def test_run_with_no_handler(self):
"""If run is called without a handler, an exception should be raised"""
handler = ''
expected = {
'exc_message': 'Handler is required',
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_module(self):
"""If run is called with a non-existing module, an exception
should be raised
"""
handler = 'not_here.test'
expected = {
'exc_message': ('Failed to import module "not_here": %s' %
exc_message["nonexisting_module"][PYVERSION]),
'exc_type':
'ImportError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_handler(self):
"""If run is called with a valid module, but non-existing handler,
an exception should be raised
"""
handler = 'module_okay.not_found'
expected = {
'exc_message':
('Handler "not_found" not found in "module_okay": %s' %
exc_message["nonexisting_handler"][PYVERSION]),
'exc_type':
'AttributeError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_builtin_module(self):
"""If a built-in module is passed as a handler to run, an exception
should be raised
To get the list of all modules that are compiled into this Python
interpreter, do: print sys.builtin_module_names
sys happens to be one of these modules
"""
handler = 'sys.exit'
expected = {
'exc_message':
('Built-in module "sys" cannot be a handler module'),
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_invalid_handler(self):
"""If run is called with an invalid handler, i.e passing only a module
or a method, an exception should be raised
"""
handler = 'test'
expected = {
'exc_message': ('Invalid format for handler "test": %s' %
exc_message["invalid_handler"][PYVERSION]),
'exc_type':
'ValueError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_import_error(self):
"""Make sure ImportErrors are handled during module imports.
If there is an ImportError raised, the exception should store the
stack trace.
"""
handler = 'module_with_import_error.test'
exc_m = exc_message["import_error"][PYVERSION]
exc_t = exc_type["import_error"][PYVERSION]
expected = {
'exc_message':
('Failed to import module "module_with_import_error": %s' % exc_m),
'exc_type':
exc_t,
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('%s: %s' % (exc_t, exc_m), error['stack_trace'])
def test_run_with_syntax_error(self):
"""Make sure SyntaxErrors are handled during module imports.
If there is a SyntaxError raised, the exception should store the
stack trace.
"""
handler = 'module_with_syntax_error.test'
expected = {
'exc_message':
('Syntax error in module "module_with_syntax_error": '
'invalid syntax (module_with_syntax_error.py, line 2)'),
'exc_type':
'SyntaxError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('this is a syntax error', error['stack_trace'])
def test_run_with_module_level_exception(self):
"""Make sure all exceptions are handled during module imports.
If there is an Exception raised, a stack trace should be available
in the exception as well
"""
handler = 'module_with_exception.test'
exc_m = exc_message["module_level_exception"][PYVERSION]
expected = {
'exc_message':
('Failed to initialise "module_with_exception": %s' % exc_m),
'exc_type':
'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: %s' % exc_m, error['stack_trace'])
def test_run_with_no_stdout_logs(self):
"""Test the simplest possible use-case: run a method with no side
effects and make sure the result is stored
"""
handler = 'module_okay.test_with_no_logs'
expected = {
'value': 42,
'error': None,
'logs': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_stdout_logs(self):
"""Run a method with side effects and make sure that the result is
stored and the stdout/stderr are captured
"""
handler = 'module_okay.test_with_logs'
expected = {
'value': 42,
'error': None,
'logs': 'This goes to stdout\n',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_handler_exception(self):
"""Run a method that raises an Exception. Store the error details as
a result from the execution and make sure that the stdout/stderr are
still captured. The stack trace is expected to be part of the logs.
"""
handler = 'module_okay.test_with_exception'
exc_m = exc_message["handler_exception"][PYVERSION]
expected = {
'exc_message': exc_m,
'exc_type': 'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Before the exception\nTraceback', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: %s' % exc_m, error['stack_trace'])
@pytest.mark.skip(reason="We don't have to worry about memory locally")
def test_run_with_memory_error(self):
"""Make sure MemoryErrors are handled when running the code.
"""
handler = 'module_with_memory_error.test'
expected = {
'exc_message': '',
'exc_type': 'MemoryError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Starting Up\n', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('MemoryError', error['stack_trace'])
def test_run_with_invalid_db_secret(self):
"""Run a method that tries to use an invalid secret for an Nflex DB
"""
handler = 'module_with_db.test'
expected = {
'exc_message':
'The mydb secret is not a valid MongoDB connection string', # noqa
'exc_type': 'Exception',
}
context = FlexerContext()
secrets = [
"test",
"mongodb://",
"mongodb://a",
"mongodb://*****:*****@",
"mongodb://*****:*****@c",
"mongodb://*****:*****@c/",
]
for s in secrets:
context.secrets = {"_nflexdb_mydb": s}
result = self.runner.run(event={},
context=context,
handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('Exception', error['stack_trace'])
def test_run_with_valid_db_secret(self):
"""Run a method that tries to use a valid secret for an Nflex DB
"""
handler = 'module_with_db.test'
context = FlexerContext()
context.secrets = {"_nflexdb_mydb": "mongodb://*****:*****@c/mydb"}
with mock.patch('pymongo.MongoClient', return_value=mock.MagicMock()):
result = self.runner.run(event={},
context=context,
handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertEqual(None, actual['error'])
self.assertTrue('result' in actual['value'])
def test_run_with_validation_error(self):
"""Run a method that returns data with validation errors and make sure
its being caught and stored in the logs
"""
handler = 'module_with_validation_error.test'
expected = {
'value': None,
'error': {
'exc_message':
'["10 is not of type \'string\' in result.foo"]',
'exc_type': 'ValidationError'
},
'logs': '["10 is not of type \'string\' in result.foo"]',
}
schema = {
'patternProperties': {
'.*': {
'type': 'string'
},
}
}
with mock.patch('flexer.runner.Flexer._get_validation_schema',
return_value=schema):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_validate_metrics_ok(self):
"""Run a method that returns metric data and check validation is ok
"""
handler = 'module_with_metrics.test_ok'
expected = {
u'value': {
u'metrics': [{
u'metric':
u'cpu-usage',
u'value':
99,
u'unit':
u'percent',
u'time':
u'2017-01-12T18:30:42.034751Z',
u'resource_id':
u'1237ab91-08eb-4164-8e68-67699c29cd4c'
}]
},
u'error': None,
u'logs': u'Running test script\n',
}
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_metrics.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_validate_metrics_error(self):
"""Run a method that returns metric data and check validation is ok
"""
handler = 'module_with_metrics.test_invalid'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_metrics.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertIsNone(actual['value'])
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertTrue("\'time\' is a required property" in actual['error']
['exc_message'])
def test_validate_logs_ok(self):
"""Run a method that returns log data and check validation is ok
"""
handler = 'module_with_logs.test_ok'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_logs.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertTrue('logs' in actual['value'])
self.assertEqual(1, len(actual['value']['logs']))
self.assertEqual(None, actual['error'])
def test_validate_logs_error(self):
"""Run a method that returns log data and check validation is ok
"""
handler = 'module_with_logs.test_invalid'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_logs.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertTrue("\'time\' is a required property" in actual['error']
['exc_message'])
def test_validate_logs_invalid_severity_error(self):
"""
Run a method that returns log data and check validation is NG
with invalid severity
"""
handler = 'module_with_logs.test_invalid_severity_value'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_logs.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertIn('HIGH', actual['error']['exc_message'])
self.assertIn('is not one of', actual['error']['exc_message'])
def test_validate_status_ok(self):
"""Run a method that returns status data and check validation is ok
"""
handler = 'module_with_status.test_ok'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_status.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertEqual(None, actual['error'])
self.assertTrue('status' in actual['value'])
self.assertEqual(3, len(actual['value']['status']))
def test_validate_status_missing_level_error(self):
"""Run a method that returns status data and check validation is NG
with missing level
"""
handler = 'module_with_status.test_missing_level'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_status.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertTrue("\'level\' is a required property" in actual['error']
['exc_message'])
def test_validate_status_missing_time_error(self):
"""Run a method that returns status data and check validation is NG
with missing time
"""
handler = 'module_with_status.test_missing_time'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_status.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertTrue("\'time\' is a required property" in actual['error']
['exc_message'])
def test_validate_status_invalid_status_error(self):
"""Run a method that returns status data and check validation is NG
with invalid status
"""
handler = 'module_with_status.test_invalid_level'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_status.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertIn("3 is not one of", actual['error']['exc_message'])
def test_validate_spend_schema(self):
"""
Run a few methods to test spend json schema validation
"""
handlers = [
'module_spend_validator.good_spend1',
'module_spend_validator.good_spend2',
'module_spend_validator.good_spend3',
]
bad_handlers = [
'module_spend_validator.bad_spend1',
'module_spend_validator.bad_spend2',
'module_spend_validator.bad_spend3',
]
to_patch = 'flexer.runner.Flexer._get_validation_schema_file'
with mock.patch(to_patch, return_value='get_spend.json'):
for handler in handlers:
result = self.runner.run(event={},
context=None,
handler=handler)
load = json.loads(result)
self.assertFalse(load['error'])
for handler in bad_handlers:
result = self.runner.run(event={},
context=None,
handler=handler)
load = json.loads(result)
self.assertTrue(load['error'])
class TestFlexer(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.runner = Flexer()
def test_run_with_no_handler(self):
"""If run is called without a handler, an exception should be raised"""
handler = ''
expected = {
'exc_message': 'Handler is required',
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_module(self):
"""If run is called with a non-existing module, an exception
should be raised
"""
handler = 'not_here.test'
expected = {
'exc_message':
('Failed to import module "not_here": No module named not_here'),
'exc_type':
'ImportError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_handler(self):
"""If run is called with a valid module, but non-existing handler,
an exception should be raised
"""
handler = 'module_okay.not_found'
expected = {
'exc_message':
('Handler "not_found" not found in "module_okay": '
'\'module\' object has no attribute \'not_found\''),
'exc_type':
'AttributeError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_builtin_module(self):
"""If a built-in module is passed as a handler to run, an exception
should be raised
To get the list of all modules that are compiled into this Python
interpreter, do: print sys.builtin_module_names
sys happens to be one of these modules
"""
handler = 'sys.exit'
expected = {
'exc_message':
('Built-in module "sys" cannot be a handler module'),
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_invalid_handler(self):
"""If run is called with an invalid handler, i.e passing only a module
or a method, an exception should be raised
"""
handler = 'test'
expected = {
'exc_message': ('Invalid format for handler "test": '
'need more than 1 value to unpack'),
'exc_type':
'ValueError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_import_error(self):
"""Make sure ImportErrors are handled during module imports.
If there is an ImportError raised, the exception should store the
stack trace.
"""
handler = 'module_with_import_error.test'
expected = {
'exc_message':
('Failed to import module "module_with_import_error": '
'No module named this_module_does_not_exist'),
'exc_type':
'ImportError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn(
'ImportError: No module named this_module_does_not_exist\n',
error['stack_trace'])
def test_run_with_syntax_error(self):
"""Make sure SyntaxErrors are handled during module imports.
If there is a SyntaxError raised, the exception should store the
stack trace.
"""
handler = 'module_with_syntax_error.test'
expected = {
'exc_message':
('Syntax error in module "module_with_syntax_error": '
'invalid syntax (module_with_syntax_error.py, line 2)'),
'exc_type':
'SyntaxError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('this is a syntax error', error['stack_trace'])
def test_run_with_module_level_exception(self):
"""Make sure all exceptions are handled during module imports.
If there is an Exception raised, a stack trace should be available
in the exception as well
"""
handler = 'module_with_exception.test'
expected = {
'exc_message': ('Failed to initialise "module_with_exception": '
'need more than 1 value to unpack'),
'exc_type':
'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: need more than 1 value to unpack',
error['stack_trace'])
def test_run_with_no_stdout_logs(self):
"""Test the simplest possible use-case: run a method with no side
effects and make sure the result is stored
"""
handler = 'module_okay.test_with_no_logs'
expected = {
'value': 42,
'error': None,
'logs': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_stdout_logs(self):
"""Run a method with side effects and make sure that the result is
stored and the stdout/stderr are captured
"""
handler = 'module_okay.test_with_logs'
expected = {
'value': 42,
'error': None,
'logs': 'This goes to stdout\n',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_handler_exception(self):
"""Run a method that raises an Exception. Store the error details as
a result from the execution and make sure that the stdout/stderr are
still captured. The stack trace is expected to be part of the logs.
"""
handler = 'module_okay.test_with_exception'
expected = {
'exc_message': ('need more than 1 value to unpack'),
'exc_type': 'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Before the exception\nTraceback', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: need more than 1 value to unpack',
error['stack_trace'])
@pytest.mark.skip(reason="We don't have to worry about memory locally")
def test_run_with_memory_error(self):
"""Make sure MemoryErrors are handled when running the code.
"""
handler = 'module_with_memory_error.test'
expected = {
'exc_message': '',
'exc_type': 'MemoryError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Starting Up\n', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('MemoryError', error['stack_trace'])
def test_run_with_validation_error(self):
"""Run a method that returns data with validation errors and make sure
its being caught and stored in the logs
"""
handler = 'module_with_validation_error.test'
expected = {
'value': {
'foo': 10
},
'error': {
'exc_message': '["10 is not of type \'string\' in [\'foo\']"]',
'exc_type': 'ValidationError'
},
'logs': '["10 is not of type \'string\' in [\'foo\']"]',
}
schema = {
'patternProperties': {
'.*': {
'type': 'string'
},
}
}
with mock.patch('flexer.runner.Flexer._get_validation_schema',
return_value=schema):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_validate_metrics_ok(self):
"""Run a method that returns metric data and check validation is ok
"""
handler = 'module_with_metrics.test_ok'
expected = {
u'value': {
u'metrics': [{
u'metric':
u'cpu-usage',
u'value':
99,
u'unit':
u'percent',
u'time':
u'2017-01-12T18:30:42.034751Z',
u'resource_id':
u'1237ab91-08eb-4164-8e68-67699c29cd4c'
}]
},
u'error': None,
u'logs': u'Running test script\n',
}
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_metrics.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_validate_metrics_error(self):
"""Run a method that returns metric data and check validation is ok
"""
handler = 'module_with_metrics.test_invalid'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_metrics.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertTrue('metrics' in actual['value'])
self.assertEqual(1, len(actual['value']['metrics']))
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertTrue("\'time\' is a required property" in actual['error']
['exc_message'])
def test_validate_logs_ok(self):
"""Run a method that returns log data and check validation is ok
"""
handler = 'module_with_logs.test_ok'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_logs.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertTrue('logs' in actual['value'])
self.assertEqual(1, len(actual['value']['logs']))
self.assertEqual(None, actual['error'])
def test_validate_logs_error(self):
"""Run a method that returns log data and check validation is ok
"""
handler = 'module_with_logs.test_invalid'
with mock.patch('flexer.runner.Flexer._get_validation_schema_file',
return_value='get_logs.json'):
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('exc_type' in actual['error'])
self.assertTrue('exc_message' in actual['error'])
self.assertEqual(u'ValidationError', actual['error']['exc_type'])
self.assertTrue("\'time\' is a required property" in actual['error']
['exc_message'])
class TestFlexer(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.runner = Flexer()
def test_run_with_no_handler(self):
"""If run is called without a handler, an exception should be raised"""
handler = ''
expected = {
'exc_message': 'Handler is required',
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_module(self):
"""If run is called with a non-existing module, an exception
should be raised
"""
handler = 'not_here.test'
expected = {
'exc_message': (
'Failed to import module "not_here": %s'
% exc_message["nonexisting_module"][PYVERSION]
),
'exc_type': 'ImportError',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_handler(self):
"""If run is called with a valid module, but non-existing handler,
an exception should be raised
"""
handler = 'module_okay.not_found'
expected = {
'exc_message': (
'Handler "not_found" not found in "module_okay": %s'
% exc_message["nonexisting_handler"][PYVERSION]
),
'exc_type': 'AttributeError',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_builtin_module(self):
"""If a built-in module is passed as a handler to run, an exception
should be raised
To get the list of all modules that are compiled into this Python
interpreter, do: print sys.builtin_module_names
sys happens to be one of these modules
"""
handler = 'sys.exit'
expected = {
'exc_message': (
'Built-in module "sys" cannot be a handler module'
),
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_invalid_handler(self):
"""If run is called with an invalid handler, i.e passing only a module
or a method, an exception should be raised
"""
handler = 'test'
expected = {
'exc_message': (
'Invalid format for handler "test": %s'
% exc_message["invalid_handler"][PYVERSION]
),
'exc_type': 'ValueError',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_import_error(self):
"""Make sure ImportErrors are handled during module imports.
If there is an ImportError raised, the exception should store the
stack trace.
"""
handler = 'module_with_import_error.test'
exc_m = exc_message["import_error"][PYVERSION]
exc_t = exc_type["import_error"][PYVERSION]
expected = {
'exc_message': (
'Failed to import module "module_with_import_error": %s'
% exc_m
),
'exc_type': exc_t,
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('%s: %s' % (exc_t, exc_m), error['stack_trace'])
def test_run_with_syntax_error(self):
"""Make sure SyntaxErrors are handled during module imports.
If there is a SyntaxError raised, the exception should store the
stack trace.
"""
handler = 'module_with_syntax_error.test'
expected = {
'exc_message': (
'Syntax error in module "module_with_syntax_error": '
'invalid syntax (module_with_syntax_error.py, line 2)'
),
'exc_type': 'SyntaxError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('this is a syntax error', error['stack_trace'])
def test_run_with_module_level_exception(self):
"""Make sure all exceptions are handled during module imports.
If there is an Exception raised, a stack trace should be available
in the exception as well
"""
handler = 'module_with_exception.test'
exc_m = exc_message["module_level_exception"][PYVERSION]
expected = {
'exc_message': (
'Failed to initialise "module_with_exception": %s' % exc_m
),
'exc_type': 'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: %s' % exc_m, error['stack_trace'])
def test_run_with_no_stdout_logs(self):
"""Test the simplest possible use-case: run a method with no side
effects and make sure the result is stored
"""
handler = 'module_okay.test_with_no_logs'
expected = {
'value': 42,
'error': None,
'logs': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_stdout_logs(self):
"""Run a method with side effects and make sure that the result is
stored and the stdout/stderr are captured
"""
handler = 'module_okay.test_with_logs'
expected = {
'value': 42,
'error': None,
'logs': 'This goes to stdout\n',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_handler_exception(self):
"""Run a method that raises an Exception. Store the error details as
a result from the execution and make sure that the stdout/stderr are
still captured. The stack trace is expected to be part of the logs.
"""
handler = 'module_okay.test_with_exception'
exc_m = exc_message["handler_exception"][PYVERSION]
expected = {
'exc_message': exc_m,
'exc_type': 'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Before the exception\nTraceback', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: %s' % exc_m, error['stack_trace'])
@pytest.mark.skip(reason="We don't have to worry about memory locally")
def test_run_with_memory_error(self):
"""Make sure MemoryErrors are handled when running the code.
"""
handler = 'module_with_memory_error.test'
expected = {
'exc_message': '',
'exc_type': 'MemoryError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Starting Up\n', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('MemoryError', error['stack_trace'])
def test_run_with_invalid_db_secret(self):
"""Run a method that tries to use an invalid secret for an Nflex DB
"""
handler = 'module_with_db.test'
expected = {
'exc_message': 'The mydb secret is not a valid MongoDB connection string', # noqa
'exc_type': 'Exception',
}
context = FlexerContext()
secrets = [
"test",
"mongodb://",
"mongodb://a",
"mongodb://*****:*****@",
"mongodb://*****:*****@c",
"mongodb://*****:*****@c/",
]
for s in secrets:
context.secrets = {"_nflexdb_mydb": s}
result = self.runner.run(
event={},
context=context,
handler=handler,
)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('Exception', error['stack_trace'])
def test_run_with_valid_db_secret(self):
"""Run a method that tries to use a valid secret for an Nflex DB
"""
handler = 'module_with_db.test'
context = FlexerContext()
context.secrets = {"_nflexdb_mydb": "mongodb://*****:*****@c/mydb"}
with mock.patch(
'flexer.context.MongoClient',
return_value=mock.MagicMock()):
result = self.runner.run(
event={},
context=context,
handler=handler,
)
actual = json.loads(result)
self.assertTrue('error' in actual)
self.assertTrue('value' in actual)
self.assertEqual(None, actual['error'])
self.assertTrue('result' in actual['value'])
def run(handler, event, cmp_client):
event = json.loads(event)
handler = "main.%s" % handler
context = FlexerContext(cmp_client=cmp_client)
runner = Flexer()
return runner.run(event=event, context=context, handler=handler)
class TestFlexer(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.runner = Flexer()
def test_run_with_no_handler(self):
"""If run is called without a handler, an exception should be raised"""
handler = ''
expected = {
'exc_message': 'Handler is required',
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_module(self):
"""If run is called with a non-existing module, an exception
should be raised
"""
handler = 'not_here.test'
expected = {
'exc_message':
('Failed to import module "not_here": No module named not_here'),
'exc_type':
'ImportError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_nonexisting_handler(self):
"""If run is called with a valid module, but non-existing handler,
an exception should be raised
"""
handler = 'module_okay.not_found'
expected = {
'exc_message':
('Handler "not_found" not found in "module_okay": '
'\'module\' object has no attribute \'not_found\''),
'exc_type':
'AttributeError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_builtin_module(self):
"""If a built-in module is passed as a handler to run, an exception
should be raised
To get the list of all modules that are compiled into this Python
interpreter, do: print sys.builtin_module_names
sys happens to be one of these modules
"""
handler = 'sys.exit'
expected = {
'exc_message':
('Built-in module "sys" cannot be a handler module'),
'exc_type': 'Exception',
'stack_trace': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_invalid_handler(self):
"""If run is called with an invalid handler, i.e passing only a module
or a method, an exception should be raised
"""
handler = 'test'
expected = {
'exc_message': ('Invalid format for handler "test": '
'need more than 1 value to unpack'),
'exc_type':
'ValueError',
'stack_trace':
'',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertDictEqual(expected, actual['error'])
def test_run_with_import_error(self):
"""Make sure ImportErrors are handled during module imports.
If there is an ImportError raised, the exception should store the
stack trace.
"""
handler = 'module_with_import_error.test'
expected = {
'exc_message':
('Failed to import module "module_with_import_error": '
'No module named this_module_does_not_exist'),
'exc_type':
'ImportError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn(
'ImportError: No module named this_module_does_not_exist\n',
error['stack_trace'])
def test_run_with_syntax_error(self):
"""Make sure SyntaxErrors are handled during module imports.
If there is a SyntaxError raised, the exception should store the
stack trace.
"""
handler = 'module_with_syntax_error.test'
expected = {
'exc_message':
('Syntax error in module "module_with_syntax_error": '
'invalid syntax (module_with_syntax_error.py, line 2)'),
'exc_type':
'SyntaxError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('this is a syntax error', error['stack_trace'])
def test_run_with_module_level_exception(self):
"""Make sure all exceptions are handled during module imports.
If there is an Exception raised, a stack trace should be available
in the exception as well
"""
handler = 'module_with_exception.test'
expected = {
'exc_message': ('Failed to initialise "module_with_exception": '
'need more than 1 value to unpack'),
'exc_type':
'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: need more than 1 value to unpack',
error['stack_trace'])
def test_run_with_no_stdout_logs(self):
"""Test the simplest possible use-case: run a method with no side
effects and make sure the result is stored
"""
handler = 'module_okay.test_with_no_logs'
expected = {
'value': 42,
'error': None,
'logs': '',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_stdout_logs(self):
"""Run a method with side effects and make sure that the result is
stored and the stdout/stderr are captured
"""
handler = 'module_okay.test_with_logs'
expected = {
'value': 42,
'error': None,
'logs': 'This goes to stdout\n',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertDictEqual(expected, actual)
def test_run_with_handler_exception(self):
"""Run a method that raises an Exception. Store the error details as
a result from the execution and make sure that the stdout/stderr are
still captured. The stack trace is expected to be part of the logs.
"""
handler = 'module_okay.test_with_exception'
expected = {
'exc_message': ('need more than 1 value to unpack'),
'exc_type': 'ValueError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Before the exception\nTraceback', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('ValueError: need more than 1 value to unpack',
error['stack_trace'])
@pytest.mark.skip(reason="We don't have to worry about memory locally")
def test_run_with_memory_error(self):
"""Make sure MemoryErrors are handled when running the code.
"""
handler = 'module_with_memory_error.test'
expected = {
'exc_message': '',
'exc_type': 'MemoryError',
}
result = self.runner.run(event={}, context=None, handler=handler)
actual = json.loads(result)
self.assertEqual(None, actual['value'])
self.assertIn('Starting Up\n', actual['logs'])
error = actual['error']
self.assertEqual(expected['exc_message'], error['exc_message'])
self.assertEqual(expected['exc_type'], error['exc_type'])
self.assertIn('MemoryError', error['stack_trace'])
