def _asserts_for_dport_80_1024(self,
parser_base_cls,
raw,
extra_parser_attrs=None,
extra_input_items={},
extra_output_items1={},
extra_output_items2={}):
class MyParser(self.MyParserMixIn, parser_base_cls):
constant_items = dict(self.MyParserMixIn.constant_items,
**extra_input_items)
if extra_parser_attrs is not None:
for k, v in extra_parser_attrs.items():
setattr(MyParser, k, v)
parser = MyParser.__new__(MyParser)
data = dict(self.base_data, raw=raw)
seq_mock = FilePagedSequence._instance_mock()
output_bodies = parser.get_output_bodies(data, seq_mock)
self.assertIs(output_bodies, seq_mock)
output_data = [json.loads(body) for body in seq_mock._list]
for d in output_data:
# check that d['id'] looks like an md5 hash...
self.assertIsInstance(d.get('id'), basestring)
self.assertEqual(len(d['id']), 32)
self.assertTrue(set('0123456789abcdef').issuperset(d['id']))
# ...then omit d['id'] for simplicity of the test
del d['id']
self.assertEqual(output_data, [
dict(self.common_output_items, dport=80, **extra_output_items1),
dict(self.common_output_items, dport=1024, **extra_output_items2),
])
def input_callback(self, routing_key, body, properties):
"""
A callback, typically called in QueuedBase.on_message().
Args:
`routing_key`:
The routing key used to publish the AMQP message.
`body`:
The AMQP message body.
`properties`:
A pika.BasicProperties instance containing properties of
the AMQP message.
This method calls the following parser-specific methods:
* prepare_data(),
* get_output_rk(),
* get_output_bodies(),
* and for each item of the sequence returned by get_output_bodies():
* publish_output() (this one is defined in a superclass --
typically it is QueuedBase.publish_output()).
Default implementations of these methods should be sensible in
most cases.
"""
body = self._fix_body(body)
data = self.prepare_data(routing_key, body, properties)
rid = data.get('properties.message_id')
with self.setting_error_event_info(rid):
output_rk = self.get_output_rk(data)
with FilePagedSequence(page_size=1000) as working_seq:
for output_body in self.get_output_bodies(data, working_seq):
self.publish_output(routing_key=output_rk,
body=output_body)
def test__get_output_bodies__parse_yielded_no_items(self):
self.mock.configure_mock(**{'parse.return_value': iter([])})
with self.assertRaises(ValueError):
self.meth.get_output_bodies(sentinel.data,
FilePagedSequence._instance_mock())
self.assertEqual(self.mock.method_calls, [
call.parse(sentinel.data),
])
def test__get_output_bodies(self):
parsed = [MagicMock(**{'__class__': RecordDict,
'used_as_context_manager': True,
'get_ready_json.return_value':
getattr(sentinel,
'output_body{}'.format(i))})
for i in (1, 2)]
self.mock.configure_mock(**{
'parse.return_value': parsed,
'get_output_message_id.side_effect': [
sentinel.msg_A,
sentinel.msg_B,
],
'setting_error_event_info': MagicMock(),
'postprocess_parsed.side_effect': (
lambda data, parsed, total, item_no: parsed
),
})
seq_mock = FilePagedSequence._instance_mock()
output_bodies = self.meth.get_output_bodies(sentinel.data, seq_mock)
self.assertIs(output_bodies, seq_mock)
self.assertEqual(seq_mock._list, [
sentinel.output_body1,
sentinel.output_body2,
])
self.assertEqual(parsed[0].mock_calls, [
call.__setitem__('id', sentinel.msg_A),
call.get_ready_json(),
])
self.assertEqual(parsed[1].mock_calls, [
call.__setitem__('id', sentinel.msg_B),
call.get_ready_json(),
])
self.assertEqual(self.mock.mock_calls, [
call.parse(sentinel.data),
call.get_output_message_id(parsed[0]),
call.delete_too_long_address(parsed[0]),
call.get_output_message_id(parsed[1]),
call.delete_too_long_address(parsed[1]),
call.setting_error_event_info(parsed[0]),
call.setting_error_event_info().__enter__(),
call.postprocess_parsed(sentinel.data,
parsed[0],
2,
item_no=1),
call.setting_error_event_info().__exit__(None, None, None),
call.setting_error_event_info(parsed[1]),
call.setting_error_event_info().__enter__(),
call.postprocess_parsed(sentinel.data,
parsed[1],
2,
item_no=2),
call.setting_error_event_info().__exit__(None, None, None),
])
def test__get_output_bodies__parse_yielded_no_items__allow_empty_results(self):
self.mock.configure_mock(**{'parse.return_value': iter([]),
'allow_empty_results': True})
seq_mock = FilePagedSequence._instance_mock()
output_bodies = self.meth.get_output_bodies(sentinel.data, seq_mock)
self.assertIs(output_bodies, seq_mock)
self.assertEqual(seq_mock._list, []) # just empty
self.assertEqual(self.mock.mock_calls, [
call.parse(sentinel.data),
])
def test__get_output_bodies__record_dict_not_used_as_context_manager(self):
parsed = [MagicMock(**{'__class__': RecordDict,
'used_as_context_manager': False})
for i in (1, 2)]
self.mock.configure_mock(**{'parse.return_value': parsed})
with self.assertRaises(AssertionError):
self.meth.get_output_bodies(sentinel.data,
FilePagedSequence._instance_mock())
self.assertEqual(self.mock.method_calls, [
call.parse(sentinel.data),
])
def test__parse__yielded_no_items(self, *args):
for base_parser_cls in (BaseParser,
AggregatedEventParser,
BlackListParser):
class MyParser(self.MyParserMixIn, base_parser_cls):
def parse(self, data):
return # "empty" generator
yield
parser = MyParser.__new__(MyParser)
data = dict(self.base_data, raw='80 1024')
with self.assertRaises(ValueError):
parser.get_output_bodies(data, FilePagedSequence._instance_mock())
def _asserts_for_loud_error(self, parser_base_cls, error_cls, raw,
extra_parser_attrs=None,
required_error_attrs=None):
class MyParser(self.MyParserMixIn, parser_base_cls):
pass
if extra_parser_attrs is not None:
for k, v in extra_parser_attrs.items():
setattr(MyParser, k, v)
parser = MyParser.__new__(MyParser)
data = dict(self.base_data, raw=raw)
with self.assertRaises(error_cls) as cm:
parser.get_output_bodies(data, FilePagedSequence._instance_mock())
if required_error_attrs is not None:
self.assertTrue(vars(cm.exception).viewitems() >=
required_error_attrs.viewitems())
def test_record_dict_context_manager_not_used(self, *args):
for base_parser_cls in (BaseParser,
AggregatedEventParser,
BlackListParser):
class MyParser(self.MyParserMixIn, base_parser_cls):
def parse(self, data):
for dport in data['raw'].split(' '):
# lacking 'with...' statement:
parsed = self.new_record_dict(data)
parsed['dport'] = dport
parsed['time'] = '2014-01-10 11:14:00.248114'
yield parsed
parser = MyParser.__new__(MyParser)
data = dict(self.base_data, raw='80 1024')
with self.assertRaises(AssertionError):
parser.get_output_bodies(data, FilePagedSequence._instance_mock())
