class FlowLogsReaderTestCase(TestCase):
@patch('flowlogs_reader.flowlogs_reader.boto3', autospec=True)
def setUp(self, mock_boto3):
self.mock_session = MagicMock()
mock_boto3.session.Session.return_value = self.mock_session
self.mock_client = MagicMock()
self.mock_session.client.return_value = self.mock_client
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(
datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time
)
self.assertEqual(
datetime.utcfromtimestamp(self.inst.end_ms // 1000),
self.end_time
)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_region_name(self, mock_session):
# Region specified
FlowLogsReader('some_group', region_name='some-region')
mock_session.Session.assert_called_with(region_name='some-region')
# No region specified - assume configuration file worked
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
# Region specified in boto_client_kwargs
FlowLogsReader(
'some_group', boto_client_kwargs={'region_name': 'my-region'}
)
mock_session.Session.assert_called_with(region_name='my-region')
# No region specified and no configuration file
def mock_response(*args, **kwargs):
if 'region_name' not in kwargs:
raise NoRegionError
mock_session.Session().client.side_effect = mock_response
FlowLogsReader('some_group')
mock_session.Session().client.assert_called_with(
'logs', region_name=DEFAULT_REGION_NAME
)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_profile_name(self, mock_session):
# profile_name specified
FlowLogsReader('some_group', profile_name='my-profile')
mock_session.Session.assert_called_with(profile_name='my-profile')
# No profile specified
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
def test_read_streams(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{'events': [0]}, {'events': [1, 2]}, {'events': [3, 4, 5]},
]
self.mock_client.get_paginator.return_value = paginator
actual = list(self.inst._read_streams())
expected = [0, 1, 2, 3, 4, 5]
self.assertEqual(actual, expected)
def test_iteration(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[0]},
{'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[1]},
],
},
{
'events': [
{'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[2]},
{'logStreamName': 'log_1', 'message': SAMPLE_RECORDS[3]},
{'logStreamName': 'log_2', 'message': SAMPLE_RECORDS[4]},
],
},
]
self.mock_client.get_paginator.return_value = paginator
# Calling list on the instance causes it to iterate through all records
actual = list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS]
self.assertEqual(actual, expected)
class FlowLogsReaderTestCase(TestCase):
@patch('flowlogs_reader.flowlogs_reader.boto3', autospec=True)
def setUp(self, mock_boto3):
self.mock_session = MagicMock()
mock_boto3.session.Session.return_value = self.mock_session
self.mock_client = MagicMock()
self.mock_session.client.return_value = self.mock_client
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time)
self.assertEqual(datetime.utcfromtimestamp(self.inst.end_ms // 1000),
self.end_time)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_region_name(self, mock_session):
# Region specified
FlowLogsReader('some_group', region_name='some-region')
mock_session.Session.assert_called_with(region_name='some-region')
# No region specified - assume configuration file worked
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
# Region specified in boto_client_kwargs
FlowLogsReader('some_group',
boto_client_kwargs={'region_name': 'my-region'})
mock_session.Session.assert_called_with(region_name='my-region')
# No region specified and no configuration file
def mock_response(*args, **kwargs):
if 'region_name' not in kwargs:
raise NoRegionError
mock_session.Session().client.side_effect = mock_response
FlowLogsReader('some_group')
mock_session.Session().client.assert_called_with(
'logs', region_name=DEFAULT_REGION_NAME)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_profile_name(self, mock_session):
# profile_name specified
FlowLogsReader('some_group', profile_name='my-profile')
mock_session.Session.assert_called_with(profile_name='my-profile')
# No profile specified
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
def test_read_streams(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [0]
},
{
'events': [1, 2]
},
{
'events': [3, 4, 5]
},
]
self.mock_client.get_paginator.return_value = paginator
actual = list(self.inst._read_streams())
expected = [0, 1, 2, 3, 4, 5]
self.assertEqual(actual, expected)
def test_iteration(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[0]
},
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[1]
},
],
},
{
'events': [
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[2]
},
{
'logStreamName': 'log_1',
'message': SAMPLE_RECORDS[3]
},
{
'logStreamName': 'log_2',
'message': SAMPLE_RECORDS[4]
},
],
},
]
self.mock_client.get_paginator.return_value = paginator
# Calling list on the instance causes it to iterate through all records
actual = list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS]
self.assertEqual(actual, expected)
class FlowLogsReaderTestCase(TestCase):
def setUp(self):
self.mock_client = MagicMock()
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
filter_pattern='REJECT',
boto_client=self.mock_client,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(
datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time
)
self.assertEqual(
datetime.utcfromtimestamp(self.inst.end_ms // 1000),
self.end_time
)
self.assertEqual(
self.inst.paginator_kwargs['filterPattern'],
'REJECT'
)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_region_name(self, mock_session):
# Region specified for session
FlowLogsReader('some_group', region_name='some-region')
mock_session.Session.assert_called_with(region_name='some-region')
# Region specified for client, not for session
FlowLogsReader(
'some_group', boto_client_kwargs={'region_name': 'my-region'}
)
mock_session.Session().client.assert_called_with(
'logs', region_name='my-region'
)
# No region specified for session or client - use the default
def mock_response(*args, **kwargs):
if 'region_name' not in kwargs:
raise NoRegionError
mock_session.Session().client.side_effect = mock_response
FlowLogsReader('some_group')
mock_session.Session().client.assert_called_with(
'logs', region_name=DEFAULT_REGION_NAME
)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_profile_name(self, mock_session):
# profile_name specified
FlowLogsReader('some_group', profile_name='my-profile')
mock_session.Session.assert_called_with(profile_name='my-profile')
# No profile specified
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
def test_read_streams(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{'events': [0]}, {'events': [1, 2]}, {'events': [3, 4, 5]},
]
self.mock_client.get_paginator.return_value = paginator
actual = list(self.inst._read_streams())
expected = [0, 1, 2, 3, 4, 5]
self.assertEqual(actual, expected)
def test_iteration(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[0]},
{'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[1]},
],
},
{
'events': [
{'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[2]},
{'logStreamName': 'log_1', 'message': SAMPLE_RECORDS[3]},
{'logStreamName': 'log_2', 'message': SAMPLE_RECORDS[4]},
],
},
]
self.mock_client.get_paginator.return_value = paginator
# Calling list on the instance causes it to iterate through all records
actual = [next(self.inst)] + list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS]
self.assertEqual(actual, expected)
def test_iteration_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[0]}
event_1 = {'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[1]}
for item in [{'events': [event_0, event_1]}]:
yield item
err_msg = '{}: {}'.format(DUPLICATE_NEXT_TOKEN_MESSAGE, 'token')
raise PaginationError(message=err_msg)
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator
)
# Don't fail if botocore's paginator raises a PaginationError
actual = [next(self.inst)] + list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS[:2]]
self.assertEqual(actual, expected)
def test_iteration_unexpecetd_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[0]}
yield {'events': [event_0]}
raise PaginationError(message='other error')
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator
)
# Fail for unexpected PaginationError
self.assertRaises(PaginationError, lambda: list(self.inst))
class FlowLogsReaderTestCase(TestCase):
@patch('flowlogs_reader.flowlogs_reader.boto3', autospec=True)
def setUp(self, mock_boto3):
self.mock_client = MagicMock()
mock_boto3.client.return_value = self.mock_client
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time)
self.assertEqual(datetime.utcfromtimestamp(self.inst.end_ms // 1000),
self.end_time)
@patch('flowlogs_reader.flowlogs_reader.boto3.client', autospec=True)
def test_region(self, mock_client):
# Region specified
FlowLogsReader('some_group', region_name='some-region')
mock_client.assert_called_with('logs', region_name='some-region')
# No region specified - assume configuration file worked
FlowLogsReader('some_group')
mock_client.assert_called_with('logs')
# No region specified and no configuration file
def mock_response(*args, **kwargs):
if 'region_name' not in kwargs:
raise NoRegionError
mock_client.side_effect = mock_response
FlowLogsReader('some_group')
mock_client.assert_called_with('logs', region_name=DEFAULT_REGION_NAME)
def test_read_streams(self):
response_list = [
{
'events': [0],
'nextToken': 'token_0'
},
{
'events': [1, 2],
'nextToken': 'token_1'
},
{
'events': [3, 4, 5],
'nextToken': None
},
{
'events': [6],
'nextForwardToken': 'token_2'
}, # Unreachable
]
def mock_filter(*args, **kwargs):
return response_list.pop(0)
self.mock_client.filter_log_events.side_effect = mock_filter
actual = list(self.inst._read_streams())
expected = [0, 1, 2, 3, 4, 5]
self.assertEqual(actual, expected)
def test_iteration(self):
response_list = [
{
'events': [
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[0]
},
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[1]
},
],
'nextToken':
'token_0',
},
{
'events': [
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[2]
},
{
'logStreamName': 'log_1',
'message': SAMPLE_RECORDS[3]
},
{
'logStreamName': 'log_2',
'message': SAMPLE_RECORDS[4]
},
],
},
]
def mock_filter(*args, **kwargs):
return response_list.pop(0)
self.mock_client.filter_log_events.side_effect = mock_filter
# Calling list on the instance causes it to iterate through all records
actual = list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS]
self.assertEqual(actual, expected)
class FlowLogsReaderTestCase(TestCase):
def setUp(self):
self.mock_client = MagicMock()
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
filter_pattern='REJECT',
boto_client=self.mock_client,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time)
self.assertEqual(datetime.utcfromtimestamp(self.inst.end_ms // 1000),
self.end_time)
self.assertEqual(self.inst.paginator_kwargs['filterPattern'], 'REJECT')
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_region_name(self, mock_session):
# Region specified for session
FlowLogsReader('some_group', region_name='some-region')
mock_session.Session.assert_called_with(region_name='some-region')
# Region specified for client, not for session
FlowLogsReader('some_group',
boto_client_kwargs={'region_name': 'my-region'})
mock_session.Session().client.assert_called_with(
'logs', region_name='my-region')
# No region specified for session or client - use the default
def mock_response(*args, **kwargs):
if 'region_name' not in kwargs:
raise NoRegionError
mock_session.Session().client.side_effect = mock_response
FlowLogsReader('some_group')
mock_session.Session().client.assert_called_with(
'logs', region_name=DEFAULT_REGION_NAME)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_profile_name(self, mock_session):
# profile_name specified
FlowLogsReader('some_group', profile_name='my-profile')
mock_session.Session.assert_called_with(profile_name='my-profile')
# No profile specified
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
def test_read_streams(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [0]
},
{
'events': [1, 2]
},
{
'events': [3, 4, 5]
},
]
self.mock_client.get_paginator.return_value = paginator
actual = list(self.inst._read_streams())
expected = [0, 1, 2, 3, 4, 5]
self.assertEqual(actual, expected)
def test_iteration(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[0]
},
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[1]
},
],
},
{
'events': [
{
'logStreamName': 'log_0',
'message': SAMPLE_RECORDS[2]
},
{
'logStreamName': 'log_1',
'message': SAMPLE_RECORDS[3]
},
{
'logStreamName': 'log_2',
'message': SAMPLE_RECORDS[4]
},
],
},
]
self.mock_client.get_paginator.return_value = paginator
# Calling list on the instance causes it to iterate through all records
actual = [next(self.inst)] + list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS]
self.assertEqual(actual, expected)
def test_iteration_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[0]}
event_1 = {'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[1]}
for item in [{'events': [event_0, event_1]}]:
yield item
err_msg = '{}: {}'.format(DUPLICATE_NEXT_TOKEN_MESSAGE, 'token')
raise PaginationError(message=err_msg)
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator)
# Don't fail if botocore's paginator raises a PaginationError
actual = [next(self.inst)] + list(self.inst)
expected = [FlowRecord.from_message(x) for x in SAMPLE_RECORDS[:2]]
self.assertEqual(actual, expected)
def test_iteration_unexpecetd_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': SAMPLE_RECORDS[0]}
yield {'events': [event_0]}
raise PaginationError(message='other error')
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator)
# Fail for unexpected PaginationError
self.assertRaises(PaginationError, lambda: list(self.inst))
class FlowLogsReaderTestCase(TestCase):
def setUp(self):
self.mock_client = MagicMock()
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
filter_pattern='REJECT',
boto_client=self.mock_client,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(
datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time,
)
self.assertEqual(
datetime.utcfromtimestamp(self.inst.end_ms // 1000), self.end_time
)
self.assertEqual(self.inst.paginator_kwargs['filterPattern'], 'REJECT')
@patch('flowlogs_reader.flowlogs_reader.boto3.client', autospec=True)
def test_region_name(self, mock_client):
# Region specified for session
FlowLogsReader('some_group', region_name='some-region')
mock_client.assert_called_with('logs', region_name='some-region')
# None specified
FlowLogsReader('some_group')
mock_client.assert_called_with('logs')
@patch('flowlogs_reader.flowlogs_reader.boto3.client', autospec=True)
def test_get_fields(self, mock_client):
cwl_client = MagicMock()
ec2_client = mock_client.return_value
ec2_client.describe_flow_logs.return_value = {
'FlowLogs': [
{'LogFormat': '${srcaddr} ${dstaddr} ${start} ${log-status}'}
]
}
reader = FlowLogsReader(
'some_group',
boto_client=cwl_client,
fields=None,
)
self.assertEqual(
reader.fields, ('srcaddr', 'dstaddr', 'start', 'log_status')
)
ec2_client.describe_flow_logs.assert_called_once_with(
Filters=[{'Name': 'log-group-name', 'Values': ['some_group']}]
)
def test_read_streams(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{'logStreamName': 'log_0', 'message': V2_RECORDS[0]},
{'logStreamName': 'log_0', 'message': V2_RECORDS[1]},
],
},
{
'events': [
{'logStreamName': 'log_0', 'message': V2_RECORDS[2]},
{'logStreamName': 'log_1', 'message': V2_RECORDS[3]},
{'logStreamName': 'log_2', 'message': V2_RECORDS[4]},
],
},
]
self.mock_client.get_paginator.return_value = paginator
actual = list(self.inst._read_streams())
expected = []
for page in paginator.paginate.return_value:
expected += page['events']
self.assertEqual(actual, expected)
def test_iteration(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{'logStreamName': 'log_0', 'message': V2_RECORDS[0]},
{'logStreamName': 'log_0', 'message': V2_RECORDS[1]},
],
},
{
'events': [
{'logStreamName': 'log_0', 'message': V2_RECORDS[2]},
{'logStreamName': 'log_1', 'message': V2_RECORDS[3]},
{'logStreamName': 'log_2', 'message': V2_RECORDS[4]},
],
},
]
self.mock_client.get_paginator.return_value = paginator
# Calling list on the instance causes it to iterate through all records
actual = [next(self.inst)] + list(self.inst)
expected = [
FlowRecord.from_cwl_event({'message': x}) for x in V2_RECORDS
]
self.assertEqual(actual, expected)
expected_bytes = 0
all_pages = paginator.paginate.return_value
expected_bytes = sum(
len(e['message']) for p in all_pages for e in p['events']
)
self.assertEqual(self.inst.bytes_processed, expected_bytes)
def test_iteration_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': V2_RECORDS[0]}
event_1 = {'logStreamName': 'log_0', 'message': V2_RECORDS[1]}
for item in [{'events': [event_0, event_1]}]:
yield item
err_msg = '{}: {}'.format(DUPLICATE_NEXT_TOKEN_MESSAGE, 'token')
raise PaginationError(message=err_msg)
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator
)
# Don't fail if botocore's paginator raises a PaginationError
actual = [next(self.inst)] + list(self.inst)
records = V2_RECORDS[:2]
expected = [FlowRecord.from_cwl_event({'message': x}) for x in records]
self.assertEqual(actual, expected)
def test_iteration_unexpecetd_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': V2_RECORDS[0]}
yield {'events': [event_0]}
raise PaginationError(message='other error')
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator
)
# Fail for unexpected PaginationError
self.assertRaises(PaginationError, lambda: list(self.inst))
def test_threads(self):
inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
filter_pattern='REJECT',
boto_client=self.mock_client,
thread_count=1,
)
paginators = []
def _get_paginator(operation):
nonlocal paginators
paginator = MagicMock()
if operation == 'describe_log_streams':
paginator.paginate.return_value = [
{
'logStreams': [
{
'logStreamName': 'too_late',
'firstEventTimestamp': inst.end_ms,
'lastEventTimestamp': inst.start_ms,
},
{
'logStreamName': 'too_late',
'firstEventTimestamp': inst.end_ms - 1,
'lastEventTimestamp': (
inst.start_ms - LAST_EVENT_DELAY_MSEC - 1
),
},
],
},
{
'logStreams': [
{
'logStreamName': 'first_stream',
'firstEventTimestamp': inst.start_ms,
'lastEventTimestamp': inst.end_ms,
},
{
'logStreamName': 'second_stream',
'firstEventTimestamp': inst.start_ms,
'lastEventTimestamp': inst.end_ms,
},
],
},
]
elif operation == 'filter_log_events':
paginator.paginate.return_value = [
{
'events': [
{'message': V2_RECORDS[0]},
{'message': V2_RECORDS[1]},
],
},
{
'events': [
{'message': V2_RECORDS[2]},
{'message': V2_RECORDS[3]},
],
},
]
else:
self.fail('invalid operation')
paginators.append(paginator)
return paginator
self.mock_client.get_paginator.side_effect = _get_paginator
events = list(inst)
self.assertEqual(len(events), 8)
paginators[0].paginate.assert_called_once_with(
logGroupName='group_name',
orderBy='LastEventTime',
descending=True,
)
paginators[1].paginate.assert_called_once_with(
logGroupName='group_name',
startTime=inst.start_ms,
endTime=inst.end_ms,
interleaved=True,
filterPattern='REJECT',
logStreamNames=['first_stream'],
)
paginators[2].paginate.assert_called_once_with(
logGroupName='group_name',
startTime=inst.start_ms,
endTime=inst.end_ms,
interleaved=True,
filterPattern='REJECT',
logStreamNames=['second_stream'],
)
class FlowLogsReaderTestCase(TestCase):
def setUp(self):
self.mock_client = MagicMock()
self.start_time = datetime(2015, 8, 12, 12, 0, 0)
self.end_time = datetime(2015, 8, 12, 13, 0, 0)
self.inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
filter_pattern='REJECT',
boto_client=self.mock_client,
)
def test_init(self):
self.assertEqual(self.inst.log_group_name, 'group_name')
self.assertEqual(datetime.utcfromtimestamp(self.inst.start_ms // 1000),
self.start_time)
self.assertEqual(datetime.utcfromtimestamp(self.inst.end_ms // 1000),
self.end_time)
self.assertEqual(self.inst.paginator_kwargs['filterPattern'], 'REJECT')
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_region_name(self, mock_session):
# Region specified for session
FlowLogsReader('some_group', region_name='some-region')
mock_session.Session.assert_called_with(region_name='some-region')
# Region specified for client, not for session
FlowLogsReader('some_group',
boto_client_kwargs={'region_name': 'my-region'})
mock_session.Session().client.assert_called_with(
'logs', region_name='my-region')
# No region specified for session or client - use the default
def mock_response(*args, **kwargs):
if 'region_name' not in kwargs:
raise NoRegionError
mock_session.Session().client.side_effect = mock_response
FlowLogsReader('some_group')
mock_session.Session().client.assert_called_with(
'logs', region_name=DEFAULT_REGION_NAME)
@patch('flowlogs_reader.flowlogs_reader.boto3.session', autospec=True)
def test_profile_name(self, mock_session):
# profile_name specified
FlowLogsReader('some_group', profile_name='my-profile')
mock_session.Session.assert_called_with(profile_name='my-profile')
# No profile specified
FlowLogsReader('some_group')
mock_session.Session.assert_called_with()
def test_read_streams(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [0]
},
{
'events': [1, 2]
},
{
'events': [3, 4, 5]
},
]
self.mock_client.get_paginator.return_value = paginator
actual = list(self.inst._read_streams())
expected = [0, 1, 2, 3, 4, 5]
self.assertEqual(actual, expected)
def test_iteration(self):
paginator = MagicMock()
paginator.paginate.return_value = [
{
'events': [
{
'logStreamName': 'log_0',
'message': V2_RECORDS[0]
},
{
'logStreamName': 'log_0',
'message': V2_RECORDS[1]
},
],
},
{
'events': [
{
'logStreamName': 'log_0',
'message': V2_RECORDS[2]
},
{
'logStreamName': 'log_1',
'message': V2_RECORDS[3]
},
{
'logStreamName': 'log_2',
'message': V2_RECORDS[4]
},
],
},
]
self.mock_client.get_paginator.return_value = paginator
# Calling list on the instance causes it to iterate through all records
actual = [next(self.inst)] + list(self.inst)
expected = [
FlowRecord.from_cwl_event({'message': x}) for x in V2_RECORDS
]
self.assertEqual(actual, expected)
def test_iteration_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': V2_RECORDS[0]}
event_1 = {'logStreamName': 'log_0', 'message': V2_RECORDS[1]}
for item in [{'events': [event_0, event_1]}]:
yield item
err_msg = '{}: {}'.format(DUPLICATE_NEXT_TOKEN_MESSAGE, 'token')
raise PaginationError(message=err_msg)
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator)
# Don't fail if botocore's paginator raises a PaginationError
actual = [next(self.inst)] + list(self.inst)
records = V2_RECORDS[:2]
expected = [FlowRecord.from_cwl_event({'message': x}) for x in records]
self.assertEqual(actual, expected)
def test_iteration_unexpecetd_error(self):
# Simulate the paginator failing
def _get_paginator(*args, **kwargs):
event_0 = {'logStreamName': 'log_0', 'message': V2_RECORDS[0]}
yield {'events': [event_0]}
raise PaginationError(message='other error')
self.mock_client.get_paginator.return_value.paginate.side_effect = (
_get_paginator)
# Fail for unexpected PaginationError
self.assertRaises(PaginationError, lambda: list(self.inst))
def test_threads(self):
inst = FlowLogsReader(
'group_name',
start_time=self.start_time,
end_time=self.end_time,
filter_pattern='REJECT',
boto_client=self.mock_client,
thread_count=1,
)
paginators = []
def _get_paginator(operation):
nonlocal paginators
paginator = MagicMock()
if operation == 'describe_log_streams':
paginator.paginate.return_value = [
{
'logStreams': [
{
'logStreamName': 'too_late',
'firstEventTimestamp': inst.end_ms,
'lastEventTimestamp': inst.start_ms,
},
{
'logStreamName':
'too_late',
'firstEventTimestamp':
inst.end_ms - 1,
'lastEventTimestamp':
(inst.start_ms - LAST_EVENT_DELAY_MSEC - 1),
},
],
},
{
'logStreams': [
{
'logStreamName': 'first_stream',
'firstEventTimestamp': inst.start_ms,
'lastEventTimestamp': inst.end_ms,
},
{
'logStreamName': 'second_stream',
'firstEventTimestamp': inst.start_ms,
'lastEventTimestamp': inst.end_ms,
},
],
},
]
elif operation == 'filter_log_events':
paginator.paginate.return_value = [
{
'events': [
{
'message': V2_RECORDS[0]
},
{
'message': V2_RECORDS[1]
},
],
},
{
'events': [
{
'message': V2_RECORDS[2]
},
{
'message': V2_RECORDS[3]
},
],
},
]
else:
self.fail('invalid operation')
paginators.append(paginator)
return paginator
self.mock_client.get_paginator.side_effect = _get_paginator
events = list(inst)
self.assertEqual(len(events), 8)
paginators[0].paginate.assert_called_once_with(
logGroupName='group_name',
orderBy='LastEventTime',
descending=True,
)
paginators[1].paginate.assert_called_once_with(
logGroupName='group_name',
startTime=inst.start_ms,
endTime=inst.end_ms,
interleaved=True,
filterPattern='REJECT',
logStreamNames=['first_stream'],
)
paginators[2].paginate.assert_called_once_with(
logGroupName='group_name',
startTime=inst.start_ms,
endTime=inst.end_ms,
interleaved=True,
filterPattern='REJECT',
logStreamNames=['second_stream'],
)
