def test_time_related_components_of_generated_queries(
self, given_time_constraints_items, expected_query_expr_reprs):
mock = MagicMock()
meth = MethodProxy(
_EventsQueryProcessor,
mock,
class_attrs=[
# for them, the actual members/methods (not mocks) will be used
# * class constants:
'queried_model_class',
'client_asoc_model_class',
# * methods:
'_prepare_result_production_tools',
'_fetch_rows_from_db',
'_time_comparisons_per_step',
'_fetch_rows_for_single_step',
'_build_query_base_for_single_step',
'_build_actual_query',
])
actual_query_expr_reprs = []
and_mock = self._make_mock_of_sqlalchemy_and(actual_query_expr_reprs)
mock._query_base = self._make_mock_of_query_base(
actual_query_expr_reprs)
mock._day_step = 1
mock._opt_limit = None
mock._time_constraints = (
given_time_constraints_items['time.min'],
given_time_constraints_items.get('time.max'),
given_time_constraints_items.get('time.until'))
with patch('n6lib.data_backend_api.and_', and_mock), \
patch('n6lib.data_backend_api.utcnow', return_value=self._UTCNOW):
list(meth.generate_query_results())
self.assertEqual(actual_query_expr_reprs, expected_query_expr_reprs)
def test_time_related_components_of_generated_queries(
self, given_params, expected_query_expr_reprs):
mock = MagicMock()
meth = MethodProxy(
_QueryProcessor,
mock,
class_attrs=[
# for them, the actual members/methods (not mocks) will be used
# class constants:
'queried_model_class',
'client_relationship',
'client_asoc_model_class',
# methods:
'pop_time_min_max_until',
'pop_limit',
'make_time_cmp_generator',
'query__ordering_by',
'query__limit',
])
actual_query_expr_reprs = []
and_mock = self._make_the_and_mock(actual_query_expr_reprs)
query_mock = self._make_the_query_mock(actual_query_expr_reprs)
mock.build_query.return_value = query_mock
with patch('n6lib.data_backend_api.and_', and_mock), \
patch('n6lib.data_backend_api.utcnow', return_value=self._UTCNOW):
list(
meth.generate_query_results(given_params,
item_number_limit=None,
day_step=1))
self.assertEqual(actual_query_expr_reprs, expected_query_expr_reprs)
def setUp(self):
self.cleaned_result_dict = {
'category': 'bots',
'source': 'hidden.42',
}
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock, 'OUTPUT_RK_PATTERN')
def setUp(self):
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock)
self.mock.data_spec = N6DataSpec()
self.mock.auth_api.get_anonymized_source_mapping.return_value = {
'forward_mapping': self.forward_source_mapping,
}
self.mock.auth_api.get_dip_anonymization_disabled_source_ids.return_value = frozenset()
class TestSSLUserAuthenticationPolicy(unittest.TestCase):
def setUp(self):
self.mock = Mock(__class__=SSLUserAuthenticationPolicy)
self.meth = MethodProxy(SSLUserAuthenticationPolicy, self.mock, [
'merge_orgid_userid',
])
def test__unauthenticated_userid__ok(self):
request = MagicMock()
request.environ = {'SSL_CLIENT_S_DN_O': 'OrgName1', 'SSL_CLIENT_S_DN_CN': 'UserName1'}
result = self.meth.unauthenticated_userid(request)
expected_result = 'OrgName1,UserName1'
self.assertEqual(result, expected_result)
def test__unauthenticated_userid__user_id_is_None(self):
request = MagicMock()
request.environ = {'SSL_CLIENT_S_DN_O': 'testorgname2', 'SSL_CLIENT_S_DN_CN': None}
result = self.meth.unauthenticated_userid(request)
self.assertIsNone(result)
def test__unauthenticated_userid__org_id_is_None(self):
request = MagicMock()
request.environ = {'SSL_CLIENT_S_DN_O': None, 'SSL_CLIENT_S_DN_CN': 'testusername3'}
result = self.meth.unauthenticated_userid(request)
self.assertIsNone(result)
@patch('n6lib.pyramid_commons._pyramid_commons.LOGGER.warning')
def test__unauthenticated_userid__comma_in_user_id(self, LOGGER_warning_mock):
request = MagicMock()
request.environ = {'SSL_CLIENT_S_DN_O': 'orgname4', 'SSL_CLIENT_S_DN_CN': 'user,name4'}
result = self.meth.unauthenticated_userid(request)
self.assertIsNone(result)
self.assertEqual(LOGGER_warning_mock.mock_calls,
[call('Comma in user_id %r.', 'user,name4')])
@patch('n6lib.pyramid_commons._pyramid_commons.LOGGER.warning')
def test__unauthenticated_userid__comma_in_org_id(self, LOGGER_warning_mock):
request = MagicMock()
request.environ = {'SSL_CLIENT_S_DN_O': 'orgname,5', 'SSL_CLIENT_S_DN_CN': 'username5'}
result = self.meth.unauthenticated_userid(request)
self.assertIsNone(result)
self.assertEqual(LOGGER_warning_mock.mock_calls,
[call('Comma in org_id %r.', 'orgname,5')])
def test_other_important_methods_are_from_BaseUserAuthenticationPolicy(self):
self.assertIs(SSLUserAuthenticationPolicy.get_auth_data,
BaseUserAuthenticationPolicy.get_auth_data)
self.assertIs(SSLUserAuthenticationPolicy.authenticated_userid.__func__,
BaseUserAuthenticationPolicy.authenticated_userid.__func__)
self.assertIs(SSLUserAuthenticationPolicy.effective_principals.__func__,
BaseUserAuthenticationPolicy.effective_principals.__func__)
def test(self,
raw_result_dict,
expected_result,
filtering_params=None,
url_normalization_data_cache=None):
mock = MagicMock()
meth = MethodProxy(_EventsQueryProcessor, mock)
mock._filtering_params = (copy.deepcopy(filtering_params)
if filtering_params is not None else {})
mock._url_normalization_data_cache = (url_normalization_data_cache
if url_normalization_data_cache
is not None else {})
raw_result_dict = copy.deepcopy(raw_result_dict)
actual_result = meth._preprocess_result_dict(raw_result_dict)
self.assertEqualIncludingTypes(actual_result, expected_result)
def test__get_redirect_url_if_no_time_min(self, request_query_string,
cleaned_param_dict,
expected_redirect_url):
mock = MagicMock()
mock.request.path_url = self.REQUEST_PATH_URL
mock.request.query_string = request_query_string
meth = MethodProxy(
RestAPIViewBase,
mock,
class_attrs=['SAFE_ACTIVE_MIN_DELTA', '_get_redirect_url'])
with patch('n6web.utcnow', return_value=self.UTC_NOW):
actual_redirect_url = meth.get_redirect_url_if_no_time_min(
cleaned_param_dict, self.DEFAULT_DELTA)
self.assertEqual(actual_redirect_url, expected_redirect_url)
def setUp(self):
self.event_type = 'bl-update'
self.event_data = {'some...': 'content...', 'id': 'some id...'}
self.routing_key = self.event_type + '.filtered.*.*'
self.body = json.dumps(self.event_data)
self.resource_to_org_ids = {}
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock, '_process_input')
self.mock._get_resource_to_org_ids.return_value = self.resource_to_org_ids
self.mock._get_result_dicts_and_output_body.return_value = (
sen.raw_result_dict,
sen.cleaned_result_dict,
sen.output_body,
)
def test(self):
mock = MagicMock()
meth = MethodProxy(_QueryProcessor, mock)
params = {
'name': 'foo-bar',
'opt.primary': True,
'time.min': [dt(2015, 1, 4)],
'time.until': [dt(2015, 1, 5)],
}
expected_params = {
'name': 'foo-bar',
'time.min': [dt(2015, 1, 4)],
'time.until': [dt(2015, 1, 5)],
}
meth.delete_opt_prefixed_params(params)
self.assertEqual(params, expected_params)
self.assertEqual(mock.mock_calls, [])
def setUp(self):
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock, '_VALID_EVENT_TYPES')
def setUp(self):
self.mock = Mock(__class__=AMQPThreadedPusher)
self.mock.DEFAULT_PROP_KWARGS = AMQPThreadedPusher.DEFAULT_PROP_KWARGS
self.meth = MethodProxy(AMQPThreadedPusher, self.mock)
class TestAMQPThreadedPusher__init__repr(unittest.TestCase):
def setUp(self):
self.mock = Mock(__class__=AMQPThreadedPusher)
self.mock.DEFAULT_PROP_KWARGS = AMQPThreadedPusher.DEFAULT_PROP_KWARGS
self.meth = MethodProxy(AMQPThreadedPusher, self.mock)
def test__init__using_defaults__no_ssl(self):
connection_params_dict_mock = RLockedMagicMock()
connection_params_dict_mock.get.return_value = False
self.meth.__init__(connection_params_dict=connection_params_dict_mock,
exchange='my-exchange')
# attrs
self.assertIs(self.mock._connection_params_dict,
connection_params_dict_mock)
self.assertEqual(self.mock._exchange, {'exchange': 'my-exchange'})
self.assertEqual(self.mock._exchange_name, 'my-exchange')
self.assertEqual(self.mock._queues_to_declare, [])
self.assertIsNone(self.mock._serialize)
self.assertEqual(self.mock._prop_kwargs,
AMQPThreadedPusher.DEFAULT_PROP_KWARGS)
self.assertEqual(self.mock._mandatory, False)
self.assertIs(self.mock._output_fifo.__class__, queue.Queue)
self.assertEqual(self.mock._output_fifo.maxsize, 20000)
self.assertIsNone(self.mock._error_callback)
# calls
self.assertEqual(self.mock.mock_calls, [
call._setup_communication(),
call._start_publishing(),
])
self.assertEqual(
connection_params_dict_mock.mock_calls,
[
call.get('ssl'),
('__contains__', ('client_properties', ),
{}), # because cannot use `call.__contains__`
call.__setitem__('client_properties', ANY),
])
self.assertIsInstance(
# 2nd argument passed to __setitem__()
connection_params_dict_mock.__setitem__.mock_calls[0][-2][1],
dict)
def test__init__specifying_all__with_ssl(self):
connection_params_dict_mock = RLockedMagicMock()
connection_params_dict_mock.get.return_value = True
connection_params_dict_mock.__contains__.return_value = True
self.meth.__init__(connection_params_dict=connection_params_dict_mock,
exchange={
'exchange': sen.exchange,
'foo': sen.foo
},
queues_to_declare=sen.queues_to_declare,
serialize=sen.serialize,
prop_kwargs=sen.prop_kwargs,
mandatory=sen.mandatory,
output_fifo_max_size=12345,
error_callback=sen.error_callback)
# attrs
self.assertIs(self.mock._connection_params_dict,
connection_params_dict_mock)
self.assertEqual(self.mock._exchange, {
'exchange': sen.exchange,
'foo': sen.foo
})
self.assertEqual(self.mock._exchange_name, sen.exchange)
self.assertEqual(self.mock._queues_to_declare, [
{
'queue': sen.queues_to_declare,
'callback': ANY
},
])
self.assertEqual(self.mock._serialize, sen.serialize)
self.assertEqual(self.mock._prop_kwargs, sen.prop_kwargs)
self.assertEqual(self.mock._mandatory, sen.mandatory)
self.assertIs(self.mock._output_fifo.__class__, queue.Queue)
self.assertEqual(self.mock._output_fifo.maxsize, 12345)
self.assertEqual(self.mock._error_callback, sen.error_callback)
# calls
self.assertEqual(self.mock.mock_calls, [
call._setup_communication(),
call._start_publishing(),
])
self.assertEqual(
connection_params_dict_mock.mock_calls,
[
call.get('ssl'),
call.setdefault('credentials', ANY),
('__contains__', ('client_properties', ),
{}), # because cannot use `call.__contains__`
])
self.assertIsInstance(
# 2nd argument passed to setdefault()
connection_params_dict_mock.setdefault.mock_calls[0][-2][1],
pika.credentials.ExternalCredentials)
def test__init__specifying_all_and_obtaining_global_conn_params__with_ssl(
self):
connection_params_dict_mock = RLockedMagicMock(
name='connection_params_dict')
connection_params_dict_mock.get.return_value = True
connection_params_dict_mock.__contains__.return_value = True
with rlocked_patch(
'n6lib.amqp_getters_pushers.get_amqp_connection_params_dict',
**{'return_value':
connection_params_dict_mock}) as get_amqp_conn_params_mock:
self.meth.__init__(connection_params_dict=None,
exchange={
'exchange': sen.exchange,
'bar': sen.bar
},
queues_to_declare=[
sen.queue1,
{
'blabla': sen.blabla
},
{
'blabla': sen.blabla,
'callback': sen.callback
},
],
serialize=sen.serialize,
prop_kwargs=sen.prop_kwargs,
mandatory=sen.mandatory,
output_fifo_max_size=54321,
error_callback=sen.error_callback)
# attrs
self.assertIs(self.mock._connection_params_dict,
connection_params_dict_mock)
self.assertEqual(self.mock._exchange, {
'exchange': sen.exchange,
'bar': sen.bar
})
self.assertEqual(self.mock._exchange_name, sen.exchange)
self.assertEqual(self.mock._queues_to_declare, [
{
'queue': sen.queue1,
'callback': ANY
},
{
'blabla': sen.blabla,
'callback': ANY
},
{
'blabla': sen.blabla,
'callback': sen.callback
},
])
self.assertEqual(self.mock._serialize, sen.serialize)
self.assertEqual(self.mock._prop_kwargs, sen.prop_kwargs)
self.assertEqual(self.mock._mandatory, sen.mandatory)
self.assertIs(self.mock._output_fifo.__class__, queue.Queue)
self.assertEqual(self.mock._output_fifo.maxsize, 54321)
self.assertEqual(self.mock._error_callback, sen.error_callback)
# calls
self.assertEqual(self.mock.mock_calls, [
call._setup_communication(),
call._start_publishing(),
])
self.assertEqual(get_amqp_conn_params_mock.mock_calls, [
call(),
])
self.assertEqual(
connection_params_dict_mock.mock_calls,
[
call.get('ssl'),
call.setdefault('credentials', ANY),
('__contains__', ('client_properties', ),
{}), # because cannot use `call.__contains__`
])
self.assertIsInstance(
connection_params_dict_mock.setdefault.mock_calls[0][-2]
[1], # 2nd arg for setdefault
pika.credentials.ExternalCredentials)
def test__repr(self):
string_repr = self.meth.__repr__()
self.assertIs(type(string_repr), str)
self.assertRegex(
string_repr, r'<AMQPThreadedPusher object at 0x[0-9a-f]+ with .*>')
class TestAnonymizer___publish_output_data(TestCaseMixin, unittest.TestCase):
def setUp(self):
self.cleaned_result_dict = {
'category': 'bots',
'source': 'hidden.42',
}
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock, 'OUTPUT_RK_PATTERN')
@foreach(
param(
resource_to_org_ids={
'inside': ['o2', 'o3'],
'threats': ['o3', 'o5', 'o8'],
},
expected_publish_output_calls=[
call(
routing_key='inside.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o3'
}},
),
call(
routing_key='inside.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o2'
}},
),
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o8'
}},
),
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o5'
}},
),
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o3'
}},
),
],
).label('for both resources'),
param(
resource_to_org_ids={
'inside': ['o2', 'o3'],
'threats': [],
},
expected_publish_output_calls=[
call(
routing_key='inside.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o3'
}},
),
call(
routing_key='inside.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o2'
}},
),
],
).label('for "inside" only'),
param(
resource_to_org_ids={
'inside': [],
'threats': ['o3', 'o5', 'o8'],
},
expected_publish_output_calls=[
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o8'
}},
),
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o5'
}},
),
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o3'
}},
),
],
).label('for "threats" only'),
param(
resource_to_org_ids={
'inside': [],
'threats': [],
},
expected_publish_output_calls=[],
).label('for no resources'),
)
def test_normal(self, resource_to_org_ids, expected_publish_output_calls):
with patch('n6.utils.anonymizer.LOGGER') as LOGGER_mock:
self.meth._publish_output_data(sen.event_type, resource_to_org_ids,
sen.raw_result_dict,
self.cleaned_result_dict,
sen.output_body)
self.assertEqual(self.mock.publish_output.mock_calls,
expected_publish_output_calls)
self.assertFalse(LOGGER_mock.error.mock_calls)
def test_error(self):
resource_to_org_ids = {
'inside': ['o2', 'o3'],
'threats': ['o3', 'o5', 'o8'],
}
expected_publish_output_calls = [
call(
routing_key='inside.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o3'
}},
),
call(
routing_key='inside.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o2'
}},
),
call(
routing_key='threats.bots.hidden.42',
body=sen.output_body,
prop_kwargs={'headers': {
'n6-client-id': 'o8'
}},
),
]
exc_type = ZeroDivisionError # (just an example exception class)
self.mock.publish_output.side_effect = [
None,
None,
exc_type,
]
with patch('n6.utils.anonymizer.LOGGER') as LOGGER_mock, \
self.assertRaises(exc_type):
self.meth._publish_output_data(sen.event_type, resource_to_org_ids,
sen.raw_result_dict,
self.cleaned_result_dict,
sen.output_body)
self.assertEqual(self.mock.publish_output.mock_calls,
expected_publish_output_calls)
self.assertEqual(LOGGER_mock.error.mock_calls, [
call(
ANY,
'threats',
'o8',
sen.event_type,
sen.raw_result_dict,
'threats.bots.hidden.42',
sen.output_body,
("for the resource 'inside' -- "
"* skipped for the org ids: none; "
"* done for the org ids: 'o3', 'o2'; "
"for the resource 'threats' -- "
"* skipped for the org ids: 'o3', 'o5', 'o8'; "
"* done for the org ids: none"),
),
])
class TestAnonymizer___get_result_dicts_and_output_body(
TestCaseMixin, unittest.TestCase):
forward_source_mapping = {
'some.source': 'hidden.42',
}
event_raw_base = dict(
id=(32 * '3'),
rid=(32 * '4'), # (restricted - to be skipped before *value* cleaning)
source='some.source', # (to be anonymized)
restriction=
'public', # (restricted - to be skipped before *value* cleaning)
confidence='low',
category='malurl',
time='2013-07-12 11:30:00',
)
cleaned_base = dict(
id=(32 * '3'),
source='hidden.42', # (after anonymization)
confidence='low',
category='malurl',
time=datetime.datetime(2013, 7, 12, 11, 30, 00),
type=sen.TO_BE_SET,
)
def setUp(self):
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock)
self.mock.data_spec = N6DataSpec()
self.mock.auth_api.get_anonymized_source_mapping.return_value = {
'forward_mapping': self.forward_source_mapping,
}
self.mock.auth_api.get_dip_anonymization_disabled_source_ids.return_value = frozenset(
)
@foreach(
param(
event_type='event',
event_data=dict(
event_raw_base,
client=
[], # (empty `client` -- to be skipped before *any* cleaning)
),
expected_raw=dict(event_raw_base, ),
expected_cleaned=dict(
cleaned_base,
type='event', # (event_type value set *after* cleaning)
),
),
param(
event_type='event',
event_data=dict(
event_raw_base,
client=['o1', 'o3', 'o2'],
address=
[], # (empty `address` -- to be skipped before *any* cleaning)
dip='192.168.0.1',
fqdn='www.example.com',
type=
'foobar', # (not a result key -- to be skipped before *any* cleaning)
blabla=
'foooo', # (not a result key -- to be skipped before *any* cleaning)
until='spamspam',
min_amplification=4000 * 'foo bar',
rid='xxxxx',
),
expected_raw=dict(
event_raw_base,
client=[
'o1', 'o3', 'o2'
], # (restricted -- to be skipped before *value* cleaning)
dip='192.168.0.1', # (to be anonymized -> as 'adip')
fqdn='www.example.com',
until=
'spamspam', # (restricted -- to be skipped before *value* cleaning)
min_amplification=4000 *
'foo bar', # (restricted [custom] -- as above)
rid='xxxxx', # (restricted [+required] -- as above)
),
expected_cleaned=dict(
cleaned_base,
adip='x.x.0.1', # ('dip' value after anonymization)
fqdn='www.example.com',
type='event', # (event_type value set *after* cleaning)
),
),
param(
event_type='bl-update',
event_data=dict(
event_raw_base,
**{
'client':
[], # (empty `client` -- to be skipped before *any* cleaning)
'address': [{
'ip': '1.2.3.4',
'cc': 'pl',
'asn': '1.1'
}],
'adip': 'x.10.20.30',
'dip': '192.168.0.1',
'_bl-series-no':
42, # (not a result field -- to be skipped before *any* cleaning)
'type':
'barfoo', # (not a result field -- to be skipped before *any* cleaning)
}),
expected_raw=dict(
event_raw_base,
**{
'address': [{
'ip': '1.2.3.4',
'cc': 'pl',
'asn': '1.1'
}],
'adip': 'x.10.20.30',
'dip':
'192.168.0.1', # (to be just omitted -- 'adip' is explicitly specified)
}),
expected_cleaned=dict(
cleaned_base,
address=[{
'ip': '1.2.3.4',
'cc': 'PL',
'asn': 65537
}],
adip='x.10.20.30', # (just given 'adip')
type='bl-update', # (event_type value set *after* cleaning)
),
),
# below -- the same two as above but with dip anonymization disabled
param(
event_type='event',
event_data=dict(
event_raw_base,
client=['o1', 'o3', 'o2'],
address=
[], # (empty `address` -- to be skipped before *any* cleaning)
dip='192.168.0.1',
fqdn='www.example.com',
type=
'foobar', # (not a result key -- to be skipped before *any* cleaning)
blabla=
'foooo', # (not a result key -- to be skipped before *any* cleaning)
until='spamspam',
min_amplification=4000 * 'foo bar',
rid='xxxxx',
),
expected_raw=dict(
event_raw_base,
client=[
'o1', 'o3', 'o2'
], # (restricted -- to be skipped before *value* cleaning)
dip='192.168.0.1', # (to be *not* anonymized [sic])
fqdn='www.example.com',
until=
'spamspam', # (restricted -- to be skipped before *value* cleaning)
min_amplification=4000 *
'foo bar', # (restricted [custom] -- as above)
rid='xxxxx', # (restricted [+required] -- as above)
),
expected_cleaned=dict(
cleaned_base,
dip='192.168.0.1', # (*not* anonymized [sic])
fqdn='www.example.com',
type='event', # (event_type value set *after* cleaning)
),
dip_anonymization_disabled_source_ids=frozenset(['some.source']),
),
param(
event_type='bl-update',
event_data=dict(
event_raw_base,
**{
'client':
[], # (empty `client` -- to be skipped before *any* cleaning)
'address': [{
'ip': '1.2.3.4',
'cc': 'pl',
'asn': '1.1'
}],
'adip': 'x.10.20.30',
'dip': '192.168.0.1',
'_bl-series-no':
42, # (not a result field -- to be skipped before *any* cleaning)
'type':
'barfoo', # (not a result field -- to be skipped before *any* cleaning)
}),
expected_raw=dict(
event_raw_base, **{
'address': [{
'ip': '1.2.3.4',
'cc': 'pl',
'asn': '1.1'
}],
'adip': 'x.10.20.30',
'dip': '192.168.0.1',
}),
expected_cleaned=dict(
cleaned_base,
address=[{
'ip': '1.2.3.4',
'cc': 'PL',
'asn': 65537
}],
adip='x.10.20.30', # (just given 'adip')
dip='192.168.0.1', # (just given 'dip' [sic])
type='bl-update', # (event_type value set *after* cleaning)
),
dip_anonymization_disabled_source_ids=frozenset(['some.source']),
),
)
def test_normal(self,
event_type,
event_data,
expected_raw,
expected_cleaned,
dip_anonymization_disabled_source_ids=frozenset()):
expected_auth_api_calls = [call.get_anonymized_source_mapping()]
if 'dip' in event_data:
expected_auth_api_calls.append(
call.get_dip_anonymization_disabled_source_ids())
self.mock.auth_api.get_dip_anonymization_disabled_source_ids.return_value = (
dip_anonymization_disabled_source_ids)
with patch('n6.utils.anonymizer.LOGGER') as LOGGER_mock:
(raw_result_dict, cleaned_result_dict,
output_body) = self.meth._get_result_dicts_and_output_body(
event_type, event_data, sen.resource_to_org_ids)
self.assertEqual(raw_result_dict, expected_raw)
self.assertEqual(cleaned_result_dict, expected_cleaned)
self.assertEqual(json.loads(output_body),
self._get_expected_body_content(expected_cleaned))
self.assertItemsEqual(self.mock.auth_api.mock_calls,
expected_auth_api_calls)
self.assertFalse(LOGGER_mock.error.mock_calls)
@staticmethod
def _get_expected_body_content(expected_cleaned):
formatted_time = expected_cleaned['time'].isoformat() + 'Z'
assert formatted_time[10] == 'T' and formatted_time[-1] == 'Z'
return dict(expected_cleaned, time=formatted_time)
@foreach(
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
),
without_keys={'id'},
exc_type=ResultKeyCleaningError,
).label('missing key: required and unrestricted'),
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
),
without_keys={'source'},
exc_type=ResultKeyCleaningError,
).label('missing key: required and anonymized'),
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
),
without_keys={'rid'},
exc_type=ResultKeyCleaningError,
).label('missing key: required and restricted'),
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
id='spam',
),
exc_type=ResultValueCleaningError,
).label('illegal value for required and unrestricted key'),
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
fqdn='foo..bar',
),
exc_type=ResultValueCleaningError,
).label('illegal value for optional and unrestricted key'),
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
dip='spam',
),
exc_type=ResultValueCleaningError,
).label('illegal value for optional and anonymized-source key'),
param(
event_data=dict(
event_raw_base,
client=['o3', 'o1', 'o2'],
adip='spam',
),
exc_type=ResultValueCleaningError,
).label('illegal value for optional and anonymized-target key'),
)
def test_error(self, event_data, exc_type, without_keys=()):
event_type = 'event'
event_data = event_data.copy()
for key in without_keys:
del event_data[key]
resource_to_org_ids = {'foo': {'bar'}, 'baz': {'spam', 'ham'}}
with patch('n6.utils.anonymizer.LOGGER') as LOGGER_mock, \
self.assertRaises(exc_type):
self.meth._get_result_dicts_and_output_body(
event_type, event_data, resource_to_org_ids)
self.assertEqual(len(LOGGER_mock.error.mock_calls), 1)
class TestAnonymizer__input_callback(TestCaseMixin, unittest.TestCase):
def setUp(self):
self.event_type = 'bl-update'
self.event_data = {'some...': 'content...', 'id': 'some id...'}
self.routing_key = self.event_type + '.filtered.*.*'
self.body = json.dumps(self.event_data)
self.resource_to_org_ids = {}
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock, '_process_input')
self.mock._get_resource_to_org_ids.return_value = self.resource_to_org_ids
self.mock._get_result_dicts_and_output_body.return_value = (
sen.raw_result_dict,
sen.cleaned_result_dict,
sen.output_body,
)
self.force_exit_on_any_remaining_entered_contexts_mock = self.patch(
'n6.utils.anonymizer.force_exit_on_any_remaining_entered_contexts')
@foreach(
param(resource_to_org_ids_items={
'foo': [sen.o1, sen.o2],
}),
param(resource_to_org_ids_items={
'foo': [sen.o1, sen.o2],
'bar': [],
}),
param(resource_to_org_ids_items={
'foo': [],
'bar': [sen.o3, sen.o4, sen.o5],
}),
param(resource_to_org_ids_items={
'foo': [sen.o1, sen.o2],
'bar': [sen.o3, sen.o4, sen.o5],
}),
)
def test_with_some_org_ids(self, resource_to_org_ids_items):
self.resource_to_org_ids.update(resource_to_org_ids_items)
self.meth.input_callback(self.routing_key, self.body, sen.properties)
self.assertEqual(
self.force_exit_on_any_remaining_entered_contexts_mock.mock_calls,
[
call(self.mock.auth_api),
])
self.assertEqual(self.mock.mock_calls, [
call.setting_error_event_info(self.event_data),
call.setting_error_event_info().__enter__(),
call._check_event_type(self.event_type, self.event_data),
call.auth_api.__enter__(),
call._get_resource_to_org_ids(self.event_type, self.event_data),
call._get_result_dicts_and_output_body(
self.event_type, self.event_data, self.resource_to_org_ids),
call._publish_output_data(
self.event_type, self.resource_to_org_ids, sen.raw_result_dict,
sen.cleaned_result_dict, sen.output_body),
call.auth_api.__exit__(None, None, None),
call.setting_error_event_info().__exit__(None, None, None),
])
@foreach(
param(resource_to_org_ids_items={}),
param(resource_to_org_ids_items={
'foo': [],
}),
param(resource_to_org_ids_items={
'foo': [],
'bar': [],
}),
)
def test_without_org_ids(self, resource_to_org_ids_items):
self.resource_to_org_ids.update(resource_to_org_ids_items)
self.meth.input_callback(self.routing_key, self.body, sen.properties)
self.assertEqual(
self.force_exit_on_any_remaining_entered_contexts_mock.mock_calls,
[
call(self.mock.auth_api),
])
self.assertEqual(self.mock.mock_calls, [
call.setting_error_event_info(self.event_data),
call.setting_error_event_info().__enter__(),
call._check_event_type(self.event_type, self.event_data),
call.auth_api.__enter__(),
call._get_resource_to_org_ids(self.event_type, self.event_data),
call.auth_api.__exit__(None, None, None),
call.setting_error_event_info().__exit__(None, None, None),
])
def test_with_some_error(self):
self.resource_to_org_ids.update({
'foo': [sen.o1, sen.o2],
'bar': [sen.o3, sen.o4, sen.o5],
})
exc_type = ZeroDivisionError # (just an example exception class)
self.mock._get_result_dicts_and_output_body.side_effect = exc_type
with self.assertRaises(exc_type) as exc_context:
self.meth.input_callback(self.routing_key, self.body,
sen.properties)
self.assertEqual(
self.force_exit_on_any_remaining_entered_contexts_mock.mock_calls,
[
call(self.mock.auth_api),
])
self.assertEqual(self.mock.mock_calls, [
call.setting_error_event_info(self.event_data),
call.setting_error_event_info().__enter__(),
call._check_event_type(self.event_type, self.event_data),
call.auth_api.__enter__(),
call._get_resource_to_org_ids(self.event_type, self.event_data),
call._get_result_dicts_and_output_body(
self.event_type, self.event_data, self.resource_to_org_ids),
call.auth_api.__exit__(exc_type, exc_context.exception, ANY),
call.setting_error_event_info().__exit__(
exc_type, exc_context.exception, ANY),
])
class Test__n6NormalizedData(unittest.TestCase):
def setUp(self):
self.mock = MagicMock()
self.meth = MethodProxy(n6NormalizedData, self.mock)
def test_class_attrs(self):
instrumented_attr_names = {
name for name, obj in vars(n6NormalizedData).items()
if isinstance(obj, sqlalchemy.orm.attributes.InstrumentedAttribute)}
column_names_to_sql_reprs = {
str(name): str(self._get_sql_repr(obj))
for name, obj in n6NormalizedData._n6columns.items()}
self.assertEqual(
n6NormalizedData.__tablename__,
'event')
self.assertEqual(
instrumented_attr_names, {
'clients',
'address',
'ip',
'asn',
'cc',
###'ipv6',
###'rdns',
###'dir',
'category',
'confidence',
'count',
'custom',
'dip',
'dport',
###'email',
'expires',
'fqdn',
###'iban',
'id',
###'injects',
'md5',
'modified',
'name',
'origin',
###'phone',
'proto',
###'registrar',
'replaces',
'restriction',
'rid',
'sha1',
'sha256',
'source',
'sport',
'status',
'target',
'time',
'until',
'url',
###'url_pattern',
###'username',
###'x509fp_sha1',
})
self.assertEqual(
instrumented_attr_names - {'clients'},
set(n6NormalizedData._n6columns))
self.assertEqual(
column_names_to_sql_reprs, {
'address': 'address TEXT', # note: in actual db it is MEDIUMTEXT
'ip': 'ip INTEGER UNSIGNED NOT NULL',
'asn': 'asn INTEGER UNSIGNED',
'cc': 'cc VARCHAR(2)',
###'ipv6': '',
###'rdns': '',
###'dir': '',
'category': (
"category ENUM('amplifier','bots','backdoor','cnc',"
"'deface','dns-query','dos-attacker','dos-victim','flow',"
"'flow-anomaly','fraud','leak','malurl','malware-action','other','phish',"
"'proxy','sandbox-url','scam','scanning','server-exploit','spam',"
"'spam-url','tor','vulnerable','webinject') NOT NULL"),
'confidence': "confidence ENUM('low','medium','high') NOT NULL",
'count': 'count SMALLINT',
'custom': 'custom TEXT', # note: in actual db it is MEDIUMTEXT
'dip': 'dip INTEGER UNSIGNED',
'dport': 'dport INTEGER',
###'email': '',
'expires': 'expires DATETIME',
'fqdn': 'fqdn VARCHAR(255)',
###'iban': '',
'id': 'id BINARY(16) NOT NULL',
###'injects': '',
'md5': 'md5 BINARY(16)',
'modified': 'modified DATETIME',
'name': 'name VARCHAR(255)',
'origin': (
"origin ENUM('c2','dropzone','proxy','p2p-crawler',"
"'p2p-drone','sinkhole','sandbox','honeypot',"
"'darknet','av','ids','waf')"),
###'phone': '',
'proto': "proto ENUM('tcp','udp','icmp')",
###'registrar': '',
'replaces': 'replaces BINARY(16)',
'restriction': "restriction ENUM('public','need-to-know','internal') NOT NULL",
'rid': 'rid BINARY(16) NOT NULL',
'sha1': 'sha1 BINARY(20)',
'sha256': 'sha256 BINARY(32)',
'source': 'source VARCHAR(32) NOT NULL',
'sport': 'sport INTEGER',
'status': "status ENUM('active','delisted','expired','replaced')",
'target': 'target VARCHAR(100)',
'time': 'time DATETIME NOT NULL',
'until': 'until DATETIME',
'url': 'url VARCHAR(2048)',
###'url_pattern': '',
###'username': '',
###'x509fp_sha1': '',
})
def _get_sql_repr(self, col):
type_name = (
str(col.type) if not isinstance(col.type, sqlalchemy.types.Enum)
else 'ENUM({0})'.format(','.join(
"'{0}'".format(e) for e in col.type.enums)))
r = '{0} {1}'.format(col.name, type_name)
if isinstance(col.type, IPAddress):
self.assertTrue(col.type.impl.mapping['mysql'].unsigned)
r += ' UNSIGNED'
self.assertIsInstance(col.nullable, bool)
if not col.nullable:
r += ' NOT NULL'
return r
def test_init_and_attrs_1(self):
obj = self.obj = n6NormalizedData(
id=sen.event_id,
ip=sen.some_ip_addr,
dport=sen.some_port_number,
time='2014-04-01 01:07:42+02:00',
)
self.assertEqual(obj.id, sen.event_id)
self.assertEqual(obj.ip, sen.some_ip_addr)
self.assertEqual(obj.dport, sen.some_port_number)
self.assertEqual(
obj.time,
datetime.datetime(2014, 3, 31, 23, 7, 42))
for name in n6NormalizedData._n6columns:
if name in ('id', 'ip', 'dport', 'time'):
continue
val = getattr(obj, name)
self.assertIsNone(val)
self.assertIsInstance(
obj.clients,
sqlalchemy.orm.collections.InstrumentedList)
self.assertEqual(obj.clients, [])
self.client1 = MagicMock()
self.client1.client = 'c1'
self.client2 = MagicMock()
self.client2.client = 'c2'
obj.clients.append(self.client2)
obj.clients.append(self.client1)
self.assertEqual(obj.clients, [self.client2, self.client1])
def test_init_and_attrs_2(self):
obj = self.obj = n6NormalizedData(
time='2014-04-01 01:07:42+02:00',
expires='2015-04-01 01:07:43+02:00',
until='2015-04-01 01:07:43+02:00',
)
self.assertIsNone(obj.id)
self.assertEqual(obj.ip, '0.0.0.0') # "no IP" placeholder
self.assertEqual(
obj.time,
datetime.datetime(2014, 3, 31, 23, 7, 42))
self.assertEqual(
obj.expires,
datetime.datetime(2015, 3, 31, 23, 7, 43))
### THIS IS A PROBLEM -- TO BE SOLVED IN #3113:
self.assertEqual(
obj.until,
'2015-04-01 01:07:43+02:00')
def test__key_query(self):
self.mock.some_key.in_.return_value = sen.result
act_result = self.meth.key_query('some_key', sen.value)
self.assertIs(act_result, sen.result)
self.mock.some_key.in_.assert_called_once_with(sen.value)
@foreach(
param(
key='url.sub',
mapped_to='url',
result=sen.or_result),
param(
key='fqdn.sub',
mapped_to='fqdn',
result=sen.or_result),
param(
key='fqdn.illegal',
exc_type=KeyError),
param(
key='illegal',
exc_type=KeyError),
)
@patch('n6lib.db_events.or_', return_value=sen.or_result)
def test__like_query(self, or_mock, key, mapped_to=None,
result=None, exc_type=None, **kwargs):
value = [
u'val',
u'ążź', # (ticket #8043 - `UnicodeEncodeError: 'ascii' codec can't encode...`)
]
if exc_type is None:
assert result is not None
getattr(self.mock, mapped_to).like.side_effect = [sen.term1, sen.term2]
act_result = self.meth.like_query(key, value)
self.assertIs(act_result, result)
or_mock.assert_called_once_with(sen.term1, sen.term2)
self.assertEqual(self.mock.mock_calls, [
getattr(call, mapped_to).like(u'%val%'),
getattr(call, mapped_to).like(u'%ążź%'),
])
else:
with self.assertRaises(exc_type):
self.meth.like_query(key, value)
@foreach(
param(
value=[('10.20.30.41', 24), ('10.20.30.41', 32)],
min_max_ips=[(169090560, 169090815), (169090601, 169090601)],
result=sen.or_result),
param(
value=[('10.20.30.41', 24)],
min_max_ips=[(169090560, 169090815)],
result=sen.or_result),
param(
value=[('10.20.30.441', 24), ('10.20.30.41', 32)],
exc_type=socket.error),
param(
value=[('10.20.30.441', 24)],
exc_type=socket.error),
param(
value=[None],
exc_type=TypeError),
param(
value=('10.20.30.41', 24),
exc_type=ValueError),
param(
value=None,
exc_type=TypeError),
)
@patch('n6lib.db_events.and_', return_value=sen.and_result)
@patch('n6lib.db_events.or_', return_value=sen.or_result)
def test__ip_net_query(self, or_mock, and_mock, value=None, min_max_ips=None,
result=None, exc_type=None, **kwargs):
key = MagicMock()
key.__ne__.side_effect = (lambda k: k != 'ip.net')
if exc_type is None:
assert result is not None
self.mock.ip.__ge__.side_effect = (lambda min_ip: (sen.term_ge, min_ip))
self.mock.ip.__le__.side_effect = (lambda max_ip: (sen.term_le, max_ip))
act_result = self.meth.ip_net_query(key, value)
self.assertIs(act_result, result)
or_mock.assert_called_once_with(*(len(value) * [sen.and_result]))
self.assertEqual(
and_mock.mock_calls,
[call(
(sen.term_ge, min_ip),
(sen.term_le, max_ip))
for min_ip, max_ip in min_max_ips])
else:
with self.assertRaises(exc_type):
self.meth.ip_net_query(key, value)
# the only operation on the key was one unequality test (against 'ip.net')
key.__ne__.assert_called_once_with('ip.net')
@foreach(
param(key='active.min', cmp_meth_name='__ge__'),
param(key='active.max', cmp_meth_name='__le__'),
param(key='active.until', cmp_meth_name='__lt__'),
param(key='active.illegal', exc_type=AssertionError),
param(key='illegal', exc_type=AssertionError),
)
@patch('n6lib.db_events.null', return_value=sen.Null)
@patch('n6lib.db_events.or_', return_value=sen.or_result)
@patch('n6lib.db_events.and_', return_value=sen.and_result)
def test__active_bl_query(self, and_mock, or_mock, null_mock,
key, cmp_meth_name=None, exc_type=None,
**kwargs):
value = [sen.val]
if exc_type is None:
self.mock.expires.is_.return_value = sen.expires_is_result
self.mock.expires.isnot.return_value = sen.expires_isnot_result
getattr(self.mock.expires, cmp_meth_name).return_value = sen.expires_cmp_result
getattr(self.mock.time, cmp_meth_name).return_value = sen.time_cmp_result
act_result = self.meth.active_bl_query(key, value)
self.assertIs(act_result, sen.or_result)
if key == 'active.min':
assert cmp_meth_name == '__ge__'
or_mock.assert_called_once_with(sen.expires_cmp_result, sen.time_cmp_result)
self.assertEqual(self.mock.expires.is_.mock_calls, [])
self.assertEqual(self.mock.expires.isnot.mock_calls, [])
else:
assert (
(key == 'active.max' and cmp_meth_name == '__le__') or
(key == 'active.until' and cmp_meth_name == '__lt__'))
or_mock.assert_called_once_with(sen.and_result, sen.and_result)
self.assertEqual(and_mock.mock_calls, [
call(sen.expires_isnot_result, sen.expires_cmp_result),
call(sen.expires_is_result, sen.time_cmp_result),
])
self.mock.expires.is_.assert_called_once_with(sen.Null)
self.mock.expires.isnot.assert_called_once_with(sen.Null)
getattr(self.mock.expires, cmp_meth_name).assert_called_once_with(sen.val)
getattr(self.mock.time, cmp_meth_name).assert_called_once_with(sen.val)
else:
with self.assertRaises(exc_type):
self.meth.active_bl_query(key, value)
@foreach(
param('modified.min', cmp_meth_name='__ge__'),
param('modified.max', cmp_meth_name='__le__'),
param('modified.until', cmp_meth_name='__lt__'),
param('modified.illegal', exc_type=AssertionError),
param('illegal', exc_type=AssertionError),
)
def test__modified_query(self, key, cmp_meth_name=None, exc_type=None):
value = [sen.val]
if exc_type is None:
getattr(self.mock.modified, cmp_meth_name).return_value = sen.result
act_result = self.meth.modified_query(key, value)
self.assertIs(act_result, sen.result)
getattr(self.mock.modified, cmp_meth_name).assert_called_once_with(sen.val)
else:
with self.assertRaises(exc_type):
self.meth.modified_query(key, value)
def test__to_raw_result_dict__1(self):
self.test_init_and_attrs_1()
d = self.obj.to_raw_result_dict()
self.assertEqual(d, {
'id': sen.event_id,
'ip': sen.some_ip_addr,
'dport': sen.some_port_number,
'time': datetime.datetime(2014, 3, 31, 23, 7, 42),
'client': ['c1', 'c2'],
})
def test__to_raw_result_dict__2(self):
self.test_init_and_attrs_2()
d = self.obj.to_raw_result_dict()
self.assertEqual(d, {
# note that ip='0.0.0.0' has been removed
'time': datetime.datetime(2014, 3, 31, 23, 7, 42),
'expires': datetime.datetime(2015, 3, 31, 23, 7, 43),
### THIS IS A PROBLEM -- TO BE SOLVED IN #3113:
'until': '2015-04-01 01:07:43+02:00',
})
class TestBaseUserAuthenticationPolicy(unittest.TestCase):
def setUp(self):
self.mock = Mock(__class__=BaseUserAuthenticationPolicy)
self.meth = MethodProxy(BaseUserAuthenticationPolicy, self.mock, [
'merge_orgid_userid',
])
def test__get_auth_data__ok(self):
request = MagicMock()
request.unauthenticated_userid = 'org_id,user_id'
request.registry.auth_api.authenticate.return_value = sen.auth_data
expected_result = sen.auth_data
result = BaseUserAuthenticationPolicy.get_auth_data(request)
self.assertEqual(request.registry.auth_api.authenticate.mock_calls,
[call('org_id', 'user_id')])
self.assertEqual(result, expected_result)
def test__get_auth_data__unauthenticated_userid_is_None(self):
request = MagicMock()
request.unauthenticated_userid = None
result = BaseUserAuthenticationPolicy.get_auth_data(request)
self.assertIsNone(result)
@patch('n6lib.pyramid_commons._pyramid_commons.LOGGER.warning')
def test__get_auth_data__authenticate_raises_exception(
self, LOGGER_warning_mock):
request = MagicMock()
request.unauthenticated_userid = 'org_id,user_id'
request.registry.auth_api.authenticate.side_effect = AuthAPIUnauthenticatedError
result = BaseUserAuthenticationPolicy.get_auth_data(request)
self.assertIsNone(result)
self.assertEqual(LOGGER_warning_mock.mock_calls, [
call(
'could not authenticate for '
'organization id %r + user id '
'%r', 'org_id', 'user_id')
])
def test__authenticated_userid__is_ok(self):
request = MagicMock()
request.auth_data = {'org_id': 'ORGid', 'user_id': 'USERid'}
result = self.meth.authenticated_userid(request)
expected_value = 'ORGid,USERid'
self.assertEqual(result, expected_value)
def test__authenticated_userid__is_None(self):
request = MagicMock()
request.auth_data = None
result = self.meth.authenticated_userid(request)
self.assertIsNone(result)
def test__effective_principals__when_authenticated(self):
self._patch_super_for_effective_principals()
request = MagicMock()
request.auth_data = {'org_id': 'ORGid', 'user_id': 'USERid'}
result = self.meth.effective_principals(request)
self.assertIn(Everyone, result)
self.assertIn(Authenticated, result)
self.assertIn('ORGid,USERid', result)
def test__effective_principals__when_not_authenticated(self):
self._patch_super_for_effective_principals()
request = MagicMock()
request.auth_data = None
result = self.meth.effective_principals(request)
self.assertIn(Everyone, result)
self.assertNotIn(Authenticated, result)
def _patch_super_for_effective_principals(self):
cls = BaseUserAuthenticationPolicy
super_effective_principals = super(cls,
cls).effective_principals.__func__
patcher = patch('n6lib.pyramid_commons._pyramid_commons.super',
create=True)
super_mock = patcher.start()
self.addCleanup(patcher.stop)
super_mock().effective_principals.side_effect = (
lambda request: super_effective_principals(sen.self, request))
class TestAnonymizer___check_event_type(TestCaseMixin, unittest.TestCase):
def setUp(self):
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock, '_VALID_EVENT_TYPES')
@foreach(
param(
event_type='event',
event_data={
'some_key': sen.some_value,
},
).label('no type in event data'),
param(
event_type='event',
event_data={
'type': 'event',
'some_key': sen.some_value,
},
).label('type "event" in event data'),
param(
event_type='bl-update',
event_data={
'type': 'bl-update',
'some_key': sen.some_value,
},
).label('another type in event data'),
)
def test_matching_and_valid(self, event_type, event_data):
assert (event_type == event_data.get('type', 'event')
and event_type in TYPE_ENUMS) # (test case self-test)
self.meth._check_event_type(event_type, event_data)
# the _check_event_type() method is called outside the AuthAPI
# context (outside its `with` statement) -- so we want to ensure
# that no AuthAPI methods are called:
self.assertEqual(self.mock.auth_api.mock_calls, [])
@foreach(
param(
event_type='event',
event_data={
'type': 'bl-update',
'some_key': sen.some_value,
},
).label('type "event" does not match another one'),
param(
event_type='bl-update',
event_data={
'type': 'event',
'some_key': sen.some_value,
},
).label('another type does not match "event"'),
)
def test_not_matching(self, event_type, event_data):
assert (event_type != event_data.get('type', 'event')
and event_type in TYPE_ENUMS) # (test case self-test)
with self.assertRaises(ValueError):
self.meth._check_event_type(event_type, event_data)
# the _check_event_type() method is called outside the AuthAPI
# context (outside its `with` statement) -- so we want to ensure
# that no AuthAPI methods are called:
self.assertEqual(self.mock.auth_api.mock_calls, [])
def test_matching_but_not_valid(self):
event_type = 'illegal'
event_data = {
'type': event_type,
'some_key': sen.some_value,
}
assert event_type not in TYPE_ENUMS # (test case self-test)
with self.assertRaises(ValueError):
self.meth._check_event_type(event_type, event_data)
# the _check_event_type() method is called outside the AuthAPI
# context (outside its `with` statement) -- so we want to ensure
# that no AuthAPI methods are called:
self.assertEqual(self.mock.auth_api.mock_calls, [])
def setUp(self):
self.mock = Mock(__class__=BaseParser, allow_empty_results=False)
self.meth = MethodProxy(BaseParser, self.mock)
class TestBaseParser(unittest.TestCase):
def setUp(self):
self.mock = Mock(__class__=BaseParser, allow_empty_results=False)
self.meth = MethodProxy(BaseParser, self.mock)
def _asserts_of_proper__new__instance_adjustment(self, instance):
# BaseQueued.__new__() ensures that
self.assertIsNot(instance.input_queue, BaseParser.input_queue)
def _asserts_of_proper_preinit_hook_instance_adjustment(
self, instance, binding_key):
# for classes with `default_binding_key`
# BaseParser.preinit_hook() ensures that
self.assertEqual(
instance.input_queue, {
'exchange': 'raw',
'exchange_type': 'topic',
'queue_name': binding_key,
'binding_keys': [binding_key],
})
self.assertEqual(BaseParser.input_queue, {
'exchange': 'raw',
'exchange_type': 'topic',
})
def _basic_init_related_asserts(self, instance, subclass, super_mock,
super_cls_mock, expected_config,
expected_config_full):
# assert that an instance of the proper type has been returned
self.assertIsInstance(instance, subclass)
# assert that super used properly
super_mock.assert_called_once_with(BaseParser, instance)
super_cls_mock.__init__.assert_called_once_with(a=sentinel.a,
bb=sentinel.bb)
# assert that configuration stuff has been obtained properly
self.assertEqual(instance.config, expected_config)
self.assertIsInstance(instance.config, ConfigSection)
self.assertEqual(instance.config_full, expected_config_full)
self.assertIsInstance(instance.config_full, Config)
def test_basics(self):
self.assertTrue(issubclass(BaseParser, QueuedBase))
self.assertTrue(hasattr(BaseParser, 'default_binding_key'))
self.assertTrue(hasattr(BaseParser, 'config_spec_pattern'))
self.assertTrue(hasattr(BaseParser, 'constant_items'))
self.assertTrue(hasattr(BaseParser, 'record_dict_class'))
self.assertTrue(hasattr(BaseParser, 'event_type'))
def test_config_spec_pattern(self):
config_spec = BaseParser.config_spec_pattern.format(
parser_class_name='example_foo')
config_spec_parsed = parse_config_spec(config_spec)
prefetch_count_opt_spec = config_spec_parsed.get_opt_spec(
'example_foo.prefetch_count')
self.assertEqual(prefetch_count_opt_spec.name, 'prefetch_count')
self.assertEqual(prefetch_count_opt_spec.converter_spec, 'int')
def test_initialization_without_default_binding_key(self):
class SomeParser(BaseParser):
pass # no `default_binding_key` defined => it's an abstract class
with self.assertRaises(NotImplementedError):
SomeParser()
unready_instance = SomeParser.__new__(SomeParser)
self._asserts_of_proper__new__instance_adjustment(unready_instance)
# for classes without `default_binding_key`
# `queue_name` and `binding_keys` items are *not* added...
self.assertEqual(unready_instance.input_queue, BaseParser.input_queue)
self.assertEqual(BaseParser.input_queue, {
'exchange': 'raw',
'exchange_type': 'topic',
})
@foreach(
param(
mocked_conf_from_files={},
expected_config=ConfigSection('SomeParser', {'prefetch_count': 1}),
expected_config_full=Config.make(
{'SomeParser': {
'prefetch_count': 1
}}),
),
param(
mocked_conf_from_files={
'SomeParser': {
'prefetch_count': '42'
},
'another_section': {
'another_opt': '123.456'
},
},
expected_config=ConfigSection('SomeParser',
{'prefetch_count': 42}),
expected_config_full=Config.make(
{'SomeParser': {
'prefetch_count': 42
}}),
),
param(
custom_config_spec_pattern=concat_reducing_indent(
BaseParser.config_spec_pattern,
'''
some_opt = [-3, null] :: json
[another_section]
another_opt :: float
yet_another_opt = Foo Bar Spam Ham
''',
),
mocked_conf_from_files={
'SomeParser': {
'prefetch_count': '42'
},
'another_section': {
'another_opt': '123.456'
},
},
expected_config=ConfigSection('SomeParser', {
'prefetch_count': 42,
'some_opt': [-3, None],
}),
expected_config_full=Config.make({
'SomeParser': {
'prefetch_count': 42,
'some_opt': [-3, None],
},
'another_section': {
'another_opt': 123.456,
'yet_another_opt': 'Foo Bar Spam Ham',
},
}),
),
)
@foreach(
param(binding_key='foo.bar'),
param(binding_key='foo.bar.33'),
)
def test_initialization_with_default_binding_key(
self,
binding_key,
mocked_conf_from_files,
expected_config,
expected_config_full,
custom_config_spec_pattern=None):
class SomeParser(BaseParser):
default_binding_key = binding_key # => it's a concrete class
if custom_config_spec_pattern is not None:
SomeParser.config_spec_pattern = custom_config_spec_pattern
unready_instance = SomeParser.__new__(SomeParser)
self._asserts_of_proper__new__instance_adjustment(unready_instance)
self._asserts_of_proper_preinit_hook_instance_adjustment(
unready_instance, binding_key)
super_cls_mock = SimpleNamespace(__init__=Mock())
with patch_always('n6.parsers.generic.super',
return_value=super_cls_mock) as super_mock, \
patch('n6.parsers.generic.Config._load_n6_config_files',
return_value=mocked_conf_from_files):
# instantiation
instance = SomeParser(a=sentinel.a, bb=sentinel.bb)
self._asserts_of_proper__new__instance_adjustment(instance)
self._asserts_of_proper_preinit_hook_instance_adjustment(
instance, binding_key)
self._basic_init_related_asserts(instance, SomeParser, super_mock,
super_cls_mock, expected_config,
expected_config_full)
def test__make_binding_keys(self):
self.mock.default_binding_key = 'fooo.barr'
binding_keys = self.meth.make_binding_keys()
self.assertEqual(binding_keys, ['fooo.barr'])
self.assertEqual(self.mock.mock_calls, [])
def test__make_binding_keys_with_raw_format_version_tag(self):
self.mock.default_binding_key = 'fooo.barr.33'
binding_keys = self.meth.make_binding_keys()
self.assertEqual(binding_keys, ['fooo.barr.33'])
self.assertEqual(self.mock.mock_calls, [])
def test__get_script_init_kwargs(self):
self.assertIsInstance(
vars(BaseParser)['get_script_init_kwargs'], classmethod)
init_kwargs = BaseParser.get_script_init_kwargs.__func__(self.mock)
self.assertEqual(init_kwargs, {})
self.assertEqual(self.mock.mock_calls, [])
def test__run_handling__interrupted(self):
self.mock.configure_mock(**{'run.side_effect': KeyboardInterrupt})
self.meth.run_handling()
self.mock.run.assert_called_once_with()
self.mock.stop.assert_called_once_with()
def test__run_handling__not_interrupted(self):
self.meth.run_handling()
self.mock.run.assert_called_once_with()
self.assertEqual(self.mock.stop.mock_calls, [])
@patch('n6.parsers.generic.FilePagedSequence')
def test__input_callback(self, FilePagedSequence_mock):
FilePagedSequence_mock.return_value = MagicMock()
FilePagedSequence_mock.return_value.__enter__.return_value = sentinel.working_seq
data = MagicMock(**{'get.return_value': sentinel.rid})
self.mock.configure_mock(
**{
'_fix_body.return_value':
sentinel.body,
'prepare_data.return_value':
data,
'setting_error_event_info':
MagicMock(),
'get_output_rk.return_value':
sentinel.output_rk,
'get_output_bodies.return_value':
[sentinel.output_body1, sentinel.output_body2],
})
self.meth.input_callback(sentinel.routing_key, sentinel.body,
sentinel.properties)
self.assertEqual(self.mock.mock_calls, [
call._fix_body(sentinel.body),
call.prepare_data(sentinel.routing_key, sentinel.body,
sentinel.properties),
call.prepare_data().get('properties.message_id'),
call.setting_error_event_info(sentinel.rid),
call.setting_error_event_info().__enter__(),
call.get_output_rk(data),
call.get_output_bodies(data, sentinel.working_seq),
call.publish_output(routing_key=sentinel.output_rk,
body=sentinel.output_body1),
call.publish_output(routing_key=sentinel.output_rk,
body=sentinel.output_body2),
call.setting_error_event_info().__exit__(None, None, None),
])
self.assertEqual(FilePagedSequence_mock.mock_calls, [
call(page_size=1000),
call().__enter__(),
call().__exit__(None, None, None),
])
def test__prepare_data(self):
data = self.meth.prepare_data(routing_key='ham.spam',
body=sentinel.body,
properties=SimpleNamespace(
foo=sentinel.foo,
bar=sentinel.bar,
timestamp=1389348840,
headers={'a': sentinel.a}))
self.assertEqual(
data, {
'a': sentinel.a,
'properties.foo': sentinel.foo,
'properties.bar': sentinel.bar,
'source': 'ham.spam',
'properties.timestamp': '2014-01-10 10:14:00',
'raw_format_version_tag': None,
'raw': sentinel.body,
})
def test__prepare_data__rk__with_raw_format_version_tag(self):
data = self.meth.prepare_data(routing_key='ham.spam.33',
body=sentinel.body,
properties=SimpleNamespace(
foo=sentinel.foo,
bar=sentinel.bar,
timestamp=1389348840,
headers={'a': sentinel.a}))
self.assertEqual(
data, {
'a': sentinel.a,
'properties.foo': sentinel.foo,
'properties.bar': sentinel.bar,
'source': 'ham.spam',
'properties.timestamp': '2014-01-10 10:14:00',
'raw_format_version_tag': '33',
'raw': sentinel.body,
})
def test__get_output_rk(self):
self.mock.configure_mock(**{
'event_type': 'foobar',
})
data = {'source': 'ham.spam'}
output_rk = self.meth.get_output_rk(data)
self.assertEqual(output_rk, 'foobar.parsed.ham.spam')
def test__get_output_bodies(self):
parsed = [
MagicMock(
**{
'__class__':
RecordDict,
'used_as_context_manager':
True,
'get_ready_json.return_value':
getattr(sentinel, 'output_body{}'.format(i))
}) for i in (1, 2)
]
self.mock.configure_mock(
**{
'parse.return_value':
parsed,
'get_output_message_id.side_effect': [
sentinel.msg_A,
sentinel.msg_B,
],
'setting_error_event_info':
MagicMock(),
'postprocess_parsed.side_effect': (
lambda data, parsed, total, item_no: parsed),
})
seq_mock = FilePagedSequence._instance_mock()
output_bodies = self.meth.get_output_bodies(sentinel.data, seq_mock)
self.assertIs(output_bodies, seq_mock)
self.assertEqual(seq_mock._list, [
sentinel.output_body1,
sentinel.output_body2,
])
self.assertEqual(parsed[0].mock_calls, [
call.__setitem__('id', sentinel.msg_A),
call.get_ready_json(),
])
self.assertEqual(parsed[1].mock_calls, [
call.__setitem__('id', sentinel.msg_B),
call.get_ready_json(),
])
self.assertEqual(self.mock.mock_calls, [
call.parse(sentinel.data),
call.get_output_message_id(parsed[0]),
call.delete_too_long_address(parsed[0]),
call.get_output_message_id(parsed[1]),
call.delete_too_long_address(parsed[1]),
call.setting_error_event_info(parsed[0]),
call.setting_error_event_info().__enter__(),
call.postprocess_parsed(sentinel.data, parsed[0], 2, item_no=1),
call.setting_error_event_info().__exit__(None, None, None),
call.setting_error_event_info(parsed[1]),
call.setting_error_event_info().__enter__(),
call.postprocess_parsed(sentinel.data, parsed[1], 2, item_no=2),
call.setting_error_event_info().__exit__(None, None, None),
])
def test__get_output_bodies__record_dict_not_used_as_context_manager(self):
parsed = [
MagicMock(**{
'__class__': RecordDict,
'used_as_context_manager': False
}) for i in (1, 2)
]
self.mock.configure_mock(**{'parse.return_value': parsed})
with self.assertRaises(AssertionError):
self.meth.get_output_bodies(sentinel.data,
FilePagedSequence._instance_mock())
self.assertEqual(self.mock.method_calls, [
call.parse(sentinel.data),
])
def test__get_output_bodies__parse_yielded_no_items(self):
self.mock.configure_mock(**{'parse.return_value': iter([])})
with self.assertRaises(ValueError):
self.meth.get_output_bodies(sentinel.data,
FilePagedSequence._instance_mock())
self.assertEqual(self.mock.method_calls, [
call.parse(sentinel.data),
])
def test__get_output_bodies__parse_yielded_no_items__allow_empty_results(
self):
self.mock.configure_mock(**{
'parse.return_value': iter([]),
'allow_empty_results': True
})
seq_mock = FilePagedSequence._instance_mock()
output_bodies = self.meth.get_output_bodies(sentinel.data, seq_mock)
self.assertIs(output_bodies, seq_mock)
self.assertEqual(seq_mock._list, []) # just empty
self.assertEqual(self.mock.mock_calls, [
call.parse(sentinel.data),
])
def test__delete_too_long_address__address_is_ok(self):
parsed = RecordDict()
parsed['address'] = [{'ip': i + 1} for i in xrange(MAX_IPS_IN_ADDRESS)]
expected = RecordDict()
expected['address'] = [{
'ip': i + 1
} for i in xrange(MAX_IPS_IN_ADDRESS)]
self.meth.delete_too_long_address(parsed)
self.assertEqual(parsed, expected)
def test__delete_too_long_address__address_is_too_long(self):
ips = MAX_IPS_IN_ADDRESS + 1
parsed = RecordDict()
parsed['id'] = '0123456789abcdef0123456789abcdef'
parsed['address'] = [{'ip': i + 1} for i in xrange(ips)]
expected = RecordDict()
expected['id'] = '0123456789abcdef0123456789abcdef'
self.meth.delete_too_long_address(parsed)
self.assertEqual(parsed, expected)
def test__delete_too_long_address__address_is_empty(self):
parsed = RecordDict()
parsed.update({'source': 'foo.bar'})
expected = RecordDict()
expected.update({'source': 'foo.bar'})
self.meth.delete_too_long_address(parsed)
self.assertEqual(parsed, expected)
def test__get_output_message_id(self):
inputs_and_resultant_hash_bases = [
# basics
({
'source': 'foo.bar'
}, 'source,foo.bar'),
({
u'source': u'foo.bar'
}, 'source,foo.bar'),
# proper sorting of multiple values
({
'key1': 2,
'key2': ['value2', 'value3', 'value1']
}, 'key1,2\nkey2,value1,value2,value3'),
# ...and of keys + proper encoding of unicode keys/values
({
u'key2': [u'value3', u'value1', u'value2'],
u'key1': 2L
}, 'key1,2\nkey2,value1,value2,value3'),
# ...as well as proper int/long normalization/representation
({
u'key2': [30, 10, 20L],
u'key1': 9000111222333444555666777888999000L
}, 'key1,9000111222333444555666777888999000\nkey2,10,20,30'),
# non-ascii values
({
'target': 'zażółć',
u'client': [u'jaźń', u'gęślą']
}, 'client,gęślą,jaźń\ntarget,zażółć'),
({
u'target': u'zażółć',
'client': ['jaźń', 'gęślą']
}, 'client,gęślą,jaźń\ntarget,zażółć'),
# subdicts
({
'dip': u'3.3.3.3',
u'address': [{
'ip': '255.255.255.0'
}, {
'ip': '127.0.0.1'
}]
},
"address,{'ip': '127.0.0.1'},{'ip': '255.255.255.0'}\ndip,3.3.3.3"
),
# non-ascii subdict keys/values
({
u'key2': [{
'ką2': 'vą2'
}, {
'ką1': 'vą1'
}],
'key1': {
'ką': 'vą'
}
}, "key1,{'k\\xc4\\x85': 'v\\xc4\\x85'}\n" +
"key2,{'k\\xc4\\x851': 'v\\xc4\\x851'},{'k\\xc4\\x852': 'v\\xc4\\x852'}"
),
# proper encoding of unicode keys/values + proper sorting of whole subdicts
({
'key1': {
u'ką': u'vą'
},
'key2': [{
u'ką2': 'vą2'
}, {
'ką1': u'vą1'
}]
}, "key1,{'k\\xc4\\x85': 'v\\xc4\\x85'}\n" +
"key2,{'k\\xc4\\x851': 'v\\xc4\\x851'},{'k\\xc4\\x852': 'v\\xc4\\x852'}"
),
# ...as well as proper int/long normalization/representation
({
'key1': {
u'k': 2L
},
'key2': [{
'k2': 2L
}, {
u'k1': 1
}]
}, "key1,{'k': 2}\nkey2,{'k1': 1},{'k2': 2}"),
({
u'key2': [{
'k2': 2
}, {
'k1': 1
}],
'key1': {
'k': 3
}
}, "key1,{'k': 3}\nkey2,{'k1': 1},{'k2': 2}"),
({
u'key2': [{
'k2': 2L
}, {
'k1': 1L
}],
'key1': {
'k': 9000111222333444555666777888999000L
}
},
"key1,{'k': 9000111222333444555666777888999000}\nkey2,{'k1': 1},{'k2': 2}"
),
# proper sorting of multiple items in subdicts
({
'key1': {
'c': 2,
u'a': 3L,
u'b': 1L
},
'key2': [{
'c': 2,
u'a': 3L,
u'b': 1L
}, {
'd': 3,
u'a': 2L,
u'b': 1L
}]
}, "key1,{'a': 3, 'b': 1, 'c': 2}\n" +
"key2,{'a': 2, 'b': 1, 'd': 3},{'a': 3, 'b': 1, 'c': 2}"),
]
class _RecordDict(RecordDict):
adjust_key1 = adjust_key2 = None
optional_keys = RecordDict.optional_keys | {'key1', 'key2'}
parser = BaseParser.__new__(BaseParser)
for input_dict, expected_base in inputs_and_resultant_hash_bases:
record_dict = _RecordDict(input_dict)
expected_result = hashlib.md5(expected_base).hexdigest()
result = parser.get_output_message_id(record_dict)
self.assertIsInstance(result, str)
self.assertEqual(result, expected_result)
def test__get_output_message_id__errors(self):
inputs_and_exc_classes = [
# bad subdict key type
(
{
'key1': {
32: 2
}
},
TypeError,
),
(
{
'key1': [{
32: 2
}]
},
TypeError,
),
# bad subdict value type
(
{
'key1': {
'k': 2.3
}
},
TypeError,
),
(
{
'key1': [{
'k': 2.3
}]
},
TypeError,
),
(
{
'key1': {
'k': {
'k': 2
}
}
}, # nesting is illegal
TypeError,
),
(
{
'key1': [{
'k': {
'k': 2
}
}]
}, # nesting is illegal
TypeError,
),
# bad value type
(
{
'key1': 2.3
},
TypeError,
),
(
{
'key1': [2.3]
},
TypeError,
),
(
{
'key1': [[2]]
}, # nesting is illegal
TypeError,
),
]
class _RecordDict(RecordDict):
adjust_key1 = adjust_key2 = None
optional_keys = RecordDict.optional_keys | {'key1', 'key2'}
parser = BaseParser.__new__(BaseParser)
for input_dict, exc_class in inputs_and_exc_classes:
record_dict = _RecordDict(input_dict)
with self.assertRaises(exc_class):
parser.get_output_message_id(record_dict)
def test__postprocess_parsed__without__do_not_resolve_fqdn_to_ip(self):
data = {}
parsed = RecordDict()
self.meth.postprocess_parsed(data, parsed, 1, item_no=1)
self.assertEqual(parsed, {})
def test__postprocess_parsed__with__do_not_resolve_fqdn_to_ip__False(self):
data = {'_do_not_resolve_fqdn_to_ip': False}
parsed = RecordDict()
self.meth.postprocess_parsed(data, parsed, 1, item_no=1)
self.assertEqual(parsed, {})
def test__postprocess_parsed__with__do_not_resolve_fqdn_to_ip__True(self):
data = {'_do_not_resolve_fqdn_to_ip': True}
parsed = RecordDict()
self.meth.postprocess_parsed(data, parsed, 1, item_no=1)
self.assertEqual(parsed, {'_do_not_resolve_fqdn_to_ip': True})
def setUp(self):
self.mock = Mock(__class__=SSLUserAuthenticationPolicy)
self.meth = MethodProxy(SSLUserAuthenticationPolicy, self.mock, [
'merge_orgid_userid',
])
class TestAnonymizer___get_resource_to_org_ids(TestCaseMixin,
unittest.TestCase):
def setUp(self):
self.event_type = 'bl-update'
def YES_predicate(record):
self.assertIsInstance(record, RecordFacadeForPredicates)
return True
def NO_predicate(record):
self.assertIsInstance(record, RecordFacadeForPredicates)
return False
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock)
self.mock.data_spec = N6DataSpec()
self.mock.auth_api.get_source_ids_to_subs_to_stream_api_access_infos.return_value = \
self.s_to_s_to_saai = {
'src.empty': {},
'src.some-1': {
sen.something_1: (
YES_predicate,
{
'inside': set(),
'threats': set(),
'search': set(),
}
),
sen.something_2: (
YES_predicate,
{
'inside': {'o4'},
'threats': set(),
'search': {'o1', 'o2', 'o3', 'o4', 'o5', 'o6'},
}
),
sen.something_3: (
NO_predicate,
{
'inside': {'o2'},
'threats': {'o3'},
'search': set(),
}
),
sen.something_4: (
NO_predicate,
{
'inside': {'o1', 'o3', 'o9'},
'threats': {'o3', 'o5', 'o6'},
'search': {'o3', 'o4', 'o5', 'o6'},
}
),
},
'src.some-2': {
sen.something_5: (
YES_predicate,
{
'inside': {'o1', 'o3', 'o9'},
'threats': {'o3', 'o5', 'o6'},
'search': {'o3', 'o4', 'o5', 'o6'},
}
),
sen.something_6: (
YES_predicate,
{
'inside': {'o2'},
'threats': {'o2'},
'search': set(),
}
),
sen.something_7: (
YES_predicate,
{
'inside': set(),
'threats': {'o8'},
'search': set(),
}
),
sen.something_8: (
YES_predicate,
{
'inside': set(),
'threats': set(),
'search': set(),
}
),
sen.something_9: (
NO_predicate,
{
'inside': {'o1', 'o5', 'o4', 'o9'},
'threats': {'o3', 'o4', 'o5', 'o9'},
'search': {'o1', 'o2', 'o3', 'o4'},
}
),
},
}
@foreach(
param(
event_data=dict(
source='src.not-found',
client=['o5', 'o1', 'o3', 'o2'],
),
expected_result=dict(
inside=[],
threats=[],
),
).label('no such source'),
param(
event_data=dict(
source='src.empty',
client=['o5', 'o1', 'o3', 'o2'],
),
expected_result=dict(
inside=[],
threats=[],
),
).label('no subsources'),
param(
event_data=dict(
source='src.some-1',
client=['o5', 'o1', 'o3', 'o2'],
),
expected_result=dict(
inside=[],
threats=[],
),
).label('no matching subsources/organizations'),
param(
event_data=dict(
source='src.some-2',
client=['o5', 'o1', 'o3', 'o2'],
),
expected_result=dict(
inside=['o1', 'o2', 'o3'],
threats=['o2', 'o3', 'o5', 'o6', 'o8'],
),
).label('some matching subsources and organizations (1)'),
param(
event_data=dict(
source='src.some-2',
client=['o2', 'o4', 'o9'],
),
expected_result=dict(
inside=['o2', 'o9'],
threats=['o2', 'o3', 'o5', 'o6', 'o8'],
),
).label('some matching subsources and organizations (2)'),
param(
event_data=dict(
source='src.some-2',
client=['o4'],
),
expected_result=dict(
inside=[],
threats=['o2', 'o3', 'o5', 'o6', 'o8'],
),
).label('some matching subsources and organizations (only "threats")'),
)
def test_normal(self, event_data, expected_result):
expected_mock_calls = [
call.auth_api.get_source_ids_to_subs_to_stream_api_access_infos(),
]
with patch('n6.utils.anonymizer.LOGGER') as LOGGER_mock:
result = self.meth._get_resource_to_org_ids(
self.event_type, event_data)
self.assertEqual(result, expected_result)
self.assertEqual(self.mock.mock_calls, expected_mock_calls)
self.assertFalse(LOGGER_mock.error.mock_calls)
def test_error(self):
event_data = dict(
source='src.some-2',
client=['o5', 'o1', 'o3', 'o2'],
)
res_to_org_ids = {
'inside': set(),
'threats': {'o8'},
'search': set(),
}
exc_type = ZeroDivisionError # (just an example exception class)
def raise_exc(rec):
raise exc_type('blablabla')
self.s_to_s_to_saai['src.some-2'][
sen.something_7] = raise_exc, res_to_org_ids
with patch('n6.utils.anonymizer.LOGGER') as LOGGER_mock, \
self.assertRaises(exc_type):
self.meth._get_resource_to_org_ids(self.event_type, event_data)
self.assertEqual(len(LOGGER_mock.error.mock_calls), 1)
def setUp(self):
self.mock = MagicMock()
self.meth = MethodProxy(n6NormalizedData, self.mock)
def setUp(self):
self.event_type = 'bl-update'
def YES_predicate(record):
self.assertIsInstance(record, RecordFacadeForPredicates)
return True
def NO_predicate(record):
self.assertIsInstance(record, RecordFacadeForPredicates)
return False
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock)
self.mock.data_spec = N6DataSpec()
self.mock.auth_api.get_source_ids_to_subs_to_stream_api_access_infos.return_value = \
self.s_to_s_to_saai = {
'src.empty': {},
'src.some-1': {
sen.something_1: (
YES_predicate,
{
'inside': set(),
'threats': set(),
'search': set(),
}
),
sen.something_2: (
YES_predicate,
{
'inside': {'o4'},
'threats': set(),
'search': {'o1', 'o2', 'o3', 'o4', 'o5', 'o6'},
}
),
sen.something_3: (
NO_predicate,
{
'inside': {'o2'},
'threats': {'o3'},
'search': set(),
}
),
sen.something_4: (
NO_predicate,
{
'inside': {'o1', 'o3', 'o9'},
'threats': {'o3', 'o5', 'o6'},
'search': {'o3', 'o4', 'o5', 'o6'},
}
),
},
'src.some-2': {
sen.something_5: (
YES_predicate,
{
'inside': {'o1', 'o3', 'o9'},
'threats': {'o3', 'o5', 'o6'},
'search': {'o3', 'o4', 'o5', 'o6'},
}
),
sen.something_6: (
YES_predicate,
{
'inside': {'o2'},
'threats': {'o2'},
'search': set(),
}
),
sen.something_7: (
YES_predicate,
{
'inside': set(),
'threats': {'o8'},
'search': set(),
}
),
sen.something_8: (
YES_predicate,
{
'inside': set(),
'threats': set(),
'search': set(),
}
),
sen.something_9: (
NO_predicate,
{
'inside': {'o1', 'o5', 'o4', 'o9'},
'threats': {'o3', 'o4', 'o5', 'o9'},
'search': {'o1', 'o2', 'o3', 'o4'},
}
),
},
}