def setUp(self):
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock)
self.mock.data_spec = N6DataSpec()
self.mock.auth_api.get_anonymized_source_mapping.return_value = {
'forward_mapping': self.forward_source_mapping,
}
self.mock.auth_api.get_dip_anonymization_disabled_source_ids.return_value = frozenset()
def get_concrete_view_class_kwargs(cls, view_class, request):
d = super(TestRestAPIViewBase_prepare_params,
cls).get_concrete_view_class_kwargs(view_class, request)
d.update({
'resource_id': '/search/events',
'data_spec': N6DataSpec(),
'data_backend_api_method': 'search_events',
'renderers': 'json',
})
return d
[time_max_until] = cleaned_param_dict.get(
'time.max', cleaned_param_dict.get('time.until', [None]))
assert isinstance(time_max_until, datetime.datetime)
time_min = min(time_min, time_max_until - default_delta)
redir_params['time_min'] = time_min
redir_template_parts.append('time.min={time_min}')
return self._get_redirect_url(redir_template_parts, redir_params)
def _get_redirect_url(self, template_parts, params):
template = '&'.join(template_parts)
query_str = template.format(**params)
return "{}?{}".format(self.request.path_url, query_str)
n6_data_spec = N6DataSpec()
n6_inside_data_spec = N6InsideDataSpec()
STREAM_RENDERERS = [
'json',
'csv',
'sjson',
'snort-dns',
'snort-http',
'snort-ip',
'snort-ip-bl',
'suricata-dns',
'suricata-http',
'suricata-ip',
'suricata-ip-bl',
]
def setUp(self):
self.event_type = 'bl-update'
def YES_predicate(record):
self.assertIsInstance(record, RecordFacadeForPredicates)
return True
def NO_predicate(record):
self.assertIsInstance(record, RecordFacadeForPredicates)
return False
self.mock = MagicMock(__class__=Anonymizer)
self.meth = MethodProxy(Anonymizer, self.mock)
self.mock.data_spec = N6DataSpec()
self.mock.auth_api.get_source_ids_to_subs_to_stream_api_access_infos.return_value = \
self.s_to_s_to_saai = {
'src.empty': {},
'src.some-1': {
sen.something_1: (
YES_predicate,
{
'inside': set(),
'threats': set(),
'search': set(),
}
),
sen.something_2: (
YES_predicate,
{
'inside': {'o4'},
'threats': set(),
'search': {'o1', 'o2', 'o3', 'o4', 'o5', 'o6'},
}
),
sen.something_3: (
NO_predicate,
{
'inside': {'o2'},
'threats': {'o3'},
'search': set(),
}
),
sen.something_4: (
NO_predicate,
{
'inside': {'o1', 'o3', 'o9'},
'threats': {'o3', 'o5', 'o6'},
'search': {'o3', 'o4', 'o5', 'o6'},
}
),
},
'src.some-2': {
sen.something_5: (
YES_predicate,
{
'inside': {'o1', 'o3', 'o9'},
'threats': {'o3', 'o5', 'o6'},
'search': {'o3', 'o4', 'o5', 'o6'},
}
),
sen.something_6: (
YES_predicate,
{
'inside': {'o2'},
'threats': {'o2'},
'search': set(),
}
),
sen.something_7: (
YES_predicate,
{
'inside': set(),
'threats': {'o8'},
'search': set(),
}
),
sen.something_8: (
YES_predicate,
{
'inside': set(),
'threats': set(),
'search': set(),
}
),
sen.something_9: (
NO_predicate,
{
'inside': {'o1', 'o5', 'o4', 'o9'},
'threats': {'o3', 'o4', 'o5', 'o9'},
'search': {'o1', 'o2', 'o3', 'o4'},
}
),
},
}
class RecordDict(collections.MutableMapping):
"""
Record dict class for non-blacklist events.
"""
_ADJUSTER_PREFIX = 'adjust_'
_APPENDER_PREFIX = 'append_'
data_spec = N6DataSpec()
required_keys = frozenset(data_spec.result_field_specs('required'))
optional_keys = frozenset(data_spec.result_field_specs('optional')) | {
# note: the 'type' item is somewhat related to
# <parser class>.event_type but *not* to <collector class>.type (!)
'type', ## <- FIXME???: shouldn't it be required? (not optional?)
'enriched', # (its values are added by enricher)
# internal keys
# (items whose keys start with '_' are neither recorded
# into database nor used for id computation)
'_do_not_resolve_fqdn_to_ip', # flag for enricher
'_parsed_old',
# internal keys of aggregated items
'_group',
'_first_time',
# internal keys of blacklist items
## FIXME?: shouldn't they be required
## (not optional) for BLRecordDict???
'_bl-series-no',
'_bl-series-total',
'_bl-series-id',
'_bl-time',
'_bl-current-time',
}
# for the following keys, if the given value is invalid,
# AdjusterError is not propagated; instead the value is just
# not stored (and a warning is logged)
without_adjuster_error = frozenset({
'fqdn',
'name',
'url',
'url_pattern',
})
#
# Instantiation-related methods
@classmethod
def from_json(cls, json_string, **kwargs):
return cls(json.loads(json_string), **kwargs)
def __init__(self,
iterable_or_mapping=(),
log_nonstandard_names=False,
context_manager_error_callback=None):
self._dict = {}
self._settable_keys = (self.required_keys | self.optional_keys)
# to catch some kinds of bugs early...
duplicated = self.required_keys & self.optional_keys
if duplicated:
raise ValueError('{} has keys declared both '
'as required and optional: {}'.format(
self.__class__.__name__,
', '.join(sorted(duplicated))))
missing_adjusters = [
key for key in self._settable_keys
if not hasattr(self, self._adjuster_name(key))
]
if missing_adjusters:
raise TypeError('{!r} has no adjusters for keys: {}'.format(
self, ', '.join(sorted(missing_adjusters))))
self.log_nonstandard_names = log_nonstandard_names
# context-manager (__enter__/__exit__) -related stuff
self.context_manager_error_callback = context_manager_error_callback
self.used_as_context_manager = False
self.update(iterable_or_mapping)
@classmethod
def _adjuster_name(cls, key):
return cls._ADJUSTER_PREFIX + key.replace('-', '')
#
# Output-related methods
def get_ready_dict(self):
current_keys = set(self._dict)
assert self._settable_keys >= current_keys
missing_keys = self.required_keys - current_keys
if missing_keys:
raise ValueError('missing keys: ' +
', '.join(sorted(missing_keys)))
ready_dict = copy.deepcopy(self._dict)
######## provide the legacy item
######## (needed by old version of RecordDict, in not-yet-updated components)
used_custom_keys = self.data_spec.custom_field_keys.intersection(
ready_dict)
if used_custom_keys:
ready_dict['__preserved_custom_keys__'] = sorted(used_custom_keys)
######## ^^^ (to be removed later)
return ready_dict
def get_ready_json(self):
# changed from json.dumps on bson.dumps
### XXX: why? bson.json_utils.dumps() pre-converts some values, but is it necessary???
return dumps(self.get_ready_dict())
def iter_db_items(self):
# to be cloned later (see below)
item_prototype = {
key: value
for key, value in self.get_ready_dict().iteritems()
if not key.startswith('_')
} # no internal keys
# pop actual custom items and place them in the "custom" field
all_custom_keys = self.data_spec.custom_field_keys
custom_items = {
key: item_prototype.pop(key)
for key in all_custom_keys if key in item_prototype
}
if custom_items:
item_prototype['custom'] = custom_items
# depending on "address" provide one or more database items (dicts)
address_list = item_prototype.pop('address',
None) # NOTE: deleting `address`
if address_list:
# the `address` list was present and not empty
# -> db item for each list item (each db item containing
# `ip`[/`cc`/`asn`] of the list item + the whole `address`)
item_prototype['address'] = address_list # restore
all_addr_keys_are_legal = {'ip', 'cc', 'asn'}.issuperset
for addr in address_list:
assert 'ip' in addr and all_addr_keys_are_legal(addr)
# cloning the prototype dict...
db_item = item_prototype.copy()
# ...and updating the copy with particular address data
db_item.update(addr)
yield db_item
else:
# the `address` list was *empty* or *not* present
# -> only one db item *without* `address`, `ip` etc.
yield item_prototype
__repr__ = attr_repr('_dict')
#
# MutableMapping interface implementation
def __iter__(self):
return iter(self._dict)
def __len__(self):
return len(self._dict)
def __getitem__(self, key):
return self._dict[key]
def __delitem__(self, key):
del self._dict[key]
def __setitem__(self, key, value):
######## silently ignore the legacy item
if key == '__preserved_custom_keys__': return
######## ^^^ (to be removed later)
try:
self._dict[key] = self._get_adjusted_value(key, value)
except AdjusterError as exc:
if key in self.without_adjuster_error:
LOGGER.warning('Invalid value not stored (%s)', exc)
else:
raise
def _get_adjusted_value(self, key, value):
if key not in self._settable_keys:
raise RuntimeError('for {!r}, key {!r} is illegal'.format(
self, key))
adjuster_method_name = self._adjuster_name(key)
try:
adjuster = getattr(self, adjuster_method_name)
except AttributeError:
raise RuntimeError('{!r} has no adjuster for key {!r}'.format(
self, key))
if adjuster is None:
# adjuster explicitly set to None -> passing value unchanged
return value
try:
return adjuster(value)
except Exception as exc:
if getattr(exc, 'propagate_it_anyway', False):
raise
adjuster_error_msg = ('{!r}.{}({value!r}) raised '
'{exc.__class__.__name__}: {exc}'.format(
self,
adjuster_method_name,
value=value,
exc=exc))
raise AdjusterError(adjuster_error_msg)
# reimplementation only for speed
def __contains__(self, key):
return key in self._dict
# reimplementation with slightly different interface
# and some additional guarantees
def update(self, iterable_or_mapping=()):
iterator = (iterable_or_mapping.iteritems() if isinstance(
iterable_or_mapping, collections.Mapping) else
iter(iterable_or_mapping))
setitem = self.__setitem__
# updating in a deterministic order: sorted by key (in particular,
# 'category' is set *before* 'name' -- see adjust_name())
sorted_items = sorted(iterator)
for key, value in sorted_items:
setitem(key, value)
# record dicts are always deep-copied (to avoid hard-to-find bugs)
def copy(self):
return copy.deepcopy(self)
__copy__ = copy
#
# Context manager interface
def __enter__(self):
self.used_as_context_manager = True
return self
def __exit__(self, exc_type, exc, tb):
try:
error_callback = self.context_manager_error_callback
except AttributeError:
raise TypeError('a record dict instance cannot be used '
'as a guarding context manager more than once')
try:
if exc_type is not None and error_callback is not None:
if exc is None:
exc = exc_type()
return error_callback(exc)
finally:
del self.context_manager_error_callback
#
# Adjusters
adjust_id = make_adjuster_using_data_spec('id')
adjust_rid = make_adjuster_using_data_spec('rid')
adjust_source = make_adjuster_using_data_spec('source')
adjust_origin = make_adjuster_using_data_spec('origin')
adjust_restriction = make_adjuster_using_data_spec('restriction')
adjust_confidence = make_adjuster_using_data_spec('confidence')
adjust_category = make_adjuster_using_data_spec('category')
adjust_md5 = make_adjuster_using_data_spec('md5')
adjust_sha1 = make_adjuster_using_data_spec('sha1')
adjust_proto = make_adjuster_using_data_spec('proto')
adjust_sport = make_adjuster_using_data_spec('sport')
adjust_dport = make_adjuster_using_data_spec('dport')
adjust_count = make_adjuster_using_data_spec('count')
adjust_time = chained(
make_adjuster_using_data_spec('time'), # will return datetime
make_adjuster_applying_callable(str)) # will transform it to str
adjust_modified = chained(
make_adjuster_using_data_spec('modified'), # will return datetime
make_adjuster_applying_callable(str)) # will transform it to str
adjust_address = chained(
make_multiadjuster(make_dict_adjuster(ip=ipv4_preadjuster)),
applied_for_nonfalse(make_adjuster_using_data_spec('address')))
adjust_dip = chained(ipv4_preadjuster,
make_adjuster_using_data_spec('dip'))
adjust_url = chained(
url_preadjuster, make_adjuster_using_data_spec('url',
on_too_long=trim))
adjust_fqdn = make_adjuster_using_data_spec('fqdn',
on_too_long=trim_domain)
adjust_client = chained(
make_multiadjuster(),
applied_for_nonfalse(make_adjuster_using_data_spec('client')))
adjust_until = chained(
make_adjuster_using_data_spec('until'), # will return datetime
make_adjuster_applying_callable(str)) # will transform it to str
adjust_expires = chained(
make_adjuster_using_data_spec('expires'), # will return datetime
make_adjuster_applying_callable(str)) # will transform it to str
adjust_target = make_adjuster_using_data_spec('target', on_too_long=trim)
adjust_type = make_adjuster_using_data_spec('_type')
# generic internal field adjusters
adjust__do_not_resolve_fqdn_to_ip = ensure_isinstance(bool)
adjust__parsed_old = rd_adjuster
# hi-freq-only internal field adjusters
adjust__group = unicode_adjuster
adjust__first_time = chained(
make_adjuster_using_data_spec('_first_time'), # will return datetime
make_adjuster_applying_callable(str)) # will transform it to str
# bl-only non-internal field adjusters
adjust_status = make_adjuster_using_data_spec('status')
adjust_replaces = make_adjuster_using_data_spec('replaces')
# bl-only internal field adjusters
adjust__blseriesno = make_adjuster_using_data_spec('_blseriesno')
adjust__blseriestotal = make_adjuster_using_data_spec('_blseriestotal')
adjust__blseriesid = make_adjuster_using_data_spec('_blseriesid')
adjust__bltime = chained(
make_adjuster_using_data_spec('_bltime'), # will return datetime
make_adjuster_applying_callable(str)) # will transform it to str
adjust__blcurrenttime = chained(
make_adjuster_using_data_spec('_blcurrenttime'),
make_adjuster_applying_callable(str))
# special custom field adjuster
# (see the comment in the code of n6.utils.enrich.Enricher.enrich())
adjust_enriched = make_adjuster_using_data_spec('enriched')
# custom field adjusters
adjust_adip = make_adjuster_using_data_spec('adip')
adjust_additional_data = make_adjuster_using_data_spec('additional_data',
on_too_long=trim)
adjust_alternative_fqdns = chained(
make_multiadjuster(),
make_adjuster_using_data_spec('alternative_fqdns',
on_too_long=trim_domain_seq))
adjust_description = make_adjuster_using_data_spec('description',
on_too_long=trim)
adjust_ip_network = make_adjuster_using_data_spec('ip_network')
adjust_min_amplification = make_adjuster_using_data_spec(
'min_amplification', on_too_long=trim)
adjust_request = make_adjuster_using_data_spec('request', on_too_long=trim)
adjust_user_agent = make_adjuster_using_data_spec('user_agent',
on_too_long=trim)
adjust_sender = make_adjuster_using_data_spec('sender', on_too_long=trim)
adjust_botid = make_adjuster_using_data_spec('botid', on_too_long=trim)
adjust_method = make_adjuster_using_data_spec('method', on_too_long=trim)
adjust_channel = make_adjuster_using_data_spec('channel', on_too_long=trim)
adjust_first_seen = make_adjuster_using_data_spec('first_seen',
on_too_long=trim)
adjust_referer = make_adjuster_using_data_spec('referer', on_too_long=trim)
adjust_proxy_type = make_adjuster_using_data_spec('proxy_type',
on_too_long=trim)
adjust_dns_version = make_adjuster_using_data_spec('dns_version',
on_too_long=trim)
adjust_internal_ip = make_adjuster_using_data_spec('internal_ip',
on_too_long=trim)
adjust_ipmi_version = make_adjuster_using_data_spec('ipmi_version',
on_too_long=trim)
adjust_mac_address = make_adjuster_using_data_spec('mac_address',
on_too_long=trim)
adjust_sysdesc = make_adjuster_using_data_spec('sysdesc', on_too_long=trim)
adjust_version = make_adjuster_using_data_spec('version', on_too_long=trim)
adjust_dataset = make_adjuster_using_data_spec('dataset', on_too_long=trim)
adjust_header = make_adjuster_using_data_spec('header', on_too_long=trim)
adjust_detected_since = make_adjuster_using_data_spec('detected_since',
on_too_long=trim)
adjust_handshake = make_adjuster_using_data_spec('handshake',
on_too_long=trim)
adjust_cert_length = make_adjuster_using_data_spec('cert_length',
on_too_long=trim)
adjust_subject_common_name = make_adjuster_using_data_spec(
'subject_common_name', on_too_long=trim)
adjust_visible_databases = make_adjuster_using_data_spec(
'visible_databases', on_too_long=trim)
adjust_url_pattern = make_adjuster_using_data_spec('url_pattern')
adjust_urls_matched = make_adjuster_using_data_spec('urls_matched')
adjust_username = make_adjuster_using_data_spec('username')
adjust_email = make_adjuster_using_data_spec('email')
adjust_facebook_id = make_adjuster_using_data_spec('facebook_id')
adjust_iban = make_adjuster_using_data_spec('iban')
adjust_injects = make_adjuster_using_data_spec('injects')
adjust_phone = make_adjuster_using_data_spec('phone')
adjust_registrar = make_adjuster_using_data_spec('registrar',
on_too_long=trim)
adjust_x509fp_sha1 = make_adjuster_using_data_spec('x509fp_sha1')
adjust_x509issuer = make_adjuster_using_data_spec('x509issuer')
adjust_x509subject = make_adjuster_using_data_spec('x509subject')
adjust_action = make_adjuster_using_data_spec('action')
# The attribute and related methods are left for the backward
# compatibility with older data from the MISP sources.
adjust_misp_eventdid = make_adjuster_using_data_spec('misp_eventdid')
adjust_misp_attr_uuid = make_adjuster_using_data_spec('misp_attr_uuid')
adjust_misp_event_uuid = make_adjuster_using_data_spec('misp_event_uuid')
adjust_product = make_adjuster_using_data_spec('product')
# custom field used for cooperation with IntelMQ
adjust_intelmq = make_adjuster_using_data_spec('intelmq')
adjust_tags = chained(
make_multiadjuster(),
make_adjuster_using_data_spec('tags', on_too_long=trim_seq))
# the `name` adjuster is a bit more complex...
@preceded_by(unicode_adjuster)
def adjust_name(self, value):
category = self.get('category')
if category is None:
exc = RuntimeError('cannot set "name" when "category" is not set')
exc.propagate_it_anyway = True # let the programmer know it!
raise exc
if not value:
raise ValueError('empty value')
if category in CATEGORY_TO_NORMALIZED_NAME:
value = self._get_normalized_name(value, category)
value = self._adjust_name_according_to_data_spec(value)
self._check_and_handle_nonstandard_name(value, category)
else:
value = self._adjust_name_according_to_data_spec(value)
return value
_adjust_name_according_to_data_spec = make_adjuster_using_data_spec(
'name', on_too_long=trim)
def _get_normalized_name(self, value, category):
value = value.lower()
first_char = value[0]
normalization = NAME_NORMALIZATION.get(first_char,
NAME_NORMALIZATION['ELSE'])
for regex, normalized_value in normalization:
if regex.search(value):
value = normalized_value
break
return value
def _check_and_handle_nonstandard_name(self, value, category):
if self.log_nonstandard_names:
category_std_names = self._get_category_std_names(category)
if value not in category_std_names:
self._log_nonstandard_name(value, category)
def _get_category_std_names(self, category_key):
while True:
category_std_names = CATEGORY_TO_NORMALIZED_NAME[category_key]
if not isinstance(category_std_names, basestring):
return category_std_names
category_key = category_std_names
# private class attribute: a cache of already logged non-standard names
# -- used in _log_nonstandard_name() to avoid cluttering the logs
_already_logged_nonstandard_names = LimitedDict(maxlen=10000)
def _log_nonstandard_name(
self,
value,
category,
_already_logged=_already_logged_nonstandard_names):
if (category, value) not in _already_logged:
category_sublogger = NONSTANDARD_NAMES_LOGGER.getChild(category)
category_sublogger.warning(ascii_str(value))
_already_logged[(category, value)] = None
#
# Appenders for multiple-adjusted attributes
# Providing methods: append_<key> -- for example:
# * append_address(<singular value>)
def __getattr__(self, name):
if name.startswith(self._APPENDER_PREFIX):
key = name[len(self._APPENDER_PREFIX):]
adjuster_method_name = self._adjuster_name(key)
adjuster = getattr(self, adjuster_method_name, None)
if self._is_multiadjuster(adjuster):
def appender(singular_value):
value_seq = list(self.get(key, []))
value_seq.append(singular_value)
self[key] = value_seq
return appender
raise AttributeError('{.__class__.__name__!r} object has '
'no attribute {!r}'.format(self, name))
@staticmethod
def _is_multiadjuster(adjuster):
factory_names = getattr(adjuster, '_factory_names', frozenset())
return ('make_multiadjuster' in factory_names)
class n6NormalizedData(Base):
__tablename__ = 'event'
_n6columns = dict(N6DataSpec().generate_sqlalchemy_columns(
id=dict(primary_key=True),
time=dict(primary_key=True),
ip=dict(primary_key=True, autoincrement=False)))
locals().update(_n6columns) # hack, but a simple one, and it works :)
clients = relationship(
n6ClientToEvent,
# XXX: is it necessary anymore?
primaryjoin=and_(
#_n6columns['time'] == n6ClientToEvent.time, ### redundant join condition
_n6columns['id'] == n6ClientToEvent.id),
#foreign_keys=[n6ClientToEvent.time, n6ClientToEvent.id],
# XXX: is it necessary anymore?
foreign_keys=[n6ClientToEvent.id],
backref="events")
def __init__(self, **kwargs):
if kwargs.get('ip') is None:
# adding the "no IP" placeholder ('0.0.0.0') which should be
# transformed into 0 in the database (because `ip` cannot be
# NULL in our SQL db; and apparently, for unknown reason, # XXX: <- check whether that's true...
# IPAddress.process_bind_param() is not called by the
# SQLAlchemy machinery if the value of `ip` is just None)
kwargs['ip'] = IPAddress.NONE_STR
kwargs['time'] = parse_iso_datetime_to_utc(kwargs["time"])
kwargs['expires'] = (parse_iso_datetime_to_utc(kwargs.get("expires"))
if kwargs.get("expires") is not None else None)
kwargs['modified'] = (parse_iso_datetime_to_utc(kwargs.get("modified"))
if kwargs.get("modified") is not None else None)
for name in self._n6columns:
setattr(self, name, kwargs.pop(name, None))
### XXX: the 'until' field is not converted here to utc datetime!
### (see ticket #3113)
kwargs.pop('client', None) # here we just ignore this arg if present
kwargs.pop('type', None) # here we just ignore this arg if present
if kwargs:
LOGGER.warning(
'n6NormalizedData.__init__() got unexpected **kwargs: %a',
kwargs)
@classmethod
def get_column_mapping_attrs(cls):
return [getattr(cls, name) for name in sorted(cls._n6columns)]
@classmethod
def key_query(cls, key, value):
return getattr(cls, key).in_(value)
@classmethod
def like_query(cls, key, value):
mapping = {"url.sub": "url", "fqdn.sub": "fqdn"}
return or_(*[
getattr(cls, mapping[key]).like(u"%{}%".format(val))
for val in value
])
@classmethod
def url_b64_experimental_query(cls, key, value):
# *EXPERIMENTAL* (likely to be changed or removed in the future
# without any warning/deprecation/etc.)
if key != 'url.b64':
raise AssertionError("key != 'url.b64' (but == {!a})".format(key))
db_key = 'url'
url_search_keys = list(map(make_provisional_url_search_key, value))
return or_(
getattr(cls, db_key).in_(value),
getattr(cls, db_key).in_(url_search_keys))
@classmethod
def ip_net_query(cls, key, value):
if key != 'ip.net':
# (`assert` not used because of the check in a unit test...)
raise AssertionError
queries = []
for val in value:
min_ip, max_ip = ip_network_tuple_to_min_max_ip(val)
queries.append(and_(cls.ip >= min_ip, cls.ip <= max_ip))
return or_(*queries)
@classmethod
def active_bl_query(cls, key, value):
assert len(value) == 1
value = value[0]
if key == "active.min":
return or_(cls.expires >= value, cls.time >= value)
elif key == "active.max":
return or_(and_(cls.expires.isnot(null()), cls.expires <= value),
and_(cls.expires.is_(null()), cls.time <= value))
elif key == "active.until":
return or_(and_(cls.expires.isnot(null()), cls.expires < value),
and_(cls.expires.is_(null()), cls.time < value))
else:
raise AssertionError
@classmethod
def modified_query(cls, key, value):
assert len(value) == 1
value = value[0]
if key == "modified.min":
return cls.modified >= value
elif key == "modified.max":
return cls.modified <= value
elif key == "modified.until":
return cls.modified < value
else:
raise AssertionError
def to_raw_result_dict(self):
client_org_ids = (c.client for c in self.clients) # noqa
return make_raw_result_dict(self, client_org_ids)
def __init__(self, **kwargs):
LOGGER.info("Anonymizer Start")
super(Anonymizer, self).__init__(**kwargs)
self.auth_api = AuthAPI()
self.data_spec = N6DataSpec()
class n6NormalizedData(Base):
__tablename__ = 'event'
_n6columns = dict(N6DataSpec().generate_sqlalchemy_columns(
id=dict(primary_key=True),
time=dict(primary_key=True),
ip=dict(primary_key=True, autoincrement=False)))
locals().update(_n6columns) # hack, but a simple one, and it works :)
clients = relationship(
n6ClientToEvent,
primaryjoin=and_(
#_n6columns['time'] == n6ClientToEvent.time, ### redundant join condition
_n6columns['id'] == n6ClientToEvent.id),
#foreign_keys=[n6ClientToEvent.time, n6ClientToEvent.id],
foreign_keys=[n6ClientToEvent.id],
backref="events")
def __init__(self, **kwargs):
if kwargs.get('ip') is None:
# adding the "no IP" placeholder ('0.0.0.0') which should be
# transformed into 0 in the database (because `ip` cannot be
# NULL in our SQL db; and apparently, for unknown reason,
# IPAddress.process_bind_param() is not called by the
# SQLAlchemy machinery if the value of `ip` is just None)
kwargs['ip'] = IPAddress.NONE_STR
kwargs['time'] = parse_iso_datetime_to_utc(kwargs["time"])
kwargs['expires'] = (parse_iso_datetime_to_utc(kwargs.get("expires"))
if kwargs.get("expires") is not None else None)
kwargs['modified'] = (parse_iso_datetime_to_utc(kwargs.get("modified"))
if kwargs.get("modified") is not None else None)
for name in self._n6columns:
setattr(self, name, kwargs.pop(name, None))
### XXX: the 'until' field is not converted here to utc datetime!
### (see ticket #3113)
kwargs.pop('client', None) # here we just ignore this arg if present
kwargs.pop('type', None) # here we just ignore this arg if present
if kwargs:
LOGGER.warning(
'n6NormalizedData.__init__() got unexpected **kwargs: %r',
kwargs)
@classmethod
def key_query(cls, key, value):
return getattr(cls, key).in_(value)
@classmethod
def like_query(cls, key, value):
mapping = {"url.sub": "url", "fqdn.sub": "fqdn"}
return or_(*[
getattr(cls, mapping[key]).like("%{}%".format(val))
for val in value
])
@classmethod
def ip_net_query(cls, key, value):
if key != 'ip.net':
# (`assert` not used because of the check in a unit test...)
raise AssertionError
queries = []
for val in value:
min_ip, max_ip = ip_network_tuple_to_min_max_ip(val)
queries.append(and_(cls.ip >= min_ip, cls.ip <= max_ip))
return or_(*queries)
@classmethod
def active_bl_query(cls, key, value):
assert len(value) == 1
value = value[0]
if key == "active.min":
return or_(cls.expires >= value, cls.time >= value)
elif key == "active.max":
return or_(and_(cls.expires.isnot(null()), cls.expires <= value),
and_(cls.expires.is_(null()), cls.time <= value))
elif key == "active.until":
return or_(and_(cls.expires.isnot(null()), cls.expires < value),
and_(cls.expires.is_(null()), cls.time < value))
else:
raise AssertionError
@classmethod
def modified_query(cls, key, value):
assert len(value) == 1
value = value[0]
if key == "modified.min":
return cls.modified >= value
elif key == "modified.max":
return cls.modified <= value
elif key == "modified.until":
return cls.modified < value
else:
raise AssertionError
# names of columns whose `type` attribute
# is IPAddress or its subclass/instance
_ip_column_names = tuple(
sorted(name for name, column in _n6columns.iteritems()
if isinstance(column.type, IPAddress) or (
isinstance(column.type, (type, types.ClassType))
and issubclass(column.type, IPAddress))))
# possible "no IP" placeholder values (such that they
# cause recording `ip` in db as 0) -- excluding None
_no_ip_placeholders = frozenset([IPAddress.NONE_STR, IPAddress.NONE, -1])
def to_raw_result_dict(
self,
# for faster (local) access:
_getattr=getattr,
_ip_column_names=_ip_column_names,
_no_ip_placeholders=_no_ip_placeholders):
# make the dict, skipping all None values
columns = self.__table__.columns
result_dict = {
name: value
for name, value in [(c.name, _getattr(self, c.name))
for c in columns] if value is not None
}
# get rid of any "no IP" placeholders (note: probably, this is not
# needed when the instance has been obtained by a DB operation so
# IPAddress.process_result_value() was used by the SQLAlchemy
# machinery)
for ip_col_name in _ip_column_names:
value = result_dict.get(ip_col_name)
if value in _no_ip_placeholders:
del result_dict[ip_col_name]
# set the 'client' item
client = [c.client for c in self.clients]
if client:
result_dict['client'] = client
return result_dict
def setUp(self):
self.ds = N6DataSpec()
