def add(self, key, value, sanitize=False, force=False, ignore=()):
if not force and key in self:
raise exceptions.KeyExists(key)
if value is None or value == "":
return
for invalid_value in ["-", "N/A"]:
if value == invalid_value:
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
try:
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize:
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
raise exceptions.InvalidValue(key, old_value)
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
raise exceptions.InvalidValue(key, value, reason=valid_value[1])
super(Message, self).__setitem__(key, value)
def is_valid(self, key: str, value: str, sanitize: bool = True) -> bool:
"""
Checks if a value is valid for the key (after sanitation).
Parameters:
key: Key of the field
value: Value of the field
sanitize: Sanitation of harmonization type will be called before validation
(default: True)
Returns:
True if the value is valid, otherwise False
Raises:
intelmq.lib.exceptions.InvalidKey: if given key is invalid.
"""
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
if value is None or value in ["", "-", "N/A"]:
return False
if sanitize:
value = self.__sanitize_value(key, value)
valid = self.__is_valid_value(key, value)
if valid[0]:
return True
return False
def is_valid(self, key, value, sanitize=True):
"""
Checks if a value is valid for the key (after sanitation).
Parameters
==========
key : string
value : string
sanitize : boolean
Sanitation of harmonization type will be called before validation (default: True)
Returns
=======
retval : boolean
Raises
======
intelmq.lib.exceptions.InvalidKey: if key is invalid.
"""
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
if value is None or value in ["", "-", "N/A"]:
return False
if sanitize:
value = self.__sanitize_value(key, value)
valid = self.__is_valid_value(key, value)
if valid[0]:
return True
return False
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
for harm_key in self.harmonization_config.keys():
if not re.match('^[a-z_](.[a-z_0-9]+)*$',
harm_key) and harm_key != '__type':
raise exceptions.InvalidKey(
"Harmonization key %r is invalid." % harm_key)
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def add(self, key, value, sanitize=True, force=False, ignore=()):
if not force and key in self:
raise exceptions.KeyExists(key)
if value is None or value == "":
return
if value in ["-", "N/A"]:
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
try:
warnings.warn('The ignore-argument will be removed in 1.0.',
DeprecationWarning)
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize and not key == '__type':
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
raise exceptions.InvalidValue(key, old_value)
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
raise exceptions.InvalidValue(key, value, reason=valid_value[1])
super(Message, self).__setitem__(key, value)
def __init__(self,
message: Union[dict, tuple] = (),
auto: bool = False,
harmonization: dict = None) -> None:
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
if (classname == 'event' and 'extra' in self.harmonization_config
and self.harmonization_config['extra']['type'] == 'JSON'):
warnings.warn(
"Assuming harmonization type 'JSONDict' for harmonization field 'extra'. "
"This assumption will be removed in version 3.0.",
DeprecationWarning)
self.harmonization_config['extra']['type'] = 'JSONDict'
for harm_key in self.harmonization_config.keys():
if not re.match('^[a-z_](.[a-z_0-9]+)*$',
harm_key) and harm_key != '__type':
raise exceptions.InvalidKey(
"Harmonization key %r is invalid." % harm_key)
super().__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
else:
raise ValueError(
"Type %r of message can't be handled, must be dict or tuple.",
type(message))
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def add(self,
key: str,
value: str,
sanitize: bool = True,
force: bool = False,
overwrite: bool = False,
ignore: Sequence = (),
raise_failure: bool = True) -> bool:
"""
Add a value for the key (after sanitation).
Parameters:
key: Key as defined in the harmonization
value: A valid value as defined in the harmonization
sanitize: Sanitation of harmonization type will be called before validation
(default: True)
force: Deprecated, use overwrite (default: False)
overwrite: Overwrite an existing value if it already exists (default: False)
ignore: List or tuple of values to ignore, deprecated (default: ())
raise_failure: If a intelmq.lib.exceptions.InvalidValue should be raised for
invalid values (default: True). If false, the return parameter will be
False in case of invalid values.
Returns:
* True if the value has been added.
* False if the value is invalid and raise_failure is False.
Raises:
intelmq.lib.exceptions.KeyExists: If key exists and won't be overwritten explicitly.
intelmq.lib.exceptions.InvalidKey: if key is invalid.
intelmq.lib.exceptions.InvalidArgument: if ignore is not list or tuple.
intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and
raise_failure is True.
"""
overwrite = force or overwrite
if force:
warnings.warn(
'The force-argument is deprecated by overwrite and will be removed in'
'1.0.', DeprecationWarning)
if not overwrite and key in self:
raise exceptions.KeyExists(key)
if value is None or value in ["", "-", "N/A"]:
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
if ignore:
warnings.warn('The ignore-argument will be removed in 1.0.',
DeprecationWarning)
try:
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize and not key == '__type':
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
if raise_failure:
raise exceptions.InvalidValue(key, old_value)
else:
return False
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
if raise_failure:
raise exceptions.InvalidValue(key,
value,
reason=valid_value[1])
else:
return False
super(Message, self).__setitem__(key, value)
return True
def add(self, key: str, value: str, sanitize: bool=True,
overwrite: Optional[bool]=None, ignore: Sequence=(),
raise_failure: bool=True) -> bool:
"""
Add a value for the key (after sanitation).
Parameters:
key: Key as defined in the harmonization
value: A valid value as defined in the harmonization
If the value is None or in _IGNORED_VALUES the value will be ignored.
If the value is ignored, the key exists and overwrite is True, the key
is deleted.
sanitize: Sanitation of harmonization type will be called before validation
(default: True)
overwrite: Overwrite an existing value if it already exists (default: None)
If True, overwrite an existing value
If False, do not overwrite an existing value
If None, raise intelmq.exceptions.KeyExists for an existing value
raise_failure: If a intelmq.lib.exceptions.InvalidValue should be raised for
invalid values (default: True). If false, the return parameter will be
False in case of invalid values.
Returns:
* True if the value has been added.
* False if the value is invalid and raise_failure is False or the value existed
and has not been overwritten.
Raises:
intelmq.lib.exceptions.KeyExists: If key exists and won't be overwritten explicitly.
intelmq.lib.exceptions.InvalidKey: if key is invalid.
intelmq.lib.exceptions.InvalidArgument: if ignore is not list or tuple.
intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and
raise_failure is True.
"""
if overwrite is None and key in self:
raise exceptions.KeyExists(key)
if overwrite is False and key in self:
return False
if value is None or value in self._IGNORED_VALUES:
if overwrite and key in self:
del self[key]
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
try:
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize and not key == '__type':
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
if raise_failure:
raise exceptions.InvalidValue(key, old_value)
else:
return False
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
if raise_failure:
raise exceptions.InvalidValue(key, value, reason=valid_value[1])
else:
return False
class_name, subitem = self.__get_type_config(key)
if class_name and class_name['type'] == 'JSONDict' and not subitem:
# for backwards compatibility allow setting the extra field as string
for extrakey, extravalue in json.loads(value).items():
if hasattr(extravalue, '__len__'):
if not len(extravalue): # ignore empty values
continue
if extravalue in self._IGNORED_VALUES:
continue
super(Message, self).__setitem__('%s.%s' % (key, extrakey),
extravalue)
else:
super(Message, self).__setitem__(key, value)
return True
