def add(self, key, value, sanitize=False, force=False, ignore=()): if not force and key in self: raise exceptions.KeyExists(key) if value is None or value == "": return for invalid_value in ["-", "N/A"]: if value == invalid_value: return if not self.__is_valid_key(key): raise exceptions.InvalidKey(key) try: if value in ignore: return except TypeError: raise exceptions.InvalidArgument('ignore', got=type(ignore), expected='list or tuple') if sanitize: old_value = value value = self.__sanitize_value(key, value) if value is None: raise exceptions.InvalidValue(key, old_value) valid_value = self.__is_valid_value(key, value) if not valid_value[0]: raise exceptions.InvalidValue(key, value, reason=valid_value[1]) super(Message, self).__setitem__(key, value)
def is_valid(self, key: str, value: str, sanitize: bool = True) -> bool: """ Checks if a value is valid for the key (after sanitation). Parameters: key: Key of the field value: Value of the field sanitize: Sanitation of harmonization type will be called before validation (default: True) Returns: True if the value is valid, otherwise False Raises: intelmq.lib.exceptions.InvalidKey: if given key is invalid. """ if not self.__is_valid_key(key): raise exceptions.InvalidKey(key) if value is None or value in ["", "-", "N/A"]: return False if sanitize: value = self.__sanitize_value(key, value) valid = self.__is_valid_value(key, value) if valid[0]: return True return False
def is_valid(self, key, value, sanitize=True): """ Checks if a value is valid for the key (after sanitation). Parameters ========== key : string value : string sanitize : boolean Sanitation of harmonization type will be called before validation (default: True) Returns ======= retval : boolean Raises ====== intelmq.lib.exceptions.InvalidKey: if key is invalid. """ if not self.__is_valid_key(key): raise exceptions.InvalidKey(key) if value is None or value in ["", "-", "N/A"]: return False if sanitize: value = self.__sanitize_value(key, value) valid = self.__is_valid_value(key, value) if valid[0]: return True return False
def __init__(self, message=(), auto=False, harmonization=None): try: classname = message['__type'].lower() del message['__type'] except (KeyError, TypeError): classname = self.__class__.__name__.lower() if harmonization is None: harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE) try: self.harmonization_config = harmonization[classname] except KeyError: raise exceptions.InvalidArgument('__type', got=classname, expected=VALID_MESSSAGE_TYPES, docs=HARMONIZATION_CONF_FILE) for harm_key in self.harmonization_config.keys(): if not re.match('^[a-z_](.[a-z_0-9]+)*$', harm_key) and harm_key != '__type': raise exceptions.InvalidKey( "Harmonization key %r is invalid." % harm_key) super(Message, self).__init__() if isinstance(message, dict): iterable = message.items() elif isinstance(message, tuple): iterable = message for key, value in iterable: if not self.add(key, value, sanitize=False, raise_failure=False): self.add(key, value, sanitize=True)
def add(self, key, value, sanitize=True, force=False, ignore=()): if not force and key in self: raise exceptions.KeyExists(key) if value is None or value == "": return if value in ["-", "N/A"]: return if not self.__is_valid_key(key): raise exceptions.InvalidKey(key) try: warnings.warn('The ignore-argument will be removed in 1.0.', DeprecationWarning) if value in ignore: return except TypeError: raise exceptions.InvalidArgument('ignore', got=type(ignore), expected='list or tuple') if sanitize and not key == '__type': old_value = value value = self.__sanitize_value(key, value) if value is None: raise exceptions.InvalidValue(key, old_value) valid_value = self.__is_valid_value(key, value) if not valid_value[0]: raise exceptions.InvalidValue(key, value, reason=valid_value[1]) super(Message, self).__setitem__(key, value)
def __init__(self, message: Union[dict, tuple] = (), auto: bool = False, harmonization: dict = None) -> None: try: classname = message['__type'].lower() del message['__type'] except (KeyError, TypeError): classname = self.__class__.__name__.lower() if harmonization is None: harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE) try: self.harmonization_config = harmonization[classname] except KeyError: raise exceptions.InvalidArgument('__type', got=classname, expected=VALID_MESSSAGE_TYPES, docs=HARMONIZATION_CONF_FILE) if (classname == 'event' and 'extra' in self.harmonization_config and self.harmonization_config['extra']['type'] == 'JSON'): warnings.warn( "Assuming harmonization type 'JSONDict' for harmonization field 'extra'. " "This assumption will be removed in version 3.0.", DeprecationWarning) self.harmonization_config['extra']['type'] = 'JSONDict' for harm_key in self.harmonization_config.keys(): if not re.match('^[a-z_](.[a-z_0-9]+)*$', harm_key) and harm_key != '__type': raise exceptions.InvalidKey( "Harmonization key %r is invalid." % harm_key) super().__init__() if isinstance(message, dict): iterable = message.items() elif isinstance(message, tuple): iterable = message else: raise ValueError( "Type %r of message can't be handled, must be dict or tuple.", type(message)) for key, value in iterable: if not self.add(key, value, sanitize=False, raise_failure=False): self.add(key, value, sanitize=True)
def add(self, key: str, value: str, sanitize: bool = True, force: bool = False, overwrite: bool = False, ignore: Sequence = (), raise_failure: bool = True) -> bool: """ Add a value for the key (after sanitation). Parameters: key: Key as defined in the harmonization value: A valid value as defined in the harmonization sanitize: Sanitation of harmonization type will be called before validation (default: True) force: Deprecated, use overwrite (default: False) overwrite: Overwrite an existing value if it already exists (default: False) ignore: List or tuple of values to ignore, deprecated (default: ()) raise_failure: If a intelmq.lib.exceptions.InvalidValue should be raised for invalid values (default: True). If false, the return parameter will be False in case of invalid values. Returns: * True if the value has been added. * False if the value is invalid and raise_failure is False. Raises: intelmq.lib.exceptions.KeyExists: If key exists and won't be overwritten explicitly. intelmq.lib.exceptions.InvalidKey: if key is invalid. intelmq.lib.exceptions.InvalidArgument: if ignore is not list or tuple. intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and raise_failure is True. """ overwrite = force or overwrite if force: warnings.warn( 'The force-argument is deprecated by overwrite and will be removed in' '1.0.', DeprecationWarning) if not overwrite and key in self: raise exceptions.KeyExists(key) if value is None or value in ["", "-", "N/A"]: return if not self.__is_valid_key(key): raise exceptions.InvalidKey(key) if ignore: warnings.warn('The ignore-argument will be removed in 1.0.', DeprecationWarning) try: if value in ignore: return except TypeError: raise exceptions.InvalidArgument('ignore', got=type(ignore), expected='list or tuple') if sanitize and not key == '__type': old_value = value value = self.__sanitize_value(key, value) if value is None: if raise_failure: raise exceptions.InvalidValue(key, old_value) else: return False valid_value = self.__is_valid_value(key, value) if not valid_value[0]: if raise_failure: raise exceptions.InvalidValue(key, value, reason=valid_value[1]) else: return False super(Message, self).__setitem__(key, value) return True
def add(self, key: str, value: str, sanitize: bool=True, overwrite: Optional[bool]=None, ignore: Sequence=(), raise_failure: bool=True) -> bool: """ Add a value for the key (after sanitation). Parameters: key: Key as defined in the harmonization value: A valid value as defined in the harmonization If the value is None or in _IGNORED_VALUES the value will be ignored. If the value is ignored, the key exists and overwrite is True, the key is deleted. sanitize: Sanitation of harmonization type will be called before validation (default: True) overwrite: Overwrite an existing value if it already exists (default: None) If True, overwrite an existing value If False, do not overwrite an existing value If None, raise intelmq.exceptions.KeyExists for an existing value raise_failure: If a intelmq.lib.exceptions.InvalidValue should be raised for invalid values (default: True). If false, the return parameter will be False in case of invalid values. Returns: * True if the value has been added. * False if the value is invalid and raise_failure is False or the value existed and has not been overwritten. Raises: intelmq.lib.exceptions.KeyExists: If key exists and won't be overwritten explicitly. intelmq.lib.exceptions.InvalidKey: if key is invalid. intelmq.lib.exceptions.InvalidArgument: if ignore is not list or tuple. intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and raise_failure is True. """ if overwrite is None and key in self: raise exceptions.KeyExists(key) if overwrite is False and key in self: return False if value is None or value in self._IGNORED_VALUES: if overwrite and key in self: del self[key] return if not self.__is_valid_key(key): raise exceptions.InvalidKey(key) try: if value in ignore: return except TypeError: raise exceptions.InvalidArgument('ignore', got=type(ignore), expected='list or tuple') if sanitize and not key == '__type': old_value = value value = self.__sanitize_value(key, value) if value is None: if raise_failure: raise exceptions.InvalidValue(key, old_value) else: return False valid_value = self.__is_valid_value(key, value) if not valid_value[0]: if raise_failure: raise exceptions.InvalidValue(key, value, reason=valid_value[1]) else: return False class_name, subitem = self.__get_type_config(key) if class_name and class_name['type'] == 'JSONDict' and not subitem: # for backwards compatibility allow setting the extra field as string for extrakey, extravalue in json.loads(value).items(): if hasattr(extravalue, '__len__'): if not len(extravalue): # ignore empty values continue if extravalue in self._IGNORED_VALUES: continue super(Message, self).__setitem__('%s.%s' % (key, extrakey), extravalue) else: super(Message, self).__setitem__(key, value) return True