def set_queues(self, queues: Optional[str], queues_type: str):
"""
:param queues: For source queue, it's just string.
For destination queue, it can be one of the following:
None or list or dict (of strings or lists, one of the key should be '_default')
:param queues_type: "source" or "destination"
The method assures self.destination_queues are in the form of dict of lists. It doesn't assure there is a '_default' key.
"""
if queues_type == "source":
self.source_queue = queues
if queues is not None:
self.internal_queue = queues + "-internal"
else:
self.internal_queue = None
elif queues_type == "destination":
type_ = type(queues)
if type_ is list:
q = {"_default": queues}
elif type_ is str:
q = {"_default": queues.split()}
elif type_ is dict:
q = queues
for key, val in queues.items():
q[key] = val if type(val) is list else val.split()
else:
raise exceptions.InvalidArgument(
'queues', got=queues,
expected=["None", "list of strings", "dict (of strings or lists that should have the _default key)"])
self.destination_queues = q
else:
raise exceptions.InvalidArgument('queues_type', got=queues_type, expected=['source', 'destination'])
def add(self, key, value, sanitize=True, force=False, ignore=()):
if not force and key in self:
raise exceptions.KeyExists(key)
if value is None or value == "":
return
if value in ["-", "N/A"]:
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
try:
warnings.warn('The ignore-argument will be removed in 1.0.',
DeprecationWarning)
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize and not key == '__type':
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
raise exceptions.InvalidValue(key, old_value)
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
raise exceptions.InvalidValue(key, value, reason=valid_value[1])
super(Message, self).__setitem__(key, value)
def hash(self, *, filter_keys=frozenset(), filter_type="blacklist"):
"""Return a SHA256 hash of the message as a hexadecimal string.
The hash is computed over almost all key/value pairs. Depending on
filter_type parameter (blacklist or whitelist), the keys defined in
filter_keys_list parameter will be considered as the keys to ignore
or the only ones to consider. If given, the filter_keys_list
parameter should be a set.
'time.observation' will always be ignored.
"""
if filter_type not in ["whitelist", "blacklist"]:
raise exceptions.InvalidArgument('filter_type',
got=filter_type,
expected=['whitelist', 'blacklist'])
event_hash = hashlib.sha256()
for key, value in sorted(self.items()):
if "time.observation" == key:
continue
if filter_type == "whitelist" and key not in filter_keys:
continue
if filter_type == "blacklist" and key in filter_keys:
continue
event_hash.update(utils.encode(key))
event_hash.update(b"\xc0")
event_hash.update(utils.encode(repr(value)))
event_hash.update(b"\xc0")
return event_hash.hexdigest()
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
try:
self.harmonization_config = harm_config[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=list(
harm_config.keys()),
docs=HARMONIZATION_CONF_FILE)
else:
self.harmonization_config = harmonization[classname]
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
try:
self.add(key, value, sanitize=False)
except exceptions.InvalidValue:
self.add(key, value, sanitize=True)
def add(self, key, value, sanitize=False, force=False, ignore=()):
if not force and key in self:
raise exceptions.KeyExists(key)
if value is None or value == "":
return
for invalid_value in ["-", "N/A"]:
if value == invalid_value:
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
try:
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize:
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
raise exceptions.InvalidValue(key, old_value)
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
raise exceptions.InvalidValue(key, value, reason=valid_value[1])
super(Message, self).__setitem__(key, value)
def create(parameters: object,
logger: object,
direction: Optional[str] = None,
queues: Optional[Union[str, list, dict]] = None):
"""
parameters: Parameters object
direction: "source" or "destination", optional, needed for queues
queues: needs direction to be set, calls set_queues
"""
if direction not in [None, "source", "destination"]:
raise exceptions.InvalidArgument(
"direction", got=direction, expected=["destination", "source"])
if direction and hasattr(parameters, "%s_pipeline_broker" % direction):
broker = getattr(parameters,
"%s_pipeline_broker" % direction).title()
elif (getattr(parameters, "source_pipeline_broker", None) == getattr(
parameters, "destination_pipeline_broker", None) and getattr(
parameters, "source_pipeline_broker", None) is not None):
broker = getattr(parameters, "source_pipeline_broker").title()
else:
if hasattr(parameters, 'broker'):
broker = parameters.broker.title()
else:
broker = "Redis"
pipe = getattr(intelmq.lib.pipeline, broker)(parameters, logger)
if queues and not direction:
raise ValueError("Parameter 'direction' must be given when using "
"the queues parameter.")
elif queues:
pipe.set_queues(queues, direction)
return pipe
def from_dict(message: dict,
harmonization=None,
default_type: Optional[str] = None) -> dict:
"""
Takes dictionary Message object, returns instance of correct class.
Parameters:
message: the message which should be converted to a Message object
harmonization: a dictionary holding the used harmonization
default_type: If '__type' is not present in message, the given type will be used
See also:
MessageFactory.unserialize
MessageFactory.serialize
"""
if default_type and "__type" not in message:
message["__type"] = default_type
try:
class_reference = getattr(intelmq.lib.message, message["__type"])
except AttributeError:
raise exceptions.InvalidArgument('__type',
got=message["__type"],
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
del message["__type"]
return class_reference(message, auto=True, harmonization=harmonization)
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def __init__(self, message=(), auto=False, harmonization=None):
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
if classname == 'event' and self.harmonization_config['extra']['type'] == 'JSON':
warnings.warn("Assuming harmonization type 'JSONDict' for harmonization field 'extra'. "
"This assumption will be removed in version 2.0.", DeprecationWarning)
self.harmonization_config['extra']['type'] = 'JSONDict'
super(Message, self).__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def unserialize(raw_message):
"""
Takes JSON-encoded Message object, returns instance of correct class.
The class is determined by __type attribute.
"""
message = Message.unserialize(raw_message)
try:
#logger = utils.log("message log nee",log_level='DEBUG')
#logger.info(message)
#if "__type" in message:
#logger.info("OK EDVARD INFO")
class_reference = getattr(intelmq.lib.message, message["__type"])
#elif type in message:
# logger.info("NOT OK EDVARD INFO")
# class_reference = getattr(intelmq.lib.message, message["type"])
# else:
# logger.info("KLURVAAAAAAAAAA TADY BUDE CHYBA PITOMY INTELMQ")
# raise "Edvarde, sem to nesmi dospet"
except AttributeError:
logger.info("TOHLE JE TA SPATNA MESSAGE")
logger.info(message)
raise exceptions.InvalidArgument('__type',
got=message["__type"],
expected=list(harm_config.keys()),
docs=HARMONIZATION_CONF_FILE)
del message["__type"]
return class_reference(message, auto=True)
def read_sieve_file(filename, metamodel):
if not os.path.exists(filename):
raise exceptions.InvalidArgument('file', got=filename, expected='existing file')
try:
sieve = metamodel.model_from_file(filename)
return sieve
except TextXError as e:
raise ValueError('Could not parse sieve file %r, error in (%d, %d): %s' % (filename, e.line, e.col, str(e)))
def read_sieve_file(filename, metamodel):
if not os.path.exists(filename):
raise exceptions.InvalidArgument('file', got=filename, expected='existing file')
try:
sieve = metamodel.model_from_file(filename)
return sieve
except TextXError as e:
raise ValueError(f'Could not parse sieve file {filename!r}, error in ({e.line}, {e.col}): {e}')
def set_queues(self, queues, queues_type):
if queues_type == "source":
self.source_queue = str(queues)
self.internal_queue = str(queues) + "-internal"
elif queues_type == "destination":
if queues and type(queues) is not list:
queues = queues.split()
self.destination_queues = queues
else:
raise exceptions.InvalidArgument('queues_type', got=queues_type,
expected=['source',
'destination'])
def init(self):
# Test if path is a directory
if not os.path.isdir(self.parameters.path):
raise exceptions.InvalidArgument('path', got=self.parameters.path,
expected="directory")
if not self.parameters.postfix:
self.logger.warn("No file extension was set. The collector will"
" read all files in %s.", self.parameters.path)
if self.parameters.delete_file:
self.logger.error("This configuration would delete all files"
" in %s. I'm stopping now....",
self.parameters.path)
self.stop()
def __init__(self, message=()):
super(Message, self).__init__(message)
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
try:
self.harmonization_config = harm_config[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=list(harm_config.keys()),
docs=HARMONIZATION_CONF_FILE)
def from_dict(message):
"""
Takes dictionary Message object, returns instance of correct class.
The class is determined by __type attribute.
"""
try:
class_reference = getattr(intelmq.lib.message, message["__type"])
except AttributeError:
raise exceptions.InvalidArgument('__type',
got=message["__type"],
expected=list(harm_config.keys()),
docs=HARMONIZATION_CONF_FILE)
del message["__type"]
return class_reference(message, auto=True)
def unserialize(raw_message, harmonization=None):
"""
Takes JSON-encoded Message object, returns instance of correct class.
The class is determined by __type attribute.
"""
message = Message.unserialize(raw_message)
try:
class_reference = getattr(intelmq.lib.message, message["__type"])
except AttributeError:
raise exceptions.InvalidArgument('__type',
got=message["__type"],
expected=list(harm_config.keys()),
docs=HARMONIZATION_CONF_FILE)
del message["__type"]
return class_reference(message, auto=True, harmonization=harmonization)
def create(logger,
broker=None,
direction=None,
queues=None,
pipeline_args=None,
load_balance=False,
is_multithreaded=False):
"""
direction: "source" or "destination", optional, needed for queues
queues: needs direction to be set, calls set_queues
bot: Bot instance
"""
if pipeline_args is None:
pipeline_args = {}
if direction not in [None, "source", "destination"]:
raise exceptions.InvalidArgument(
"direction", got=direction, expected=["destination", "source"])
if 'load_balance' not in pipeline_args:
pipeline_args['load_balance'] = load_balance
if direction == 'source' and 'source_pipeline_broker' in pipeline_args:
broker = pipeline_args['source_pipeline_broker'].title()
if direction == 'destination' and 'destination_pipeline_broker' in pipeline_args:
broker = pipeline_args['destination_pipeline_broker'].title()
elif (pipeline_args.get('source_pipeline_broker', None)
== pipeline_args.get('destination_pipeline_broker', None) and
pipeline_args.get('source_pipeline_broker', None) is not None):
broker = pipeline_args['source_pipeline_broker'].title()
else:
if broker is not None:
broker = broker.title()
else:
broker = "Redis"
pipe = getattr(intelmq.lib.pipeline,
broker)(logger=logger,
pipeline_args=pipeline_args,
load_balance=load_balance,
is_multithreaded=is_multithreaded)
if queues and not direction:
raise ValueError("Parameter 'direction' must be given when using "
"the queues parameter.")
elif queues:
pipe.set_queues(queues, direction)
return pipe
def __init__(self,
message: Union[dict, tuple] = (),
auto: bool = False,
harmonization: dict = None) -> None:
try:
classname = message['__type'].lower()
del message['__type']
except (KeyError, TypeError):
classname = self.__class__.__name__.lower()
if harmonization is None:
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
try:
self.harmonization_config = harmonization[classname]
except KeyError:
raise exceptions.InvalidArgument('__type',
got=classname,
expected=VALID_MESSSAGE_TYPES,
docs=HARMONIZATION_CONF_FILE)
if (classname == 'event' and 'extra' in self.harmonization_config
and self.harmonization_config['extra']['type'] == 'JSON'):
warnings.warn(
"Assuming harmonization type 'JSONDict' for harmonization field 'extra'. "
"This assumption will be removed in version 3.0.",
DeprecationWarning)
self.harmonization_config['extra']['type'] = 'JSONDict'
for harm_key in self.harmonization_config.keys():
if not re.match('^[a-z_](.[a-z_0-9]+)*$',
harm_key) and harm_key != '__type':
raise exceptions.InvalidKey(
"Harmonization key %r is invalid." % harm_key)
super().__init__()
if isinstance(message, dict):
iterable = message.items()
elif isinstance(message, tuple):
iterable = message
else:
raise ValueError(
"Type %r of message can't be handled, must be dict or tuple.",
type(message))
for key, value in iterable:
if not self.add(key, value, sanitize=False, raise_failure=False):
self.add(key, value, sanitize=True)
def add(self,
key: str,
value: str,
sanitize: bool = True,
force: bool = False,
overwrite: bool = False,
ignore: Sequence = (),
raise_failure: bool = True) -> bool:
"""
Add a value for the key (after sanitation).
Parameters:
key: Key as defined in the harmonization
value: A valid value as defined in the harmonization
sanitize: Sanitation of harmonization type will be called before validation
(default: True)
force: Deprecated, use overwrite (default: False)
overwrite: Overwrite an existing value if it already exists (default: False)
ignore: List or tuple of values to ignore, deprecated (default: ())
raise_failure: If a intelmq.lib.exceptions.InvalidValue should be raised for
invalid values (default: True). If false, the return parameter will be
False in case of invalid values.
Returns:
* True if the value has been added.
* False if the value is invalid and raise_failure is False.
Raises:
intelmq.lib.exceptions.KeyExists: If key exists and won't be overwritten explicitly.
intelmq.lib.exceptions.InvalidKey: if key is invalid.
intelmq.lib.exceptions.InvalidArgument: if ignore is not list or tuple.
intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and
raise_failure is True.
"""
overwrite = force or overwrite
if force:
warnings.warn(
'The force-argument is deprecated by overwrite and will be removed in'
'1.0.', DeprecationWarning)
if not overwrite and key in self:
raise exceptions.KeyExists(key)
if value is None or value in ["", "-", "N/A"]:
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
if ignore:
warnings.warn('The ignore-argument will be removed in 1.0.',
DeprecationWarning)
try:
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize and not key == '__type':
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
if raise_failure:
raise exceptions.InvalidValue(key, old_value)
else:
return False
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
if raise_failure:
raise exceptions.InvalidValue(key,
value,
reason=valid_value[1])
else:
return False
super(Message, self).__setitem__(key, value)
return True
def add(self, key: str, value: str, sanitize: bool=True,
overwrite: Optional[bool]=None, ignore: Sequence=(),
raise_failure: bool=True) -> bool:
"""
Add a value for the key (after sanitation).
Parameters:
key: Key as defined in the harmonization
value: A valid value as defined in the harmonization
If the value is None or in _IGNORED_VALUES the value will be ignored.
If the value is ignored, the key exists and overwrite is True, the key
is deleted.
sanitize: Sanitation of harmonization type will be called before validation
(default: True)
overwrite: Overwrite an existing value if it already exists (default: None)
If True, overwrite an existing value
If False, do not overwrite an existing value
If None, raise intelmq.exceptions.KeyExists for an existing value
raise_failure: If a intelmq.lib.exceptions.InvalidValue should be raised for
invalid values (default: True). If false, the return parameter will be
False in case of invalid values.
Returns:
* True if the value has been added.
* False if the value is invalid and raise_failure is False or the value existed
and has not been overwritten.
Raises:
intelmq.lib.exceptions.KeyExists: If key exists and won't be overwritten explicitly.
intelmq.lib.exceptions.InvalidKey: if key is invalid.
intelmq.lib.exceptions.InvalidArgument: if ignore is not list or tuple.
intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and
raise_failure is True.
"""
if overwrite is None and key in self:
raise exceptions.KeyExists(key)
if overwrite is False and key in self:
return False
if value is None or value in self._IGNORED_VALUES:
if overwrite and key in self:
del self[key]
return
if not self.__is_valid_key(key):
raise exceptions.InvalidKey(key)
try:
if value in ignore:
return
except TypeError:
raise exceptions.InvalidArgument('ignore',
got=type(ignore),
expected='list or tuple')
if sanitize and not key == '__type':
old_value = value
value = self.__sanitize_value(key, value)
if value is None:
if raise_failure:
raise exceptions.InvalidValue(key, old_value)
else:
return False
valid_value = self.__is_valid_value(key, value)
if not valid_value[0]:
if raise_failure:
raise exceptions.InvalidValue(key, value, reason=valid_value[1])
else:
return False
class_name, subitem = self.__get_type_config(key)
if class_name and class_name['type'] == 'JSONDict' and not subitem:
# for backwards compatibility allow setting the extra field as string
for extrakey, extravalue in json.loads(value).items():
if hasattr(extravalue, '__len__'):
if not len(extravalue): # ignore empty values
continue
if extravalue in self._IGNORED_VALUES:
continue
super(Message, self).__setitem__('%s.%s' % (key, extrakey),
extravalue)
else:
super(Message, self).__setitem__(key, value)
return True