def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
ioc_keys: List[str] = [
'received',
'x-orig-ip',
'x-originating-ip',
'x-remote-ip',
'x-sender-ip',
'body',
'body_html',
]
self.omit_body = config.getboolean('options',
'omit_body',
fallback=False)
self.always_dispatch = config.getlist('options',
'always_dispatch',
fallback=[])
self.archive_attachments = config.getboolean('options',
'archive_attachments',
fallback=True)
self.extract_iocs = config.getboolean('options',
'extract_iocs',
fallback=False)
self.ioc_keys = config.getlist('options',
'ioc_keys',
fallback=ioc_keys)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.required_workers.add('mimetype')
self.timeout = config.getint('options', 'timeout', fallback=45)
self.passwords = config.getlist(
'options', 'passwords', fallback=['-', 'infected', 'password']
)
self.maximum_size = config.getint(
'options', 'maximum_size', fallback=50_000_000
)
self.always_dispatch = config.getlist('options', 'always_dispatch', fallback=[])
self.archive_extracted = config.getboolean(
'options', 'archive_extracted', fallback=True
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.required_workers.add('mimetype')
self.timeout = config.getint('options', 'timeout', fallback=45)
self.passwords = config.getlist(
'options', 'passwords', fallback=['-', 'infected', 'password']
)
self.maximum_size = config.getint(
'options', 'maximum_size', fallback=50_000_000
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.producer = None
self.servers = config.getlist('options', 'servers', fallback=['127.0.0.1:9092'])
self.group = config.get('options', 'group', fallback='stoq')
self.topic = config.get('options', 'topic', fallback="stoq")
self.publish_archive = config.getboolean(
'options', 'publish_archive', fallback=True
)
self.retries = config.getint('options', 'retries', fallback=5)
self.session_timeout_ms = config.getint(
'options', 'session_timeout_ms', fallback=15000
)
self.heartbeat_interval_ms = config.getint(
'options', 'heartbeat_interval_ms', fallback=5000
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
filename = getframeinfo(currentframe()).filename
parent = Path(filename).resolve().parent
self.bin = config.get('options', 'bin', fallback='trid')
self.skip_warnings = config.getlist(
'options',
'skip_warnings',
fallback=['file seems to be plain text/ASCII'])
self.trid_defs = config.get('options',
'trid_defs',
fallback='triddefs.trd')
if not os.path.isabs(self.trid_defs) and self.trid_defs:
self.trid_defs = os.path.join(parent, self.trid_defs)
if not os.path.isfile(self.trid_defs):
raise StoqPluginException(
f'TrID definitions do not exist at {self.trid_defs}')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.whitelist_file = config.getlist(
'options', 'whitelist_file', fallback=['whitelist.txt']
)
self.iana_url = config.get(
'options',
'iana_url',
fallback='https://data.iana.org/TLD/tlds-alpha-by-domain.txt',
)
self.iana_tld_file = config.get(
'options', 'iana_tld_file', fallback='tlds-alpha-by-domain.txt'
)
iana_tlds = self._get_iana_tlds()
# Helper regexes
self.helpers: Dict = {}
self.helpers[
'dot'
] = r"(?:\.|\[\.\]|\<\.\>|\{\.\}|\(\.\)|\<DOT\>|\[DOT\]|\{DOT\}|\(DOT\))"
self.helpers[
'at'
] = r"(?:@|\[@\]|\<@\>|\{@\}|\(@\)|\<AT\>|\[AT\]|\{AT\}|\(AT\))"
self.helpers['http'] = r"\b(?:H(?:XX|TT)P|MEOW):\/\/"
self.helpers['https'] = r"\b(?:H(?:XX|TT)PS|MEOWS):\/\/"
self.helpers['tld'] = self.helpers['dot'] + r"(?:%s)\b" % iana_tlds
self.helpers['host'] = r"\b(?:[A-Z0-9\-]+%s){0,4}" % self.helpers['dot']
self.helpers['domain'] = r"[A-Z0-9\-]{2,50}" + self.helpers['tld']
self.helpers['fqdn'] = "{0}{1}".format(
self.helpers['host'], self.helpers['domain']
)
# Simple normalizers, post-processing
# key=regex_name, value=replacement value
# re.sub(regex_name, replacement)
self.normalizers = {
'dot': '.',
'at': '@',
'http': 'http://',
'https': 'https://',
'tld': lambda m: m.group(0).lower(),
'domain': lambda m: m.group(0).lower(),
'fqdn': lambda m: m.group(0).lower(),
}
# Data-type regexes
self.ioctypes: Dict = {}
self.ioctypes['md5'] = r"\b[A-F0-9]{32}\b"
self.ioctypes['sha1'] = r"\b[A-F0-9]{40}\b"
self.ioctypes['sha256'] = r"\b[A-F0-9]{64}\b"
self.ioctypes['sha512'] = r"\b[A-F0-9]{128}\b"
self.ioctypes['ipv4'] = (
r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)%s){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
% self.helpers['dot']
)
self.ioctypes[
'ipv6'
] = r"(?:(?:(?:\b|::)(?:(?:[\dA-F]{1,4}(?::|::)){1,7})(?:[\dA-F]{1,4}))(?:(?:(?:\.\d{1,3})?){3})(?:::|\b))|(?:[\dA-F]{1,4}::)|(?:::[\dA-F]{1,4}(?:(?:(?:\.\d{1,3})?){3}))"
self.ioctypes['mac_address'] = r"\b(?i)(?:[0-9A-F]{2}[:-]){5}(?:[0-9A-F]{2})\b"
self.ioctypes['email'] = "{0}{1}{2}".format(
r"\b[A-Z0-9\.\_\%\+\-]+", self.helpers['at'], self.helpers['fqdn']
)
self.ioctypes['domain'] = self.helpers['fqdn']
self.ioctypes['url'] = "(?:{0}|{1})(?:{2}|{3}|{4}){5}".format(
self.helpers['http'],
self.helpers['https'],
self.helpers['fqdn'],
self.ioctypes['ipv4'],
self.ioctypes['ipv6'],
r"(?:[\:\/][A-Z0-9\/\:\+\%\.\_\-\=\~\&\\#\?]*){0,1}",
)
# Compile regexes for faster repeat usage
self.compiled_re: Dict = {}
self.whitelist_patterns: Dict[str, Set] = {}
for ioc in self.ioctypes:
self.whitelist_patterns[ioc] = set()
self.compiled_re[ioc] = re.compile(self.ioctypes[ioc], re.IGNORECASE)
self._load_whitelist()
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.elements = config.getlist('options', 'elements', fallback=['chunk'])