def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) ioc_keys: List[str] = [ 'received', 'x-orig-ip', 'x-originating-ip', 'x-remote-ip', 'x-sender-ip', 'body', 'body_html', ] self.omit_body = config.getboolean('options', 'omit_body', fallback=False) self.always_dispatch = config.getlist('options', 'always_dispatch', fallback=[]) self.archive_attachments = config.getboolean('options', 'archive_attachments', fallback=True) self.extract_iocs = config.getboolean('options', 'extract_iocs', fallback=False) self.ioc_keys = config.getlist('options', 'ioc_keys', fallback=ioc_keys)
def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) self.required_workers.add('mimetype') self.timeout = config.getint('options', 'timeout', fallback=45) self.passwords = config.getlist( 'options', 'passwords', fallback=['-', 'infected', 'password'] ) self.maximum_size = config.getint( 'options', 'maximum_size', fallback=50_000_000 ) self.always_dispatch = config.getlist('options', 'always_dispatch', fallback=[]) self.archive_extracted = config.getboolean( 'options', 'archive_extracted', fallback=True )
def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) self.required_workers.add('mimetype') self.timeout = config.getint('options', 'timeout', fallback=45) self.passwords = config.getlist( 'options', 'passwords', fallback=['-', 'infected', 'password'] ) self.maximum_size = config.getint( 'options', 'maximum_size', fallback=50_000_000 )
def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) self.producer = None self.servers = config.getlist('options', 'servers', fallback=['127.0.0.1:9092']) self.group = config.get('options', 'group', fallback='stoq') self.topic = config.get('options', 'topic', fallback="stoq") self.publish_archive = config.getboolean( 'options', 'publish_archive', fallback=True ) self.retries = config.getint('options', 'retries', fallback=5) self.session_timeout_ms = config.getint( 'options', 'session_timeout_ms', fallback=15000 ) self.heartbeat_interval_ms = config.getint( 'options', 'heartbeat_interval_ms', fallback=5000 )
def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) filename = getframeinfo(currentframe()).filename parent = Path(filename).resolve().parent self.bin = config.get('options', 'bin', fallback='trid') self.skip_warnings = config.getlist( 'options', 'skip_warnings', fallback=['file seems to be plain text/ASCII']) self.trid_defs = config.get('options', 'trid_defs', fallback='triddefs.trd') if not os.path.isabs(self.trid_defs) and self.trid_defs: self.trid_defs = os.path.join(parent, self.trid_defs) if not os.path.isfile(self.trid_defs): raise StoqPluginException( f'TrID definitions do not exist at {self.trid_defs}')
def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) self.whitelist_file = config.getlist( 'options', 'whitelist_file', fallback=['whitelist.txt'] ) self.iana_url = config.get( 'options', 'iana_url', fallback='https://data.iana.org/TLD/tlds-alpha-by-domain.txt', ) self.iana_tld_file = config.get( 'options', 'iana_tld_file', fallback='tlds-alpha-by-domain.txt' ) iana_tlds = self._get_iana_tlds() # Helper regexes self.helpers: Dict = {} self.helpers[ 'dot' ] = r"(?:\.|\[\.\]|\<\.\>|\{\.\}|\(\.\)|\<DOT\>|\[DOT\]|\{DOT\}|\(DOT\))" self.helpers[ 'at' ] = r"(?:@|\[@\]|\<@\>|\{@\}|\(@\)|\<AT\>|\[AT\]|\{AT\}|\(AT\))" self.helpers['http'] = r"\b(?:H(?:XX|TT)P|MEOW):\/\/" self.helpers['https'] = r"\b(?:H(?:XX|TT)PS|MEOWS):\/\/" self.helpers['tld'] = self.helpers['dot'] + r"(?:%s)\b" % iana_tlds self.helpers['host'] = r"\b(?:[A-Z0-9\-]+%s){0,4}" % self.helpers['dot'] self.helpers['domain'] = r"[A-Z0-9\-]{2,50}" + self.helpers['tld'] self.helpers['fqdn'] = "{0}{1}".format( self.helpers['host'], self.helpers['domain'] ) # Simple normalizers, post-processing # key=regex_name, value=replacement value # re.sub(regex_name, replacement) self.normalizers = { 'dot': '.', 'at': '@', 'http': 'http://', 'https': 'https://', 'tld': lambda m: m.group(0).lower(), 'domain': lambda m: m.group(0).lower(), 'fqdn': lambda m: m.group(0).lower(), } # Data-type regexes self.ioctypes: Dict = {} self.ioctypes['md5'] = r"\b[A-F0-9]{32}\b" self.ioctypes['sha1'] = r"\b[A-F0-9]{40}\b" self.ioctypes['sha256'] = r"\b[A-F0-9]{64}\b" self.ioctypes['sha512'] = r"\b[A-F0-9]{128}\b" self.ioctypes['ipv4'] = ( r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)%s){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" % self.helpers['dot'] ) self.ioctypes[ 'ipv6' ] = r"(?:(?:(?:\b|::)(?:(?:[\dA-F]{1,4}(?::|::)){1,7})(?:[\dA-F]{1,4}))(?:(?:(?:\.\d{1,3})?){3})(?:::|\b))|(?:[\dA-F]{1,4}::)|(?:::[\dA-F]{1,4}(?:(?:(?:\.\d{1,3})?){3}))" self.ioctypes['mac_address'] = r"\b(?i)(?:[0-9A-F]{2}[:-]){5}(?:[0-9A-F]{2})\b" self.ioctypes['email'] = "{0}{1}{2}".format( r"\b[A-Z0-9\.\_\%\+\-]+", self.helpers['at'], self.helpers['fqdn'] ) self.ioctypes['domain'] = self.helpers['fqdn'] self.ioctypes['url'] = "(?:{0}|{1})(?:{2}|{3}|{4}){5}".format( self.helpers['http'], self.helpers['https'], self.helpers['fqdn'], self.ioctypes['ipv4'], self.ioctypes['ipv6'], r"(?:[\:\/][A-Z0-9\/\:\+\%\.\_\-\=\~\&\\#\?]*){0,1}", ) # Compile regexes for faster repeat usage self.compiled_re: Dict = {} self.whitelist_patterns: Dict[str, Set] = {} for ioc in self.ioctypes: self.whitelist_patterns[ioc] = set() self.compiled_re[ioc] = re.compile(self.ioctypes[ioc], re.IGNORECASE) self._load_whitelist()
def __init__(self, config: StoqConfigParser) -> None: super().__init__(config) self.elements = config.getlist('options', 'elements', fallback=['chunk'])