def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.source_dir = config.get('options', 'source_dir', fallback=None)
self.recursive = config.getboolean('options',
'recursive',
fallback=False)
self.results_dir = config.get('options',
'results_dir',
fallback=os.path.join(
os.getcwd(), 'results'))
self.date_mode = config.getboolean('options',
'date_mode',
fallback=False)
self.date_format = config.get('options',
'date_format',
fallback='%Y/%m/%d')
self.compactly = config.getboolean('options',
'compactly',
fallback=True)
self.archive_dir = config.get('options',
'archive_dir',
fallback=os.path.join(
os.getcwd(), 'archive'))
self.use_sha = config.getboolean('options', 'use_sha', fallback=True)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.dispatch_rules = None
self.worker_rules = None
filename = getframeinfo(currentframe()).filename # type: ignore
parent = Path(filename).resolve().parent
self.timeout = config.getint('options', 'timeout', fallback=60)
self.strings_limit = config.getint('options',
'strings_limit',
fallback=None)
self.xor_first_match = config.getboolean('options',
'xor_first_match',
fallback=True)
dispatch_ruleset = config.get('options',
'dispatch_rules',
fallback='rules/dispatcher.yar')
if dispatch_ruleset:
if not os.path.isabs(dispatch_ruleset):
dispatch_ruleset = os.path.join(parent, dispatch_ruleset)
self.dispatch_rules = self._compile_rules(dispatch_ruleset)
worker_ruleset = config.get('options',
'worker_rules',
fallback='rules/stoq.yar')
if worker_ruleset:
if not os.path.isabs(worker_ruleset):
worker_ruleset = os.path.join(parent, worker_ruleset)
self.worker_rules = self._compile_rules(worker_ruleset)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.mongo_client = None
self.mongodb_uri = config.get('options', 'mongodb_uri', fallback=None)
self.mongodb_collection = config.get(
'options', 'mongodb_collection', fallback='stoq'
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.client = None
self.access_key = config.get('options', 'access_key', fallback=None)
self.secret_key = config.get('options', 'secret_key', fallback=None)
self.archive_bucket = config.get('options', 'archive_bucket', fallback=None)
self.connector_bucket = config.get('options', 'connector_bucket', fallback=None)
self.use_sha = config.getboolean('archiver', 'use_sha', fallback=True)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
filename = getframeinfo(currentframe()).filename # type: ignore
parent = Path(filename).resolve().parent
self.terms = config.get('options', 'terms', fallback='terms.txt')
if not os.path.isabs(self.terms):
self.terms = os.path.join(parent, self.terms)
self.bin = config.get('options', 'bin_path', fallback='xorsearch')
def __init__(self, config: StoqConfigParser) -> None:
self.config = config
self.plugin_name = config.get('Core', 'Name')
self.__author__ = config.get('Documentation', 'Author', fallback='')
self.__version__ = config.get('Documentation', 'Version', fallback='')
self.__website__ = config.get('Documentation', 'Website', fallback='')
self.__description__ = config.get('Documentation',
'Description',
fallback='')
self.log = logging.getLogger(f'stoq.{self.plugin_name}')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.apikey = config.get('options', 'apikey', fallback=None)
self.time_since = config.get('options', 'time_since', fallback='1m')
self.download = config.getboolean('options',
'download',
fallback=False)
if not self.apikey:
raise StoqPluginException('VTMIS API Key does not exist')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.publish_archive = config.getboolean(
'options', 'publish_archive', fallback=True
)
self.redis_host = config.get('options', 'redis_host', fallback='127.0.0.1')
self.redis_port = config.getint('options', 'redis_port', fallback=6379)
self.max_connections = config.getint('options', 'max_connections', fallback=15)
self.redis_queue = config.get('options', 'redis_queue', fallback='stoq')
self._connect()
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.opswat_url = config.get('options', 'opswat_url', fallback=None)
if not self.opswat_url:
raise StoqPluginException('MetaDefender URL was not provided')
self.apikey = config.get('options', 'apikey', fallback=None)
if not self.apikey:
raise StoqPluginException('MetaDefender API Key was not provided')
self.delay = config.getint('options', 'delay', fallback=10)
self.max_attempts = config.getint('options',
'max_attempts',
fallback=10)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.workspaceid = config.get('options', 'workspaceid', fallback=None)
if not self.workspaceid:
raise StoqPluginException('workspaceid has not been defined')
self.workspacekey = config.get('options',
'workspacekey',
fallback=None)
if not self.workspacekey:
raise StoqPluginException('workspacekey has not been defined')
self.logtype = config.get('options', 'logtype', fallback='stoQ')
self.uri = f'https://{self.workspaceid}.ods.opinsights.azure.com{self.API_RESOURCE}?api-version={self.API_VERSION}'
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.conn_str = config.get('options', 'conn_str', fallback=None)
if not self.conn_str:
raise StoqPluginException('conn_str has not been defined')
self.results_container = config.get('options',
'results_container',
fallback='stoq-results')
self.archive_container = config.get('options',
'archive_container',
fallback='stoq-archive')
self.use_sha = config.getboolean('options', 'use_sha', fallback=True)
self.use_datetime = config.getboolean('options',
'use_datetime',
fallback=False)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.es = None
self.es_host = config.get('options', 'es_host', fallback=None)
self.es_options = json.loads(
config.get('options', 'es_options', fallback='{}'))
self.es_timeout = config.getint('options', 'es_timeout', fallback=60)
self.es_retry = config.getboolean('options', 'es_retry', fallback=True)
self.es_max_retries = config.getint('options',
'es_max_retries',
fallback=10)
self.es_index = config.get('options', 'es_index', fallback='stoq')
self.index_by_month = config.getboolean('options',
'index_by_month',
fallback=True)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.publish_client = None
self.ingest_client = None
self.project_id = config.get('options', 'project_id')
self.max_messages = config.getint('options',
'max_messages',
fallback=10)
self.publish_archive = config.getboolean('options',
'publish_archive',
fallback=True)
self.topic = config.get('options', 'topic', fallback='stoq')
self.subscription = config.get('options',
'subscription',
fallback='stoq')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.source_dir = config.get('options', 'source_dir', fallback=None)
if not self.source_dir or not os.path.exists(self.source_dir):
raise StoqPluginException(
f"Source directory not defined or doesn't exist: '{self.source_dir}'"
)
self.source_dir = os.path.abspath(self.source_dir)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
filename = getframeinfo(currentframe()).filename
parent = Path(filename).resolve().parent
self.template = config.get('options', 'template', fallback='stoq.tpl')
if self.template:
self.template = os.path.abspath(os.path.join(
parent, self.template))
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
filename = getframeinfo(currentframe()).filename
parent = Path(filename).resolve().parent
self.bin = config.get('options', 'bin', fallback='trid')
self.skip_warnings = config.getlist(
'options',
'skip_warnings',
fallback=['file seems to be plain text/ASCII'])
self.trid_defs = config.get('options',
'trid_defs',
fallback='triddefs.trd')
if not os.path.isabs(self.trid_defs) and self.trid_defs:
self.trid_defs = os.path.join(parent, self.trid_defs)
if not os.path.isfile(self.trid_defs):
raise StoqPluginException(
f'TrID definitions do not exist at {self.trid_defs}')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.sandbox_url = config.get('options', 'sandbox_url', fallback=None)
if not self.sandbox_url:
raise StoqPluginException("Falcon Sandbox URL was not provided")
self.apikey = config.get('options', 'apikey', fallback=None)
if not self.apikey:
raise StoqPluginException("Falcon Sandbox API Key was not provided")
self.delay = config.getint('options', 'delay', fallback=30)
self.max_attempts = config.getint('options', 'max_attempts', fallback=10)
self.useragent = config.get('options', 'useragent', fallback='Falcon Sandbox')
# Available environments ID:
# 300: 'Linux (Ubuntu 16.04, 64 bit)',
# 200: 'Android Static Analysis’,
# 160: 'Windows 10 64 bit’,
# 110: 'Windows 7 64 bit’,
# 100: ‘Windows 7 32 bit’
self.environment_id = config.getint('options', 'environment_id', fallback=160)
self.wait_for_results = config.getboolean(
'options', 'wait_for_results', fallback=True
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.producer = None
self.servers = config.getlist('options', 'servers', fallback=['127.0.0.1:9092'])
self.group = config.get('options', 'group', fallback='stoq')
self.topic = config.get('options', 'topic', fallback="stoq")
self.publish_archive = config.getboolean(
'options', 'publish_archive', fallback=True
)
self.retries = config.getint('options', 'retries', fallback=5)
self.session_timeout_ms = config.getint(
'options', 'session_timeout_ms', fallback=15000
)
self.heartbeat_interval_ms = config.getint(
'options', 'heartbeat_interval_ms', fallback=5000
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.project_id = config.get('options', 'project_id', fallback=None)
if not self.project_id:
raise StoqPluginException('project_id has not been defined')
self.archive_bucket = config.get('options', 'archive_bucket', fallback='')
self.connector_bucket = config.get('options', 'connector_bucket', fallback='')
self.use_sha = config.getboolean('options', 'use_sha', fallback=True)
self.use_datetime = config.getboolean('options', 'use_datetime', fallback=False)
self.max_retries = config.getint('options', 'max_retries', fallback=5)
self.use_encryption = config.getboolean(
'options', 'use_encryption', fallback=False
)
if self.use_encryption:
self.crypto_id = config.get('options', 'crypto_id')
self.keyring_id = config.get('options', 'keyring_id')
self.location_id = config.get('options', 'location_id')
# Creates an API client for the KMS API.
self.kms_client = googleapiclient.discovery.build(
'cloudkms', 'v1', cache_discovery=False
)
self.kms_key = f'projects/{self.project_id}/locations/{self.location_id}/keyRings/{self.keyring_id}/cryptoKeys/{self.crypto_id}'
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.bin = config.get('options', 'bin', fallback='exiftool')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.tika_url = config.get(
'options', 'tika_url', fallback='http://localhost:9998/tika'
)
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.swf_headers = config.get('options',
'swf_headers',
fallback='SWF|CWS|FWS').encode()
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.apikey = config.get('options', 'apikey', fallback=None)
if not self.apikey:
raise StoqPluginException("VTMIS API Key does not exist")
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.whitelist_file = config.getlist(
'options', 'whitelist_file', fallback=['whitelist.txt']
)
self.iana_url = config.get(
'options',
'iana_url',
fallback='https://data.iana.org/TLD/tlds-alpha-by-domain.txt',
)
self.iana_tld_file = config.get(
'options', 'iana_tld_file', fallback='tlds-alpha-by-domain.txt'
)
iana_tlds = self._get_iana_tlds()
# Helper regexes
self.helpers: Dict = {}
self.helpers[
'dot'
] = r"(?:\.|\[\.\]|\<\.\>|\{\.\}|\(\.\)|\<DOT\>|\[DOT\]|\{DOT\}|\(DOT\))"
self.helpers[
'at'
] = r"(?:@|\[@\]|\<@\>|\{@\}|\(@\)|\<AT\>|\[AT\]|\{AT\}|\(AT\))"
self.helpers['http'] = r"\b(?:H(?:XX|TT)P|MEOW):\/\/"
self.helpers['https'] = r"\b(?:H(?:XX|TT)PS|MEOWS):\/\/"
self.helpers['tld'] = self.helpers['dot'] + r"(?:%s)\b" % iana_tlds
self.helpers['host'] = r"\b(?:[A-Z0-9\-]+%s){0,4}" % self.helpers['dot']
self.helpers['domain'] = r"[A-Z0-9\-]{2,50}" + self.helpers['tld']
self.helpers['fqdn'] = "{0}{1}".format(
self.helpers['host'], self.helpers['domain']
)
# Simple normalizers, post-processing
# key=regex_name, value=replacement value
# re.sub(regex_name, replacement)
self.normalizers = {
'dot': '.',
'at': '@',
'http': 'http://',
'https': 'https://',
'tld': lambda m: m.group(0).lower(),
'domain': lambda m: m.group(0).lower(),
'fqdn': lambda m: m.group(0).lower(),
}
# Data-type regexes
self.ioctypes: Dict = {}
self.ioctypes['md5'] = r"\b[A-F0-9]{32}\b"
self.ioctypes['sha1'] = r"\b[A-F0-9]{40}\b"
self.ioctypes['sha256'] = r"\b[A-F0-9]{64}\b"
self.ioctypes['sha512'] = r"\b[A-F0-9]{128}\b"
self.ioctypes['ipv4'] = (
r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)%s){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
% self.helpers['dot']
)
self.ioctypes[
'ipv6'
] = r"(?:(?:(?:\b|::)(?:(?:[\dA-F]{1,4}(?::|::)){1,7})(?:[\dA-F]{1,4}))(?:(?:(?:\.\d{1,3})?){3})(?:::|\b))|(?:[\dA-F]{1,4}::)|(?:::[\dA-F]{1,4}(?:(?:(?:\.\d{1,3})?){3}))"
self.ioctypes['mac_address'] = r"\b(?i)(?:[0-9A-F]{2}[:-]){5}(?:[0-9A-F]{2})\b"
self.ioctypes['email'] = "{0}{1}{2}".format(
r"\b[A-Z0-9\.\_\%\+\-]+", self.helpers['at'], self.helpers['fqdn']
)
self.ioctypes['domain'] = self.helpers['fqdn']
self.ioctypes['url'] = "(?:{0}|{1})(?:{2}|{3}|{4}){5}".format(
self.helpers['http'],
self.helpers['https'],
self.helpers['fqdn'],
self.ioctypes['ipv4'],
self.ioctypes['ipv6'],
r"(?:[\:\/][A-Z0-9\/\:\+\%\.\_\-\=\~\&\\#\?]*){0,1}",
)
# Compile regexes for faster repeat usage
self.compiled_re: Dict = {}
self.whitelist_patterns: Dict[str, Set] = {}
for ioc in self.ioctypes:
self.whitelist_patterns[ioc] = set()
self.compiled_re[ioc] = re.compile(self.ioctypes[ioc], re.IGNORECASE)
self._load_whitelist()
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.conn_str = config.get('options', 'conn_str', fallback=None)
if not self.conn_str:
raise StoqPluginException('conn_str has not been defined')
def __init__(self, config: StoqConfigParser) -> None:
super().__init__(config)
self.pe_headers = config.get('options',
'pe_headers',
fallback='\x4d\x5a|\x5a\x4d').encode()