def __init__(self,
argv=None,
base_dir=None,
log_dir=None,
results_dir=None,
temp_dir=None,
plugin_dir_list=None,
archive_base=None,
config_file=None,
dispatch_rules=None,
useragent=None,
plugin_options=None,
log_level=None,
log_maxbytes=None,
log_backup_count=None,
default_connector=None,
default_source=None,
filename_suffix=None,
max_recursion=None,
max_queue=None,
source_base_tuple=None,
url_prefix_tuple=None,
log_syntax=None,
sentry_url=None,
sentry_ignore_list=None,
default_tlp=None):
"""
Initialize a stoQ class
:param list argv: sys.argv or list of command line arguments
:param str base_dir: Base directory that is the root for all paths
:param str log_dir: Directory to save log to
:param str results_dir: Directory to save results to
:param str temp_dir: Default temporary working directory
:param list plugin_dir_list: Directories to search for plugins in
:param str archive_base: Directory to save archived files to
:param str config_file: stoQ configuration file to use for settings
:param str dispatch_rules: Path to rules used for dispatching
:param str useragent: Useragent to use when making HTTP queries
:param dict plugin_options: Options to be passed to the plugins in lieu of command line arguments
:param str log_level: Log level for stoQ and all loaded plugins
:param int log_maxbytes: Maximum log file size in bytes
:param int log_backup_count: Maximum amount of log files to retain
:param str default_connector: Default connector plugin to use for output
:param str default_source: Default source plugin to use for ingesting
:param str filename_suffix: The filename suffix to use when saving files without a filename
:param int max_recursion: Maximum recursion level when dispatching payloads
:param int max_queue: When using multiprocessing, maximum amount of messages permitted in queue
:param tuple source_base_tuple: Base directories permitted to read from when ingesting
:param tuple url_prefix_tuple: Permitted URL prefixes
:param str log_syntax: Defines the format for log files
:param list sentry_ignore_list: Exceptions to ignore when sending to sentry
:param str default_tlp: Default TLP level set for all results
"""
# If Stoq is instantiated from a command line script, such as
# `stoq`, we will parse the command line parameters. If not,
# we will set the command line parameters to an empty list so we
# can still have our required variables set without making spaghetti
# code
self.argv = argv if argv else ['']
# Default to the base directory as the working directory, otherwise
# it will be set to the value passed at instantiation. This value
# will determine the default values for all paths required by stoQ,
# unless they are overridden within the configuration file.
if not base_dir:
self.base_dir = os.path.realpath(os.path.dirname(self.argv[0]))
else:
self.base_dir = os.path.realpath(base_dir)
self.config_file = config_file if config_file else os.path.join(
self.base_dir, "stoq.cfg")
if os.path.exists(self.config_file):
self.load_config()
# Make sure the stoQ objects we require exist.
# Setup our basic directory structure. This is overwritten
# if we have anything set in our configuration file, unless
self.worker = None
self.log_dir = self._set_opt('log_dir', log_dir,
os.path.join(self.base_dir, "logs"))
self.results_dir = self._set_opt(
'results_dir', results_dir, os.path.join(self.base_dir, "results"))
self.temp_dir = self._set_opt('temp_dir', temp_dir,
os.path.join(self.base_dir, "temp"))
self.plugin_dir_list = self._set_opt(
'plugin_dir_list', plugin_dir_list,
[os.path.join(self.base_dir, "plugins")])
self.archive_base = self._set_opt(
'archive_base', archive_base, os.path.join(self.base_dir,
"archive"))
self.dispatch_rules = self._set_opt(
'dispatch_rules', dispatch_rules,
os.path.join(self.base_dir, 'dispatcher.yar'))
self.useragent = self._set_opt(
'useragent', useragent,
'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1)')
self.plugin_options = self._set_opt('plugin_options', plugin_options,
{})
self.log_level = self._set_opt('log_level', log_level, 'info')
self.log_maxbytes = self._set_opt('log_maxbytes', log_maxbytes,
1500000)
self.log_backup_count = self._set_opt('log_backup_count',
log_backup_count, 5)
self.default_connector = self._set_opt('default_connector',
default_connector, 'stdout')
self.default_source = self._set_opt('default_source', default_source,
'filedir')
self.filename_suffix = self._set_opt('filename_suffix',
filename_suffix, 'stoq')
self.max_recursion = self._set_opt('max_recursion', max_recursion, 3)
self.max_queue = self._set_opt('max_queue', max_queue, 100)
self.source_base_tuple = self._set_opt('source_base_tuple',
source_base_tuple,
(self.base_dir, ))
self.url_prefix_tuple = self._set_opt('url_prefix_tuple',
url_prefix_tuple,
('http://', 'https://'))
self.log_syntax = self._set_opt('log_syntax', log_syntax, 'text')
self.sentry_url = self._set_opt('sentry_url', sentry_url)
self.sentry_ignore_list = self._set_opt('sentry_ignore_list',
sentry_ignore_list, [])
self.default_tlp = self._set_opt('default_tlp', default_tlp, 'white')
self.tlps = {'red': 0, 'amber': 1, 'green': 2, 'white': 3}
self.logger_init()
# Ensure our plugin manager is initiated
StoqPluginManager.__init__(self)
def __init__(self, argv=None, base_dir=None):
"""
Initialize a stoQ class
:param list argv: sys.argv or list of command line arguments
:param str base_dir: Base directory that is the root for all paths
"""
# If Stoq is instantiated from a command line script, such as
# stoq-cli.py, we will parse the command line parameters. If not,
# we will set the command line parameters to an empty list so we
# can still have our required variables set without making spaghetti
# code.
if argv:
self.argv = argv
else:
self.argv = ['']
# Default to the base directory as the working directory, otherwise
# it will be set to the value passed at instantiation. This value
# will determine the default values for all paths required by stoQ,
# unless they are overridden within the configuration file.
if not base_dir:
self.base_dir = os.path.realpath(os.path.dirname(self.argv[0]))
else:
self.base_dir = os.path.realpath(base_dir)
# Make sure the stoQ objects we require exist.
# Setup our basic directory structure. This is overwritten
# if we have anything set in our configuration file.
self.log_dir = os.path.join(self.base_dir, "logs")
self.results_dir = os.path.join(self.base_dir, "results")
self.temp_dir = os.path.join(self.base_dir, "temp")
self.plugin_dir = os.path.join(self.base_dir, "plugins")
self.archive_base = os.path.join(self.base_dir, "archive")
self.config_file = os.path.join(self.base_dir, "stoq.cfg")
self.dispatch_rules = os.path.join(self.base_dir, 'dispatcher.yar')
# What should be our default user agent when retrieving urls?
self.useragent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1)"
self.worker = None
# Default logging options
# Valid options: DEBUG, INFO, WARNING, ERROR, CRITICAL
self.log_level = "INFO"
self.log_maxbytes = 1500000
self.log_backup_count = 5
# Default connector plugin to be used for output
self.default_connector = "stdout"
# Default source plugin to be used for input
self.default_source = "filedir"
# The default suffix to append to a filename if
# a filename is not provided.
self.filename_suffix = "stoq"
# Define the default maximum recursion depth for the dispatcher
self.max_recursion = 3
# Maximum queue size for multiprocessing support
self.max_queue = 100
# tuple() to match the root directory of where files can be ingested
# from. Need for get_file().
self.source_base_tuple = (self.base_dir)
# Define what URL prefixes we accept
self.url_prefix_tuple = ('http://', 'https://')
# Load the configuration file, if it exists
if os.path.exists(self.config_file):
self.load_config()
# Initialize the logger
self.logger_init()
# Ensure our plugin manager is initiated
StoqPluginManager.__init__(self)
def __init__(self, argv=None, base_dir=None):
"""
Initialize a stoQ class
:param list argv: sys.argv or list of command line arguments
:param str base_dir: Base directory that is the root for all paths
"""
# If Stoq is instantiated from a command line script, such as
# stoq-cli.py, we will parse the command line parameters. If not,
# we will set the command line parameters to an empty list so we
# can still have our required variables set without making spaghetti
# code.
if argv:
self.argv = argv
else:
self.argv = ['']
# Default to the base directory as the working directory, otherwise
# it will be set to the value passed at instantiation. This value
# will determine the default values for all paths required by stoQ,
# unless they are overridden within the configuration file.
if not base_dir:
self.base_dir = os.path.realpath(os.path.dirname(self.argv[0]))
else:
self.base_dir = os.path.realpath(base_dir)
self.version = __version__
# Make sure the stoQ objects we require exist.
# Setup our basic directory structure. This is overwritten
# if we have anything set in our configuration file.
self.log_dir = os.path.join(self.base_dir, "logs")
self.results_dir = os.path.join(self.base_dir, "results")
self.temp_dir = os.path.join(self.base_dir, "temp")
self.plugin_dir = os.path.join(self.base_dir, "plugins")
self.archive_base = os.path.join(self.base_dir, "archive")
self.config_file = os.path.join(self.base_dir, "stoq.cfg")
self.dispatch_rules = os.path.join(self.base_dir, 'dispatcher.yar')
# What should be our default user agent when retrieving urls?
self.useragent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1)"
self.worker = None
# Default logging options
# Valid options: DEBUG, INFO, WARN, ERROR, CRITICAL
self.log_level = "INFO"
self.log_maxbytes = 1500000
self.log_backup_count = 5
# Default connector plugin to be used for output
self.default_connector = "stdout"
# Default source plugin to be used for input
self.default_source = "filedir"
# The default suffix to append to a filename if
# a filename is not provided.
self.filename_suffix = "stoq"
# Define the default maximum recursion depth for the dispatcher
self.max_recursion = 3
# Maximum queue size for multiprocessing support
self.max_queue = 100
# tuple() to match the root directory of where files can be ingested
# from. Need for get_file().
self.source_base_tuple = (self.base_dir)
# Define what URL prefixes we accept
self.url_prefix_tuple = ('http://', 'https://')
# Load the configuration file, if it exists
if os.path.exists(self.config_file):
self.load_config()
# Initialize the logger
self.logger_init()
# Default TLP for each payload processed
self.default_tlp = "white"
self.tlps = {'red': 0,
'amber': 1,
'green': 2,
'white': 3
}
# Ensure our plugin manager is initiated
StoqPluginManager.__init__(self)