def init(self):
if url_normalize is None:
raise MissingDependencyError("url-normalize")
url_version = pkg_resources.get_distribution("url-normalize").version
if tuple(int(v) for v in url_version.split('.')) < (
1, 4, 1) and self.default_scheme is not None:
raise ValueError(
"Parameter 'default_scheme' given but 'url-normalize' version %r does not support it. "
"Get at least version '1.4.1'." % url_version)
if get_tld is None:
raise MissingDependencyError("tld")
try:
update_tld_names()
except tld.exceptions.TldIOError:
self.logger.info("Could not update TLD names cache.")
if self.domain_whitelist != '':
self._domain_whitelist.extend(self.domain_whitelist.split(','))
if self.substitutions != '':
temp = self.substitutions.split(';')
if len(temp) % 2 != 0:
raise InvalidArgument(
'substitutions',
got=self.substitutions,
expected="even number of ; separated strings")
for i in range(int(len(temp) / 2)):
self._substitutions.append([temp[2 * i], temp[2 * i + 1]])
if not ClassificationType.is_valid(self.classification_type):
self.classification_type = 'unknown'
if self.default_scheme is not None:
self.url_kwargs = {'default_scheme': self.default_scheme}
else:
self.url_kwargs = {}
def init(self):
if requests is None:
raise MissingDependencyError("requests")
if twitter is None:
raise MissingDependencyError("twitter")
self.current_time_in_seconds = int(time.time())
self.target_timelines = []
if getattr(self.parameters, "target_timelines", '') != '':
self.target_timelines.extend(
self.parameters.target_timelines.split(','))
self.tweet_count = int(getattr(self.parameters, "tweet_count", 20))
self.follow_urls = []
if getattr(self.parameters, "follow_urls", '') != '':
self.follow_urls.extend(self.parameters.follow_urls.split(','))
self.include_rts = getattr(self.parameters, "include_rts", False)
self.exclude_replies = getattr(self.parameters, "exclude_replies",
False)
self.timelimit = int(
getattr(self.parameters, "timelimit", 24 * 60 * 60))
self.api = twitter.Api(
consumer_key=self.parameters.consumer_key,
consumer_secret=self.parameters.consumer_secret,
access_token_key=self.parameters.access_token_key,
access_token_secret=self.parameters.access_token_secret,
tweet_mode="extended")
self.set_request_parameters()
self.session = create_request_session_from_bot(self)
def init(self):
if stomp is None:
raise MissingDependencyError("stomp")
elif stomp.__version__ < (4, 1, 8):
raise MissingDependencyError("stomp",
version="4.1.8",
installed=stomp.__version__)
self.ssl_ca_cert = self.ssl_ca_certificate
self.ssl_cl_cert = self.ssl_client_certificate
self.ssl_cl_cert_key = self.ssl_client_certificate_key
# check if certificates exist
for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]:
if not os.path.isfile(f):
raise ValueError("Could not open file %r." % f)
_host = [(self.server, self.port)]
self.__conn = stomp.Connection(host_and_ports=_host,
use_ssl=True,
ssl_key_file=self.ssl_cl_cert_key,
ssl_cert_file=self.ssl_cl_cert,
ssl_ca_certs=self.ssl_ca_cert,
heartbeats=(self.heartbeat,
self.heartbeat))
self.__conn.set_listener(
'', StompListener(self, self.__conn, self.exchange))
connect_and_subscribe(self.__conn,
self.logger,
self.exchange,
start=stomp.__version__ < (4, 1, 20))
def init(self):
if pika is None:
raise MissingDependencyError("pika", version=">=1.0")
self.connection = None
self.channel = None
pika_version = tuple(int(x) for x in pika.__version__.split('.'))
if pika_version < (1, ):
raise MissingDependencyError("pika",
version=">=1.0",
installed=pika.__version__)
self.kwargs = {}
self.kwargs['heartbeat'] = self.connection_heartbeat
if self.username and self.password:
self.kwargs['credentials'] = pika.PlainCredentials(
self.username, self.password)
if self.use_ssl:
self.kwargs['ssl_options'] = pika.SSLOptions(
context=ssl.create_default_context(ssl.Purpose.CLIENT_AUTH))
self.connection_parameters = pika.ConnectionParameters(
host=self.connection_host,
port=self.connection_port,
virtual_host=self.connection_vhost,
connection_attempts=self.connection_attempts,
**self.kwargs)
self.connect_server()
def init(self):
if DxlClient is None:
raise MissingDependencyError('dxlclient')
if MarClient is None:
raise MissingDependencyError('dxlmarclient')
self.config = DxlClientConfig.create_dxl_config_from_file(
self.dxl_config_file)
def init(self):
if requests is None:
raise MissingDependencyError("requests")
self.set_request_parameters()
self.session = create_request_session(self)
self.use_gpg = getattr(self.parameters, "verify_pgp_signatures", False)
if self.use_gpg and gnupg is None:
raise MissingDependencyError("gnupg")
else:
self.logger.info('PGP signature verification is active.')
def init(self):
if requests is None:
raise MissingDependencyError("requests")
if base64 is None:
raise MissingDependencyError("base64")
self.esm = ESM(self.parameters.esm_ip, self.parameters.esm_user,
self.parameters.esm_password)
try:
self.esm.login()
self.esm.logout()
except Exception:
raise ValueError('Could not Login to ESM.')
def init(self):
if OTXv2 is None:
raise MissingDependencyError("OTXv2")
self.modified_pulses_only = False
if hasattr(self, 'modified_pulses_only'):
self.modified_pulses_only = self.modified_pulses_only
def init(self):
if kafka is None:
raise MissingDependencyError("kafka")
self.topic = []
if getattr(self.parameters, 'topic', '') != '':
self.topic = self.parameters.topic
self.bootstrap_servers = 'localhost:9092'
if getattr(self.parameters, 'bootstrap_servers', '') != '':
self.bootstrap_servers = self.parameters.bootstrap_servers
self.ssl_cafile = getattr(self.parameters, 'ssl_ca_certificate', None)
self.ssl_certfile = getattr(self.parameters, 'ssl_client_certificate',
None)
self.ssl_check_hostname = getattr(self.parameters,
'ssl_check_hostname', False)
self.logger.debug(
"Topic set to {}, bootstrap_servers set to {}".format(
self.topic, self.bootstrap_servers))
self.logger.debug(
"ssl_cafile set to {}, ssl_certfile set to {}, ssl_check_hostname set to {}"
.format(self.ssl_cafile, self.ssl_certfile,
self.ssl_check_hostname))
def init(self):
if requests is None:
raise MissingDependencyError("requests")
self.set_request_parameters()
self.session = create_request_session_from_bot(self)
def init(self):
if ExpandedPyMISP is None:
raise MissingDependencyError('pymisp', '>=2.4.117.3')
# Initialize MISP connection
self.misp = ExpandedPyMISP(self.misp_url, self.misp_key,
self.http_verify_cert)
def init(self):
if xmltodict is None:
raise MissingDependencyError("xmltodict")
self.set_request_parameters()
self.session = create_request_session_from_bot(self)
self.dns_name = getattr(self.parameters, "dns_name", None)
if self.dns_name is None:
raise ValueError('No dns name provided.')
self.request_duration = getattr(self.parameters, "request_duration",
None)
if self.request_duration is None:
raise ValueError('No request_duration provided.')
user = getattr(self.parameters, "http_username", None)
if user is None:
raise ValueError('No http_username provided.')
pw = getattr(self.parameters, "http_password", None)
if pw is None:
raise ValueError('No http_password provided.')
# create auth token
token = user + ":" + pw
message_bytes = token.encode('ascii')
base64_bytes = base64.b64encode(message_bytes)
base64_message = base64_bytes.decode('ascii')
self.http_header = {'Authorization': 'Basic ' + base64_message}
self.custom_auth_url = "https://" + self.dns_name + "/wsapis/v2.0.0/auth/login"
def init(self):
if stomp is None:
raise MissingDependencyError("stomp")
self.server = getattr(self.parameters, 'server', '127.0.0.1')
self.port = getattr(self.parameters, 'port', 61614)
self.exchange = getattr(self.parameters, 'exchange', '/exchange/_push')
self.heartbeat = getattr(self.parameters, 'heartbeat', 60000)
self.ssl_ca_cert = getattr(self.parameters, 'ssl_ca_certificate',
'ca.pem')
self.ssl_cl_cert = getattr(self.parameters, 'ssl_client_certificate',
'client.pem')
self.ssl_cl_cert_key = getattr(self.parameters,
'ssl_client_certificate_key',
'client.key')
self.http_verify_cert = getattr(self.parameters, 'http_verify_cert',
True)
# check if certificates exist
for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]:
if not os.path.isfile(f):
raise ValueError(
"Could not open SSL (certificate) file '%s'." % f)
_host = [(self.server, self.port)]
self.conn = stomp.Connection(host_and_ports=_host,
use_ssl=True,
ssl_key_file=self.ssl_cl_cert_key,
ssl_cert_file=self.ssl_cl_cert,
ssl_ca_certs=self.ssl_ca_cert,
heartbeats=(self.heartbeat,
self.heartbeat))
self.connect()
def init(self):
if validators is None:
raise MissingDependencyError('validators')
self.__supported_feeds = {
'StrangerealIntel/DailyIOC':
lambda logger: self.StrangerealIntelDailyIOC(logger)
}
def init(self):
if imbox is None:
raise MissingDependencyError("imbox")
if getattr(self.parameters, 'attach_unzip', None) and not self.extract_files:
self.extract_files = True
self.logger.warning("The parameter 'attach_unzip' is deprecated and will "
"be removed in version 4.0. Use 'extract_files' instead.")
def init(self):
if OTXv2 is None:
raise MissingDependencyError("OTXv2")
self.modified_pulses_only = False
if hasattr(self.parameters, 'modified_pulses_only'):
self.modified_pulses_only = self.parameters.modified_pulses_only
self.interval = getattr(self.parameters, 'interval', 24)
def init(self):
if cabby is None:
raise MissingDependencyError('cabby')
self.user = self.parameters.username
self.passwd = self.parameters.password
self.endpoint = self.parameters.endpoint
self.time_delta = int(self.parameters.time_delta)
self.collection = self.parameters.collection
def init(self):
if requests is None:
raise MissingDependencyError("requests")
self.set_request_parameters()
self.session = create_request_session(self)
self.__error_count = 0
def init(self):
if sleekxmpp is None:
raise MissingDependencyError("sleekxmpp")
# Retrieve Parameters from configuration
xmpp_user = getattr(self.parameters, "xmpp_user", None)
xmpp_server = getattr(self.parameters, "xmpp_server", None)
xmpp_password = getattr(self.parameters, "xmpp_password", None)
if None in (xmpp_user, xmpp_server, xmpp_password):
raise ValueError('No User / Password provided.')
else:
xmpp_login = xmpp_user + '@' + xmpp_server
self.muc = getattr(self.parameters, "use_muc", None)
xmpp_to_user = getattr(self.parameters, "xmpp_to_user", None)
xmpp_to_server = getattr(self.parameters, "xmpp_to_server", None)
xmpp_room = getattr(self.parameters, "xmpp_room",
None) if self.muc else None
xmpp_room_nick = getattr(self.parameters, "xmpp_room_nick",
None) if self.muc else None
xmpp_room_password = getattr(self.parameters, "xmpp_room_password",
None) if self.muc else None
ca_certs = getattr(self.parameters, "ca_certs", None)
# Be sure the receiver was set up
if not self.muc and None in (xmpp_to_user, xmpp_to_server):
raise ValueError('No receiver for direct messages provided.')
else:
self.xmpp_receiver = xmpp_to_user + '@' +\
xmpp_to_server
if self.muc and not xmpp_room:
raise ValueError('No room provided.')
else:
self.xmpp_receiver = xmpp_room
if self.muc:
if not xmpp_room_nick:
# create the room_nick from user and server
xmpp_room_nick = xmpp_login
self.xmpp = XMPPClient(xmpp_login, xmpp_password, xmpp_room,
xmpp_room_nick, xmpp_room_password, self.logger)
if ca_certs:
# Set CA-Certificates
self.xmpp.ca_certs = ca_certs
if self.xmpp.connect(reattempt=False):
self.xmpp.process()
# Add Handlers and register Plugins
self.xmpp.register_plugin('xep_0030') # Service Discovery
self.xmpp.register_plugin('xep_0045') # Multi-User Chat
else:
raise ValueError("Could not connect to XMPP-Server.")
def init(self):
if pymisp is None and import_fail_reason == 'syntax':
raise MissingDependencyError(
"pymisp",
version='>=2.4.120',
additional_text="Python versions >= 3.6 are "
"required for this 'pymisp' version.")
elif pymisp is None:
raise MissingDependencyError('pymisp', version='>=2.4.120')
self.logger.info(
f'Significant fields are {self.significant_fields!r}.')
self.logger.info(f'Connecting to MISP instance at {self.misp_url!r}.')
self.misp = pymisp.api.PyMISP(self.misp_url, self.misp_key,
self.http_verify_cert)
self.misp.toggle_global_pythonify()
def init(self):
if requests is None:
raise MissingDependencyError("requests")
self.retry_interval = getattr(self.parameters, "retry_interval", 5)
self.url = getattr(self.parameters, "url", None)
if not self.url:
raise ConfigurationError("Connection",
"No Splunk API URL specified")
self.auth_token = getattr(self.parameters, "auth_token", None)
if not self.auth_token:
raise ConfigurationError(
"Connection", "No Splunk API authorization token specified")
self.saved_search = getattr(self.parameters, "saved_search", None)
if not self.saved_search:
raise ConfigurationError("Search",
"No Splunk saved search specified")
self.search_parameters = getattr(self.parameters, "search_parameters",
{})
self.result_fields = getattr(self.parameters, "result_fields", {})
self.not_found = getattr(self.parameters, "not_found",
["warn", "send"])
if "send" in self.not_found and "drop" in self.not_found:
raise ConfigurationError(
"Processing",
"Cannot both drop and send messages without search results")
self.multiple_result_handling = getattr(self.parameters,
"multiple_result_handling",
["warn", "use_first", "send"])
if "limit" in self.multiple_result_handling and len(
self.multiple_result_handling) != 1:
raise ConfigurationError(
"Processing",
"Search results limited to one, no processing of multiple results possible"
)
if "send" in self.multiple_result_handling and "drop" in self.multiple_result_handling:
raise ConfigurationError(
"Processing",
"Cannot both drop and send messages with multiple search results"
)
if "ignore" in self.multiple_result_handling and "use_first" in self.multiple_result_handling:
raise ConfigurationError(
"Processing",
"Cannot both ignore and use multiple search results")
self.overwrite = getattr(self.parameters, "overwrite", None)
self.set_request_parameters()
self.http_header.update(
{"Authorization": "Bearer {}".format(self.auth_token)})
self.session = utils.create_request_session(self)
self.session.keep_alive = False
def init(self):
if PyMISP is None and import_fail_reason == 'syntax':
raise MissingDependencyError(
"pymisp",
version='>=2.4.36,<=2.4.119.1',
additional_text="Python versions below 3.6 are "
"only supported by pymisp <= 2.4.119.1.")
elif PyMISP is None:
raise MissingDependencyError("pymisp")
if hasattr(self.parameters, 'misp_verify'):
self.parameters.http_verify_cert = self.parameters.misp_verify
warnings.warn(
"The parameter 'misp_verify' is deprecated in favor of"
"'http_verify_cert'.", DeprecationWarning)
# Initialize MISP connection
self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key,
self.parameters.http_verify_cert)
def init(self):
if requests is None:
raise MissingDependencyError("requests")
if rt is None:
raise MissingDependencyError("rt")
if getattr(self.parameters, 'search_not_older_than', None):
try:
self.not_older_than = parser.parse(
self.parameters.search_not_older_than)
self.not_older_than_type = 'absolute'
except ValueError:
try:
self.not_older_than_relative = timedelta(
minutes=parse_relative(
self.parameters.search_not_older_than))
except ValueError:
self.logger.error(
"Parameter 'search_not_older_than' could not be parsed. "
"Check your configuration.")
raise
self.not_older_than_type = 'relative'
else:
self.not_older_than_type = False
self.set_request_parameters()
self.session = create_request_session(self)
self._parse_extract_file_parameter('extract_attachment')
self._parse_extract_file_parameter('extract_download')
if hasattr(self.parameters, 'unzip_attachment'):
self.logger.warning(
"The parameter 'unzip_attachment' is deprecated and "
"will be removed in version 3.0 in favor of the "
"more generic and powerful 'extract_attachment'. "
"Look at the Bots documentation for more details.")
if not self.extract_attachment:
self.extract_attachment = self.parameters.unzip_attachment
else:
self.logger.warn(
"Both 'extract_attachment' and the deprecated "
"'unzip_attachment' parameter are in use. Ignoring "
"the latter one.")
def init(self):
if sys.version_info < (3, 6):
raise ValueError('This bot requires Python >= 3.6.')
if ExpandedPyMISP is None:
raise MissingDependencyError('pymisp', '>=2.4.117.3')
# Initialize MISP connection
self.misp = ExpandedPyMISP(self.parameters.misp_url,
self.parameters.misp_key,
self.parameters.http_verify_cert)
def init(self):
if MISPEvent is None and import_fail_reason == 'syntax':
raise MissingDependencyError("pymisp",
version='>=2.4.117.3',
additional_text="Python versions below 3.6 are "
"only supported by pymisp <= 2.4.119.1.")
elif MISPEvent is None:
raise MissingDependencyError('pymisp', version='>=2.4.117.3')
self.current_event = None
self.misp_org = MISPOrganisation()
self.misp_org.name = self.misp_org_name
self.misp_org.uuid = self.misp_org_uuid
self.output_dir = Path(self.output_dir)
MISPFeedOutputBot.check_output_dir(self.output_dir)
if self.interval_event is None:
self.timedelta = datetime.timedelta(hours=1)
else:
self.timedelta = datetime.timedelta(minutes=parse_relative(self.interval_event))
if (self.output_dir / '.current').exists():
with (self.output_dir / '.current').open() as f:
self.current_file = Path(f.read())
self.current_event = MISPEvent()
self.current_event.load_file(self.current_file)
last_min_time, last_max_time = re.findall('IntelMQ event (.*) - (.*)', self.current_event.info)[0]
last_min_time = datetime.datetime.strptime(last_min_time, '%Y-%m-%dT%H:%M:%S.%f')
last_max_time = datetime.datetime.strptime(last_max_time, '%Y-%m-%dT%H:%M:%S.%f')
if last_max_time < datetime.datetime.now():
self.min_time_current = datetime.datetime.now()
self.max_time_current = self.min_time_current + self.timedelta
self.current_event = None
else:
self.min_time_current = last_min_time
self.max_time_current = last_max_time
else:
self.min_time_current = datetime.datetime.now()
self.max_time_current = self.min_time_current + self.timedelta
def init(self):
if stomp is None:
raise MissingDependencyError("stomp")
elif stomp.__version__ < (4, 1, 8):
raise MissingDependencyError("stomp",
version="4.1.8",
installed=stomp.__version__)
self.server = getattr(self.parameters, 'server', 'n6stream.cert.pl')
self.port = getattr(self.parameters, 'port', 61614)
self.exchange = getattr(self.parameters, 'exchange', '')
self.heartbeat = getattr(self.parameters, 'heartbeat', 60000)
self.ssl_ca_cert = getattr(self.parameters, 'ssl_ca_certificate',
'ca.pem')
self.ssl_cl_cert = getattr(self.parameters, 'ssl_client_certificate',
'client.pem')
self.ssl_cl_cert_key = getattr(self.parameters,
'ssl_client_certificate_key',
'client.key')
self.http_verify_cert = getattr(self.parameters, 'http_verify_cert',
True)
# check if certificates exist
for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]:
if not os.path.isfile(f):
raise ValueError("Could not open file %r." % f)
_host = [(self.server, self.port)]
self.conn = stomp.Connection(host_and_ports=_host,
use_ssl=True,
ssl_key_file=self.ssl_cl_cert_key,
ssl_cert_file=self.ssl_cl_cert,
ssl_ca_certs=self.ssl_ca_cert,
heartbeats=(self.heartbeat,
self.heartbeat))
self.conn.set_listener('', StompListener(self, self.conn,
self.exchange))
connect_and_subscribe(self.conn,
self.logger,
self.exchange,
start=stomp.__version__ < (4, 1, 20))
def init(self):
if not Template:
raise MissingDependencyError("jinja2")
if self.ssl:
self.smtp_class = smtplib.SMTP_SSL
else:
self.smtp_class = smtplib.SMTP
if not self.mail_from:
raise ConfigurationError("Mail", "No sender specified")
if not self.mail_to:
raise ConfigurationError("Mail", "No recipient specified")
if self.username and not self.password:
raise ConfigurationError(
"Server", "SMTP username provided, but not password")
if self.password and not self.username:
raise ConfigurationError(
"Server", "SMTP password provided, but not username")
self.smtp_authentication = self.username and self.password
self.templates = {
"subject": Template(self.subject),
"from": Template(self.mail_from),
"to": Template(self.mail_to),
"body": Template(self.body),
}
for tmpl in self.templates.values():
tmpl.globals.update({"from_json": json.loads})
self.templates["attachments"] = []
for att in self.attachments:
if "content-type" not in att:
self.logger.error(
"Attachment does not have a content-type, ignoring: %s.",
att)
continue
elif "text" not in att:
self.logger.error(
"Attachment does not have a text, ignoring: %s.", att)
continue
elif "name" not in att:
self.logger.error(
"Attachment does not have a name, ignoring: %s.", att)
continue
attachment_template = {
"content-type": Template(att["content-type"]),
"text": Template(att["text"]),
"name": Template(att["name"])
}
for tmpl in attachment_template.values():
tmpl.globals.update({"from_json": json.loads})
self.templates["attachments"].append(attachment_template)
def init(self):
if kafka is None:
raise MissingDependencyError("kafka")
self.logger.debug(
"Topic set to {}, bootstrap_servers set to {}".format(
self.topic, self.bootstrap_servers))
self.logger.debug(
"ssl_cafile set to {}, ssl_certfile set to {}, ssl_check_hostname set to {}"
.format(self.ssl_cafile, self.ssl_certfile,
self.ssl_check_hostname))
def init(self):
if ContainerClient is None or create_configuration is None:
raise MissingDependencyError("azure.storage", version='>=12.0.0')
self.config = create_configuration(storage_sdk='blob')
if hasattr(self, 'https_proxy'):
# Create a storage configuration object and update the proxy policy
self.config.proxy_policy.proxies = {
'http': self.http_proxy,
'https': self.https_proxy,
}
def init(self):
super().init()
if requests is None:
raise MissingDependencyError("requests")
# Build request
self.set_request_parameters()
self.session = create_request_session(self)
self.chunk_size = getattr(self.parameters, 'chunk_size', None)
self.chunk_replicate_header = getattr(self.parameters,
'chunk_replicate_header', None)
