def init(self): if url_normalize is None: raise MissingDependencyError("url-normalize") url_version = pkg_resources.get_distribution("url-normalize").version if tuple(int(v) for v in url_version.split('.')) < ( 1, 4, 1) and self.default_scheme is not None: raise ValueError( "Parameter 'default_scheme' given but 'url-normalize' version %r does not support it. " "Get at least version '1.4.1'." % url_version) if get_tld is None: raise MissingDependencyError("tld") try: update_tld_names() except tld.exceptions.TldIOError: self.logger.info("Could not update TLD names cache.") if self.domain_whitelist != '': self._domain_whitelist.extend(self.domain_whitelist.split(',')) if self.substitutions != '': temp = self.substitutions.split(';') if len(temp) % 2 != 0: raise InvalidArgument( 'substitutions', got=self.substitutions, expected="even number of ; separated strings") for i in range(int(len(temp) / 2)): self._substitutions.append([temp[2 * i], temp[2 * i + 1]]) if not ClassificationType.is_valid(self.classification_type): self.classification_type = 'unknown' if self.default_scheme is not None: self.url_kwargs = {'default_scheme': self.default_scheme} else: self.url_kwargs = {}
def init(self): if requests is None: raise MissingDependencyError("requests") if twitter is None: raise MissingDependencyError("twitter") self.current_time_in_seconds = int(time.time()) self.target_timelines = [] if getattr(self.parameters, "target_timelines", '') != '': self.target_timelines.extend( self.parameters.target_timelines.split(',')) self.tweet_count = int(getattr(self.parameters, "tweet_count", 20)) self.follow_urls = [] if getattr(self.parameters, "follow_urls", '') != '': self.follow_urls.extend(self.parameters.follow_urls.split(',')) self.include_rts = getattr(self.parameters, "include_rts", False) self.exclude_replies = getattr(self.parameters, "exclude_replies", False) self.timelimit = int( getattr(self.parameters, "timelimit", 24 * 60 * 60)) self.api = twitter.Api( consumer_key=self.parameters.consumer_key, consumer_secret=self.parameters.consumer_secret, access_token_key=self.parameters.access_token_key, access_token_secret=self.parameters.access_token_secret, tweet_mode="extended") self.set_request_parameters() self.session = create_request_session_from_bot(self)
def init(self): if stomp is None: raise MissingDependencyError("stomp") elif stomp.__version__ < (4, 1, 8): raise MissingDependencyError("stomp", version="4.1.8", installed=stomp.__version__) self.ssl_ca_cert = self.ssl_ca_certificate self.ssl_cl_cert = self.ssl_client_certificate self.ssl_cl_cert_key = self.ssl_client_certificate_key # check if certificates exist for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]: if not os.path.isfile(f): raise ValueError("Could not open file %r." % f) _host = [(self.server, self.port)] self.__conn = stomp.Connection(host_and_ports=_host, use_ssl=True, ssl_key_file=self.ssl_cl_cert_key, ssl_cert_file=self.ssl_cl_cert, ssl_ca_certs=self.ssl_ca_cert, heartbeats=(self.heartbeat, self.heartbeat)) self.__conn.set_listener( '', StompListener(self, self.__conn, self.exchange)) connect_and_subscribe(self.__conn, self.logger, self.exchange, start=stomp.__version__ < (4, 1, 20))
def init(self): if pika is None: raise MissingDependencyError("pika", version=">=1.0") self.connection = None self.channel = None pika_version = tuple(int(x) for x in pika.__version__.split('.')) if pika_version < (1, ): raise MissingDependencyError("pika", version=">=1.0", installed=pika.__version__) self.kwargs = {} self.kwargs['heartbeat'] = self.connection_heartbeat if self.username and self.password: self.kwargs['credentials'] = pika.PlainCredentials( self.username, self.password) if self.use_ssl: self.kwargs['ssl_options'] = pika.SSLOptions( context=ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)) self.connection_parameters = pika.ConnectionParameters( host=self.connection_host, port=self.connection_port, virtual_host=self.connection_vhost, connection_attempts=self.connection_attempts, **self.kwargs) self.connect_server()
def init(self): if DxlClient is None: raise MissingDependencyError('dxlclient') if MarClient is None: raise MissingDependencyError('dxlmarclient') self.config = DxlClientConfig.create_dxl_config_from_file( self.dxl_config_file)
def init(self): if requests is None: raise MissingDependencyError("requests") self.set_request_parameters() self.session = create_request_session(self) self.use_gpg = getattr(self.parameters, "verify_pgp_signatures", False) if self.use_gpg and gnupg is None: raise MissingDependencyError("gnupg") else: self.logger.info('PGP signature verification is active.')
def init(self): if requests is None: raise MissingDependencyError("requests") if base64 is None: raise MissingDependencyError("base64") self.esm = ESM(self.parameters.esm_ip, self.parameters.esm_user, self.parameters.esm_password) try: self.esm.login() self.esm.logout() except Exception: raise ValueError('Could not Login to ESM.')
def init(self): if OTXv2 is None: raise MissingDependencyError("OTXv2") self.modified_pulses_only = False if hasattr(self, 'modified_pulses_only'): self.modified_pulses_only = self.modified_pulses_only
def init(self): if kafka is None: raise MissingDependencyError("kafka") self.topic = [] if getattr(self.parameters, 'topic', '') != '': self.topic = self.parameters.topic self.bootstrap_servers = 'localhost:9092' if getattr(self.parameters, 'bootstrap_servers', '') != '': self.bootstrap_servers = self.parameters.bootstrap_servers self.ssl_cafile = getattr(self.parameters, 'ssl_ca_certificate', None) self.ssl_certfile = getattr(self.parameters, 'ssl_client_certificate', None) self.ssl_check_hostname = getattr(self.parameters, 'ssl_check_hostname', False) self.logger.debug( "Topic set to {}, bootstrap_servers set to {}".format( self.topic, self.bootstrap_servers)) self.logger.debug( "ssl_cafile set to {}, ssl_certfile set to {}, ssl_check_hostname set to {}" .format(self.ssl_cafile, self.ssl_certfile, self.ssl_check_hostname))
def init(self): if requests is None: raise MissingDependencyError("requests") self.set_request_parameters() self.session = create_request_session_from_bot(self)
def init(self): if ExpandedPyMISP is None: raise MissingDependencyError('pymisp', '>=2.4.117.3') # Initialize MISP connection self.misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.http_verify_cert)
def init(self): if xmltodict is None: raise MissingDependencyError("xmltodict") self.set_request_parameters() self.session = create_request_session_from_bot(self) self.dns_name = getattr(self.parameters, "dns_name", None) if self.dns_name is None: raise ValueError('No dns name provided.') self.request_duration = getattr(self.parameters, "request_duration", None) if self.request_duration is None: raise ValueError('No request_duration provided.') user = getattr(self.parameters, "http_username", None) if user is None: raise ValueError('No http_username provided.') pw = getattr(self.parameters, "http_password", None) if pw is None: raise ValueError('No http_password provided.') # create auth token token = user + ":" + pw message_bytes = token.encode('ascii') base64_bytes = base64.b64encode(message_bytes) base64_message = base64_bytes.decode('ascii') self.http_header = {'Authorization': 'Basic ' + base64_message} self.custom_auth_url = "https://" + self.dns_name + "/wsapis/v2.0.0/auth/login"
def init(self): if stomp is None: raise MissingDependencyError("stomp") self.server = getattr(self.parameters, 'server', '127.0.0.1') self.port = getattr(self.parameters, 'port', 61614) self.exchange = getattr(self.parameters, 'exchange', '/exchange/_push') self.heartbeat = getattr(self.parameters, 'heartbeat', 60000) self.ssl_ca_cert = getattr(self.parameters, 'ssl_ca_certificate', 'ca.pem') self.ssl_cl_cert = getattr(self.parameters, 'ssl_client_certificate', 'client.pem') self.ssl_cl_cert_key = getattr(self.parameters, 'ssl_client_certificate_key', 'client.key') self.http_verify_cert = getattr(self.parameters, 'http_verify_cert', True) # check if certificates exist for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]: if not os.path.isfile(f): raise ValueError( "Could not open SSL (certificate) file '%s'." % f) _host = [(self.server, self.port)] self.conn = stomp.Connection(host_and_ports=_host, use_ssl=True, ssl_key_file=self.ssl_cl_cert_key, ssl_cert_file=self.ssl_cl_cert, ssl_ca_certs=self.ssl_ca_cert, heartbeats=(self.heartbeat, self.heartbeat)) self.connect()
def init(self): if validators is None: raise MissingDependencyError('validators') self.__supported_feeds = { 'StrangerealIntel/DailyIOC': lambda logger: self.StrangerealIntelDailyIOC(logger) }
def init(self): if imbox is None: raise MissingDependencyError("imbox") if getattr(self.parameters, 'attach_unzip', None) and not self.extract_files: self.extract_files = True self.logger.warning("The parameter 'attach_unzip' is deprecated and will " "be removed in version 4.0. Use 'extract_files' instead.")
def init(self): if OTXv2 is None: raise MissingDependencyError("OTXv2") self.modified_pulses_only = False if hasattr(self.parameters, 'modified_pulses_only'): self.modified_pulses_only = self.parameters.modified_pulses_only self.interval = getattr(self.parameters, 'interval', 24)
def init(self): if cabby is None: raise MissingDependencyError('cabby') self.user = self.parameters.username self.passwd = self.parameters.password self.endpoint = self.parameters.endpoint self.time_delta = int(self.parameters.time_delta) self.collection = self.parameters.collection
def init(self): if requests is None: raise MissingDependencyError("requests") self.set_request_parameters() self.session = create_request_session(self) self.__error_count = 0
def init(self): if sleekxmpp is None: raise MissingDependencyError("sleekxmpp") # Retrieve Parameters from configuration xmpp_user = getattr(self.parameters, "xmpp_user", None) xmpp_server = getattr(self.parameters, "xmpp_server", None) xmpp_password = getattr(self.parameters, "xmpp_password", None) if None in (xmpp_user, xmpp_server, xmpp_password): raise ValueError('No User / Password provided.') else: xmpp_login = xmpp_user + '@' + xmpp_server self.muc = getattr(self.parameters, "use_muc", None) xmpp_to_user = getattr(self.parameters, "xmpp_to_user", None) xmpp_to_server = getattr(self.parameters, "xmpp_to_server", None) xmpp_room = getattr(self.parameters, "xmpp_room", None) if self.muc else None xmpp_room_nick = getattr(self.parameters, "xmpp_room_nick", None) if self.muc else None xmpp_room_password = getattr(self.parameters, "xmpp_room_password", None) if self.muc else None ca_certs = getattr(self.parameters, "ca_certs", None) # Be sure the receiver was set up if not self.muc and None in (xmpp_to_user, xmpp_to_server): raise ValueError('No receiver for direct messages provided.') else: self.xmpp_receiver = xmpp_to_user + '@' +\ xmpp_to_server if self.muc and not xmpp_room: raise ValueError('No room provided.') else: self.xmpp_receiver = xmpp_room if self.muc: if not xmpp_room_nick: # create the room_nick from user and server xmpp_room_nick = xmpp_login self.xmpp = XMPPClient(xmpp_login, xmpp_password, xmpp_room, xmpp_room_nick, xmpp_room_password, self.logger) if ca_certs: # Set CA-Certificates self.xmpp.ca_certs = ca_certs if self.xmpp.connect(reattempt=False): self.xmpp.process() # Add Handlers and register Plugins self.xmpp.register_plugin('xep_0030') # Service Discovery self.xmpp.register_plugin('xep_0045') # Multi-User Chat else: raise ValueError("Could not connect to XMPP-Server.")
def init(self): if pymisp is None and import_fail_reason == 'syntax': raise MissingDependencyError( "pymisp", version='>=2.4.120', additional_text="Python versions >= 3.6 are " "required for this 'pymisp' version.") elif pymisp is None: raise MissingDependencyError('pymisp', version='>=2.4.120') self.logger.info( f'Significant fields are {self.significant_fields!r}.') self.logger.info(f'Connecting to MISP instance at {self.misp_url!r}.') self.misp = pymisp.api.PyMISP(self.misp_url, self.misp_key, self.http_verify_cert) self.misp.toggle_global_pythonify()
def init(self): if requests is None: raise MissingDependencyError("requests") self.retry_interval = getattr(self.parameters, "retry_interval", 5) self.url = getattr(self.parameters, "url", None) if not self.url: raise ConfigurationError("Connection", "No Splunk API URL specified") self.auth_token = getattr(self.parameters, "auth_token", None) if not self.auth_token: raise ConfigurationError( "Connection", "No Splunk API authorization token specified") self.saved_search = getattr(self.parameters, "saved_search", None) if not self.saved_search: raise ConfigurationError("Search", "No Splunk saved search specified") self.search_parameters = getattr(self.parameters, "search_parameters", {}) self.result_fields = getattr(self.parameters, "result_fields", {}) self.not_found = getattr(self.parameters, "not_found", ["warn", "send"]) if "send" in self.not_found and "drop" in self.not_found: raise ConfigurationError( "Processing", "Cannot both drop and send messages without search results") self.multiple_result_handling = getattr(self.parameters, "multiple_result_handling", ["warn", "use_first", "send"]) if "limit" in self.multiple_result_handling and len( self.multiple_result_handling) != 1: raise ConfigurationError( "Processing", "Search results limited to one, no processing of multiple results possible" ) if "send" in self.multiple_result_handling and "drop" in self.multiple_result_handling: raise ConfigurationError( "Processing", "Cannot both drop and send messages with multiple search results" ) if "ignore" in self.multiple_result_handling and "use_first" in self.multiple_result_handling: raise ConfigurationError( "Processing", "Cannot both ignore and use multiple search results") self.overwrite = getattr(self.parameters, "overwrite", None) self.set_request_parameters() self.http_header.update( {"Authorization": "Bearer {}".format(self.auth_token)}) self.session = utils.create_request_session(self) self.session.keep_alive = False
def init(self): if PyMISP is None and import_fail_reason == 'syntax': raise MissingDependencyError( "pymisp", version='>=2.4.36,<=2.4.119.1', additional_text="Python versions below 3.6 are " "only supported by pymisp <= 2.4.119.1.") elif PyMISP is None: raise MissingDependencyError("pymisp") if hasattr(self.parameters, 'misp_verify'): self.parameters.http_verify_cert = self.parameters.misp_verify warnings.warn( "The parameter 'misp_verify' is deprecated in favor of" "'http_verify_cert'.", DeprecationWarning) # Initialize MISP connection self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.http_verify_cert)
def init(self): if requests is None: raise MissingDependencyError("requests") if rt is None: raise MissingDependencyError("rt") if getattr(self.parameters, 'search_not_older_than', None): try: self.not_older_than = parser.parse( self.parameters.search_not_older_than) self.not_older_than_type = 'absolute' except ValueError: try: self.not_older_than_relative = timedelta( minutes=parse_relative( self.parameters.search_not_older_than)) except ValueError: self.logger.error( "Parameter 'search_not_older_than' could not be parsed. " "Check your configuration.") raise self.not_older_than_type = 'relative' else: self.not_older_than_type = False self.set_request_parameters() self.session = create_request_session(self) self._parse_extract_file_parameter('extract_attachment') self._parse_extract_file_parameter('extract_download') if hasattr(self.parameters, 'unzip_attachment'): self.logger.warning( "The parameter 'unzip_attachment' is deprecated and " "will be removed in version 3.0 in favor of the " "more generic and powerful 'extract_attachment'. " "Look at the Bots documentation for more details.") if not self.extract_attachment: self.extract_attachment = self.parameters.unzip_attachment else: self.logger.warn( "Both 'extract_attachment' and the deprecated " "'unzip_attachment' parameter are in use. Ignoring " "the latter one.")
def init(self): if sys.version_info < (3, 6): raise ValueError('This bot requires Python >= 3.6.') if ExpandedPyMISP is None: raise MissingDependencyError('pymisp', '>=2.4.117.3') # Initialize MISP connection self.misp = ExpandedPyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.http_verify_cert)
def init(self): if MISPEvent is None and import_fail_reason == 'syntax': raise MissingDependencyError("pymisp", version='>=2.4.117.3', additional_text="Python versions below 3.6 are " "only supported by pymisp <= 2.4.119.1.") elif MISPEvent is None: raise MissingDependencyError('pymisp', version='>=2.4.117.3') self.current_event = None self.misp_org = MISPOrganisation() self.misp_org.name = self.misp_org_name self.misp_org.uuid = self.misp_org_uuid self.output_dir = Path(self.output_dir) MISPFeedOutputBot.check_output_dir(self.output_dir) if self.interval_event is None: self.timedelta = datetime.timedelta(hours=1) else: self.timedelta = datetime.timedelta(minutes=parse_relative(self.interval_event)) if (self.output_dir / '.current').exists(): with (self.output_dir / '.current').open() as f: self.current_file = Path(f.read()) self.current_event = MISPEvent() self.current_event.load_file(self.current_file) last_min_time, last_max_time = re.findall('IntelMQ event (.*) - (.*)', self.current_event.info)[0] last_min_time = datetime.datetime.strptime(last_min_time, '%Y-%m-%dT%H:%M:%S.%f') last_max_time = datetime.datetime.strptime(last_max_time, '%Y-%m-%dT%H:%M:%S.%f') if last_max_time < datetime.datetime.now(): self.min_time_current = datetime.datetime.now() self.max_time_current = self.min_time_current + self.timedelta self.current_event = None else: self.min_time_current = last_min_time self.max_time_current = last_max_time else: self.min_time_current = datetime.datetime.now() self.max_time_current = self.min_time_current + self.timedelta
def init(self): if stomp is None: raise MissingDependencyError("stomp") elif stomp.__version__ < (4, 1, 8): raise MissingDependencyError("stomp", version="4.1.8", installed=stomp.__version__) self.server = getattr(self.parameters, 'server', 'n6stream.cert.pl') self.port = getattr(self.parameters, 'port', 61614) self.exchange = getattr(self.parameters, 'exchange', '') self.heartbeat = getattr(self.parameters, 'heartbeat', 60000) self.ssl_ca_cert = getattr(self.parameters, 'ssl_ca_certificate', 'ca.pem') self.ssl_cl_cert = getattr(self.parameters, 'ssl_client_certificate', 'client.pem') self.ssl_cl_cert_key = getattr(self.parameters, 'ssl_client_certificate_key', 'client.key') self.http_verify_cert = getattr(self.parameters, 'http_verify_cert', True) # check if certificates exist for f in [self.ssl_ca_cert, self.ssl_cl_cert, self.ssl_cl_cert_key]: if not os.path.isfile(f): raise ValueError("Could not open file %r." % f) _host = [(self.server, self.port)] self.conn = stomp.Connection(host_and_ports=_host, use_ssl=True, ssl_key_file=self.ssl_cl_cert_key, ssl_cert_file=self.ssl_cl_cert, ssl_ca_certs=self.ssl_ca_cert, heartbeats=(self.heartbeat, self.heartbeat)) self.conn.set_listener('', StompListener(self, self.conn, self.exchange)) connect_and_subscribe(self.conn, self.logger, self.exchange, start=stomp.__version__ < (4, 1, 20))
def init(self): if not Template: raise MissingDependencyError("jinja2") if self.ssl: self.smtp_class = smtplib.SMTP_SSL else: self.smtp_class = smtplib.SMTP if not self.mail_from: raise ConfigurationError("Mail", "No sender specified") if not self.mail_to: raise ConfigurationError("Mail", "No recipient specified") if self.username and not self.password: raise ConfigurationError( "Server", "SMTP username provided, but not password") if self.password and not self.username: raise ConfigurationError( "Server", "SMTP password provided, but not username") self.smtp_authentication = self.username and self.password self.templates = { "subject": Template(self.subject), "from": Template(self.mail_from), "to": Template(self.mail_to), "body": Template(self.body), } for tmpl in self.templates.values(): tmpl.globals.update({"from_json": json.loads}) self.templates["attachments"] = [] for att in self.attachments: if "content-type" not in att: self.logger.error( "Attachment does not have a content-type, ignoring: %s.", att) continue elif "text" not in att: self.logger.error( "Attachment does not have a text, ignoring: %s.", att) continue elif "name" not in att: self.logger.error( "Attachment does not have a name, ignoring: %s.", att) continue attachment_template = { "content-type": Template(att["content-type"]), "text": Template(att["text"]), "name": Template(att["name"]) } for tmpl in attachment_template.values(): tmpl.globals.update({"from_json": json.loads}) self.templates["attachments"].append(attachment_template)
def init(self): if kafka is None: raise MissingDependencyError("kafka") self.logger.debug( "Topic set to {}, bootstrap_servers set to {}".format( self.topic, self.bootstrap_servers)) self.logger.debug( "ssl_cafile set to {}, ssl_certfile set to {}, ssl_check_hostname set to {}" .format(self.ssl_cafile, self.ssl_certfile, self.ssl_check_hostname))
def init(self): if ContainerClient is None or create_configuration is None: raise MissingDependencyError("azure.storage", version='>=12.0.0') self.config = create_configuration(storage_sdk='blob') if hasattr(self, 'https_proxy'): # Create a storage configuration object and update the proxy policy self.config.proxy_policy.proxies = { 'http': self.http_proxy, 'https': self.https_proxy, }
def init(self): super().init() if requests is None: raise MissingDependencyError("requests") # Build request self.set_request_parameters() self.session = create_request_session(self) self.chunk_size = getattr(self.parameters, 'chunk_size', None) self.chunk_replicate_header = getattr(self.parameters, 'chunk_replicate_header', None)