def __init__(self): self.logs_all_queue = "logs_all" self.consumer_threads = {} self.logger_threads = {} # Get configuration config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.log_level = os.getenv( "WORKER_LOG_LEVEL") or config["worker"]["log_level"] self.opencti_url = os.getenv("OPENCTI_URL") or config["opencti"]["url"] self.opencti_token = os.getenv( "OPENCTI_TOKEN") or config["opencti"]["token"] # Check if openCTI is available self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) # Configure logger numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError("Invalid log level: " + self.log_level) logging.basicConfig(level=numeric_level) # Get logger config self.logger_config = self.api.get_logs_worker_config() # Initialize variables self.connectors = [] self.queues = []
def __init__( self, connector: Dict[str, Any], opencti_url: str, opencti_token: str, log_level: str, ) -> None: threading.Thread.__init__(self) self.api = OpenCTIApiClient(opencti_url, opencti_token, log_level) self.queue_name = connector["config"]["push"] self.pika_credentials = pika.PlainCredentials( connector["config"]["connection"]["user"], connector["config"]["connection"]["pass"], ) self.pika_parameters = pika.ConnectionParameters( connector["config"]["connection"]["host"], connector["config"]["connection"]["port"], "/", self.pika_credentials, ssl_options=pika.SSLOptions(create_ssl_context()) if connector["config"]["connection"]["use_ssl"] else None, ) self.pika_connection = pika.BlockingConnection(self.pika_parameters) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1) self.processing_count: int = 0
def __init__(self): self.logs_all_queue = 'logs_all' self.consumer_threads = {} self.logger_threads = {} # Get configuration config_file_path = os.path.dirname( os.path.abspath(__file__)) + '/config.yml' config = yaml.load(open(config_file_path), Loader=yaml.FullLoader ) if os.path.isfile(config_file_path) else {} self.log_level = os.getenv( 'WORKER_LOG_LEVEL') or config['worker']['log_level'] self.opencti_url = os.getenv('OPENCTI_URL') or config['opencti']['url'] self.opencti_token = os.getenv( 'OPENCTI_TOKEN') or config['opencti']['token'] # Check if openCTI is available self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) # Configure logger numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: ' + self.log_level) logging.basicConfig(level=numeric_level) # Get logger config self.logger_config = self.api.get_logs_worker_config() # Initialize variables self.connectors = [] self.queues = []
def __post_init__(self) -> None: super().__init__() self.api = OpenCTIApiClient( url=self.opencti_url, token=self.opencti_token, log_level=self.log_level, ssl_verify=self.ssl_verify, json_logging=self.json_logging, ) self.queue_name = self.connector["config"]["push"] self.pika_credentials = pika.PlainCredentials( self.connector["config"]["connection"]["user"], self.connector["config"]["connection"]["pass"], ) self.pika_parameters = pika.ConnectionParameters( self.connector["config"]["connection"]["host"], self.connector["config"]["connection"]["port"], "/", self.pika_credentials, ssl_options=pika.SSLOptions(create_ssl_context()) if self.connector["config"]["connection"]["use_ssl"] else None, ) self.pika_connection = pika.BlockingConnection(self.pika_parameters) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1) self.processing_count: int = 0
def __init__(self): self.consumer_threads = {} # Get configuration config_file_path = os.path.dirname( os.path.abspath(__file__)) + '/config.yml' if os.path.isfile(config_file_path): config = yaml.load(open(config_file_path), Loader=yaml.FullLoader) self.type = config['worker']['type'] self.log_level = config['worker']['log_level'] self.opencti_url = config['opencti']['url'] self.opencti_token = config['opencti']['token'] self.rabbitmq_hostname = config['rabbitmq']['hostname'] self.rabbitmq_port = config['rabbitmq']['port'] self.rabbitmq_port_management = config['rabbitmq'][ 'port_management'] self.rabbitmq_management_ssl = config['rabbitmq']['management_ssl'] self.rabbitmq_username = config['rabbitmq']['username'] self.rabbitmq_password = config['rabbitmq']['password'] else: self.type = os.getenv('WORKER_TYPE', 'import') self.log_level = os.getenv('WORKER_LOG_LEVEL', 'info') self.opencti_url = os.getenv('OPENCTI_URL', 'http://localhost:4000') self.opencti_token = os.getenv('OPENCTI_TOKEN', 'ChangeMe') self.rabbitmq_hostname = os.getenv('RABBITMQ_HOSTNAME', 'localhost') self.rabbitmq_port = os.getenv('RABBITMQ_PORT', 5672) self.rabbitmq_port_management = os.getenv( 'RABBITMQ_PORT_MANAGEMENT', 15672) self.rabbitmq_management_ssl = os.getenv('RABBITMQ_MANAGEMENT_SSL', "false") == "true" self.rabbitmq_username = os.getenv('RABBITMQ_USERNAME', 'guest') self.rabbitmq_password = os.getenv('RABBITMQ_PASSWORD', 'guest') # Check configuration if len(self.opencti_token) == 0 or self.opencti_token == 'ChangeMe': raise ValueError( 'Token configuration must be the same as APP__ADMIN__TOKEN') # Check if openCTI is available self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) if not self.api.health_check(): raise ValueError('OpenCTI API seems down') if self.type == 'import': self.DEFAULT_QUEUE_NAME = 'import-platform' self.DEFAULT_ROUTING_KEY = 'import.platform' self.CONNECTOR_QUEUE_PREFIX = 'import-connectors-' elif self.type == 'export': self.DEFAULT_QUEUE_NAME = 'export-platform' self.DEFAULT_ROUTING_KEY = 'export.platform' self.CONNECTOR_QUEUE_PREFIX = 'export-connectors-' # Configure logger numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: ' + self.log_level) logging.basicConfig(level=numeric_level)
def __init__(self, connector, opencti_url, opencti_token): threading.Thread.__init__(self) self.opencti_url = opencti_url self.opencti_token = opencti_token self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) self.queue_name = connector['config']['push'] self.pika_connection = pika.BlockingConnection(pika.URLParameters(connector['config']['uri'])) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1)
def __init__(self): logging.getLogger("pika").setLevel(logging.ERROR) welcome_art = text2art("OpenCTI migrator") print(welcome_art) config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" self.config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.config.update({ key.lower(): value for key, value in os.environ.items() if key.startswith("OPENCTI") }) for config in mandatory_configs: if config not in config: raise ValueError("Missing configuration parameter: " + config) # Test connection to the V3 API self.opencti_api_client = OpenCTIApiClient( self.config["opencti_v3_url"], self.config["opencti_v3_token"], "error") print("Checking access to OpenCTI version 3.3.2 instance... OK") # Check RabbitMQ self.connection = None self.channel = None self._rabbitmq_connect() print("Checking access to the OpenCTI versio 4.X.X RabbitMQ... OK") # Check if state already here or is creatable self.state_file = os.path.dirname( os.path.abspath(__file__)) + "/state.json" print("Checking if the state file is writtable... OK") self.get_state()
def get_client(opencti_url, opencti_token, ssl_verify, http_proxies): client = OpenCTIApiClient( opencti_url, opencti_token, log_level=opencti_config["log_level"], ssl_verify=ssl_verify, proxies=http_proxies, ) return client
def __init__(self, connector, opencti_url, opencti_token, log_level): threading.Thread.__init__(self) self.api = OpenCTIApiClient(opencti_url, opencti_token, log_level) self.queue_name = connector["config"]["push"] self.pika_credentials = pika.PlainCredentials( connector["config"]["connection"]["user"], connector["config"]["connection"]["pass"], ) self.pika_parameters = pika.ConnectionParameters( connector["config"]["connection"]["host"], connector["config"]["connection"]["port"], "/", self.pika_credentials, ) self.pika_connection = pika.BlockingConnection(self.pika_parameters) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1) self.processing_count = 0
def __init__( # pylint: disable=too-many-arguments self, source_url, source_token, target_url, target_token, consuming_count, start_timestamp, live_stream_id=None, ): self.source_url = source_url self.source_token = source_token self.target_url = target_url self.target_token = target_token self.live_stream_id = live_stream_id self.count_number = 0 self.consuming_count = consuming_count self.start_timestamp = start_timestamp self.stream = None # Source config = { "id": "673ba380-d229-4160-9213-ac5afdaabf96", "type": "STREAM", "name": "Synchronizer", "scope": "synchronizer", "confidence_level": 15, "live_stream_id": self.live_stream_id, "log_level": "info", } self.opencti_source_client = OpenCTIApiClient(source_url, source_token) self.opencti_source_helper = OpenCTIConnectorHelper( { "opencti": {"url": self.source_url, "token": self.source_token}, "connector": config, } ) # Target self.opencti_target_client = OpenCTIApiClient(target_url, target_token) self.opencti_target_helper = OpenCTIConnectorHelper( { "opencti": {"url": self.target_url, "token": self.target_token}, "connector": config, } )
def main(): params = demisto.params() args = demisto.args() credentials = params.get('credentials', {}) api_key = credentials.get('password') base_url = params.get('base_url').strip('/') indicator_types = params.get('indicator_types', ['ALL']) max_fetch = params.get('max_indicator_to_fetch') tlp_color = params.get('tlp_color') tags = argToList(params.get('feedTags')) if max_fetch: max_fetch = arg_to_number(max_fetch) else: max_fetch = 500 start = arg_to_number(args.get('score_start', 1)) end = arg_to_number(args.get('score_end', 100)) + 1 # type:ignore score = None if start or end: score = [str(i) for i in range(start, end)] # type:ignore try: client = OpenCTIApiClient(base_url, api_key, ssl_verify=params.get('insecure'), log_level='error') command = demisto.command() demisto.info(f"Command being called is {command}") # Switch case if command == "fetch-indicators": fetch_indicators_command(client, indicator_types, max_fetch, tlp_color=tlp_color, score=score, tags=tags) elif command == "test-module": '''When setting up an OpenCTI Client it is checked that it is valid and allows requests to be sent. and if not he immediately sends an error''' fetch_indicators_command(client, indicator_types, max_fetch, is_test=True) return_results('ok') elif command == "opencti-get-indicators": return_results(get_indicators_command(client, args)) elif command == "opencti-reset-fetch-indicators": return_results(reset_last_run()) except Exception as e: demisto.error(traceback.format_exc()) # print the traceback return_error(f"Error:\n [{e}]")
def __init__(self): config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) self.lastinfosec_cti_url = "https://api.client.lastinfosec.com/v2/stix21/getbyminutes/{}?api_key={}&headers=false&platform=opencti" self.lastinfosec_cve_url = "https://api.client.lastinfosec.com/v2/stix21/vulnerabilities/getlasthour?api_key={}&headers=false&platform=opencti" self.lastinfosec_tactic_url = "https://api.client.lastinfosec.com/v2/stix21/tactic/getlast24hour?api_key={}&headers=false&platform=opencti" self.lastinfosec_apikey = get_config_variable( "CONFIG_LIS_APIKEY", ["lastinfosec", "api_key"], config) self.lastinfosec_cti_enabled = get_config_variable( "CONFIG_LIS_CTI_ENABLED", ["lastinfosec", "cti", "is_enabled"], config) self.lastinfosec_cti_interval = get_config_variable( "CONFIG_LIS_CTI_INTERVAL", ["lastinfosec", "cti", "interval"], config) self.lastinfosec_cve_enabled = get_config_variable( "CONFIG_LIS_CVE_ENABLED", ["lastinfosec", "cve", "is_enabled"], config) self.lastinfosec_tactic_enabled = get_config_variable( "CONFIG_LIS_TACTIC_ENABLED", ["lastinfosec", "tactic", "is_enabled"], config) self.opencti_url = get_config_variable("OPENCTI_URL", ["opencti", "url"], config) self.opencti_id = get_config_variable("OPENCTI_TOKEN", ["opencti", "token"], config) self.update_existing_data = get_config_variable( "OPENCTI_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config) self.proxy_http = get_config_variable("PROXY_HTTP", ["opencti", "proxy_http"], config) self.proxy_https = get_config_variable("PROXY_HTTPS", ["opencti", "proxy_https"], config) total_enabled = 0 if self.lastinfosec_cti_enabled: total_enabled += 1 if self.lastinfosec_cve_enabled: total_enabled += 1 if self.lastinfosec_tactic_enabled: total_enabled += 1 if total_enabled == 0: raise Exception("You must enable one feed") elif total_enabled > 1: raise Exception("You can enable only one feed per connector") self.api = OpenCTIApiClient(self.opencti_url, self.opencti_id)
def main(): params = demisto.params() args = demisto.args() api_key = params.get('apikey') base_url = params.get('base_url') if base_url.endswith('/'): base_url = base_url[:-1] indicator_types = params.get('indicator_types') max_fetch = params.get('max_indicator_to_fetch') tlp_color = params.get('tlp_color') if max_fetch: max_fetch = int(max_fetch) else: max_fetch = 500 try: client = OpenCTIApiClient(base_url, api_key, ssl_verify=params.get('insecure')) command = demisto.command() demisto.info("Command being called is {}".format(command)) # Switch case if command == "fetch-indicators": indicators = fetch_indicators_command(client, indicator_types, max_fetch, tlp_color=tlp_color) # we submit the indicators in batches for b in batch(indicators, batch_size=2000): demisto.createIndicators(b) elif command == "test-module": '''When setting up an OpenCTI Client it is checked that it is valid and allows requests to be sent. and if not he immediately sends an error''' fetch_indicators_command(client, indicator_types, max_fetch, is_test=True) return_outputs('ok') elif command == "opencti-get-indicators": return_results(get_indicators_command(client, args)) elif command == "opencti-reset-fetch-indicators": return_results(reset_last_run()) except Exception as e: return_error(f"Error [{e}]")
def __init__(self): config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) self.lastinfosec_url = get_config_variable("CONFIG_LIS_URL", ["lastinfosec", "api_url"], config) self.lastinfosec_apikey = get_config_variable( "CONFIG_LIS_APIKEY", ["lastinfosec", "api_key"], config) self.opencti_url = get_config_variable("OPENCTI_URL", ["opencti", "url"], config) self.opencti_id = get_config_variable("OPENCTI_TOKEN", ["opencti", "token"], config) self.update_existing_data = True self.api = OpenCTIApiClient(self.opencti_url, self.opencti_id)
def __post_init__(self) -> None: # Get configuration config_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "config.yml") if os.path.isfile(config_file_path): with open(config_file_path, "r") as f: config = yaml.load(f, Loader=yaml.FullLoader) else: config = {} # Load API config self.opencti_url = get_config_variable("OPENCTI_URL", ["opencti", "url"], config) self.opencti_token = get_config_variable("OPENCTI_TOKEN", ["opencti", "token"], config) self.opencti_ssl_verify = get_config_variable( "OPENCTI_SSL_VERIFY", ["opencti", "ssl_verify"], config, False, False) self.opencti_json_logging = get_config_variable( "OPENCTI_JSON_LOGGING", ["opencti", "json_logging"], config) # Load worker config self.log_level = get_config_variable("WORKER_LOG_LEVEL", ["worker", "log_level"], config) # Check if openCTI is available self.api = OpenCTIApiClient( url=self.opencti_url, token=self.opencti_token, log_level=self.log_level, ssl_verify=self.opencti_ssl_verify, json_logging=self.opencti_json_logging, ) # Configure logger numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError(f"Invalid log level: {self.log_level}") logging.basicConfig(level=numeric_level) # Initialize variables self.connectors: List[Any] = [] self.queues: List[Any] = []
def __init__(self): Analyzer.__init__(self) self.service = self.get_param('config.service', "search_observable", None) ssl = self.get_param('config.cert_check', True) names = self.get_param('config.name', None, 'No OpenCTI instance name given.') urls = self.get_param('config.url', None, 'No OpenCTI url given.') keys = self.get_param('config.key', None, 'No OpenCTI api key given.') if len(names) != len(urls) or len(urls) != len(keys): self.error( "Config error: please add a name, an url and a key for each OpenCTI instance." ) else: try: self.openctis = [] for i in range(len(names)): self.openctis.append({ "name": names[i], "url": urls[i], "api_client": OpenCTIApiClient( urls[i], keys[i], "error", ssl, ) }) except Exception as e: self.error(str(e))
# coding: utf-8 import os import yaml from pycti import OpenCTIApiClient # Load configuration config = yaml.load(open(os.path.dirname(__file__) + '/../config.yml')) # OpenCTI initialization opencti_api_client = OpenCTIApiClient(config['opencti']['url'], config['opencti']['token'], 'info') # Get observables and their context observables = opencti_api_client.get_stix_observables(10) for observable in observables: observable_value = observable['observable_value'] for relation in observable['stixRelations']: first_seen = relation['first_seen'] last_seen = relation['last_seen'] print('Observable with value "' + observable_value + '" (first_seen: ' + first_seen + ', last_seen: ' + last_seen + ')')
class Consumer(threading.Thread): def __init__( self, connector: Dict[str, Any], opencti_url: str, opencti_token: str, log_level: str, ) -> None: threading.Thread.__init__(self) self.api = OpenCTIApiClient(opencti_url, opencti_token, log_level) self.queue_name = connector["config"]["push"] self.pika_credentials = pika.PlainCredentials( connector["config"]["connection"]["user"], connector["config"]["connection"]["pass"], ) self.pika_parameters = pika.ConnectionParameters( connector["config"]["connection"]["host"], connector["config"]["connection"]["port"], "/", self.pika_credentials, ssl_options=pika.SSLOptions(create_ssl_context()) if connector["config"]["connection"]["use_ssl"] else None, ) self.pika_connection = pika.BlockingConnection(self.pika_parameters) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1) self.processing_count: int = 0 def get_id(self) -> Any: # pylint: disable=inconsistent-return-statements if hasattr(self, "_thread_id"): return self._thread_id # type: ignore # pylint: disable=no-member # pylint: disable=protected-access,redefined-builtin for id, thread in threading._active.items(): # type: ignore if thread is self: return id def terminate(self) -> None: thread_id = self.get_id() res = ctypes.pythonapi.PyThreadState_SetAsyncExc( thread_id, ctypes.py_object(SystemExit)) if res > 1: ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0) logging.info("Unable to kill the thread") def nack_message(self, channel: BlockingChannel, delivery_tag: int) -> None: if channel.is_open: logging.info("%s", f"Message (delivery_tag={delivery_tag}) rejected") channel.basic_nack(delivery_tag) else: logging.info( "%s", f"Message (delivery_tag={delivery_tag}) NOT rejected (channel closed)", ) def ack_message(self, channel: BlockingChannel, delivery_tag: int) -> None: if channel.is_open: logging.info( "%s", f"Message (delivery_tag={delivery_tag}) acknowledged") channel.basic_ack(delivery_tag) else: logging.info( "%s", (f"Message (delivery_tag={delivery_tag}) " "NOT acknowledged (channel closed)"), ) def stop_consume(self, channel: BlockingChannel) -> None: if channel.is_open: channel.stop_consuming() # Callable for consuming a message def _process_message( self, channel: BlockingChannel, method: Any, properties: None, # pylint: disable=unused-argument body: str, ) -> None: data = json.loads(body) logging.info( "%s", (f"Processing a new message (delivery_tag={method.delivery_tag})" ", launching a thread..."), ) thread = threading.Thread( target=self.data_handler, args=[self.pika_connection, channel, method.delivery_tag, data], ) thread.start() while thread.is_alive(): # Loop while the thread is processing self.pika_connection.sleep(0.05) logging.info("Message processed, thread terminated") # Data handling def data_handler( # pylint: disable=too-many-statements,too-many-locals self, connection: Any, channel: BlockingChannel, delivery_tag: str, data: Dict[str, Any], ) -> Optional[bool]: # Set the API headers applicant_id = data["applicant_id"] self.api.set_applicant_id_header(applicant_id) work_id = data["work_id"] if "work_id" in data else None # Execute the import self.processing_count += 1 content = "Unparseable" try: content = base64.b64decode(data["content"]).decode("utf-8") types = (data["entities_types"] if "entities_types" in data and len(data["entities_types"]) > 0 else None) update = data["update"] if "update" in data else False processing_count = self.processing_count if self.processing_count == PROCESSING_COUNT: processing_count = None # type: ignore self.api.stix2.import_bundle_from_json(content, update, types, processing_count) # Ack the message cb = functools.partial(self.ack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) if work_id is not None: self.api.work.report_expectation(work_id, None) self.processing_count = 0 return True except Timeout as te: logging.warning("%s", f"A connection timeout occurred: {{ {te} }}") # Platform is under heavy load: wait for unlock & retry almost indefinitely. sleep_jitter = round(random.uniform(10, 30), 2) time.sleep(sleep_jitter) self.data_handler(connection, channel, delivery_tag, data) return True except RequestException as re: logging.error("%s", f"A connection error occurred: {{ {re} }}") time.sleep(60) logging.info( "%s", f"Message (delivery_tag={delivery_tag}) NOT acknowledged") cb = functools.partial(self.nack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) self.processing_count = 0 return False except Exception as ex: # pylint: disable=broad-except error = str(ex) if "LockError" in error and self.processing_count < MAX_PROCESSING_COUNT: # Platform is under heavy load: # wait for unlock & retry almost indefinitely. sleep_jitter = round(random.uniform(10, 30), 2) time.sleep(sleep_jitter) self.data_handler(connection, channel, delivery_tag, data) elif ("MissingReferenceError" in error and self.processing_count < PROCESSING_COUNT): # In case of missing reference, wait & retry sleep_jitter = round(random.uniform(1, 3), 2) time.sleep(sleep_jitter) logging.info( "%s", (f"Message (delivery_tag={delivery_tag}) " f"reprocess (retry nb: {self.processing_count})"), ) self.data_handler(connection, channel, delivery_tag, data) else: # Platform does not know what to do and raises an error: # fail and acknowledge the message. logging.error(error) self.processing_count = 0 cb = functools.partial(self.ack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) if work_id is not None: self.api.work.report_expectation(work_id, { "error": error, "source": content }) return False return None def run(self) -> None: try: # Consume the queue logging.info("%s", f"Thread for queue {self.queue_name} started") self.channel.basic_consume( queue=self.queue_name, on_message_callback=self._process_message, ) self.channel.start_consuming() finally: self.channel.stop_consuming() logging.info("%s", f"Thread for queue {self.queue_name} terminated")
# coding: utf-8 from pycti import OpenCTIApiClient # Variables api_url = "https://demo.opencti.io" api_token = "2b4f29e3-5ea8-4890-8cf5-a76f61f1e2b2" # OpenCTI initialization opencti_api_client = OpenCTIApiClient(api_url, api_token) # Upload the file file = opencti_api_client.upload_file(file_name="./file.pdf", ) print(file)
def api_client(): return OpenCTIApiClient( "https://demo.opencti.io", "681b01f9-542d-4c8c-be0c-b6c850b087c8", ssl_verify=True, )
class Consumer(threading.Thread): def __init__(self, connector, opencti_url, opencti_token): threading.Thread.__init__(self) self.opencti_url = opencti_url self.opencti_token = opencti_token self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) self.queue_name = connector['config']['push'] self.pika_connection = pika.BlockingConnection( pika.URLParameters(connector['config']['uri'])) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1) def get_id(self): if hasattr(self, '_thread_id'): return self._thread_id for id, thread in threading._active.items(): if thread is self: return id def terminate(self): thread_id = self.get_id() res = ctypes.pythonapi.PyThreadState_SetAsyncExc( thread_id, ctypes.py_object(SystemExit)) if res > 1: ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0) logging.info('Unable to kill the thread') def ack_message(self, channel, delivery_tag): if channel.is_open: logging.info('Message (delivery_tag=' + str(delivery_tag) + ') acknowledged') channel.basic_ack(delivery_tag) else: logging.info('Message (delivery_tag=' + str(delivery_tag) + ') NOT acknowledged (channel closed)') pass def stop_consume(self, channel): if channel.is_open: channel.stop_consuming() # Callable for consuming a message def _process_message(self, channel, method, properties, body): data = json.loads(body) logging.info('Processing a new message (delivery_tag=' + str(method.delivery_tag) + '), launching a thread...') thread = threading.Thread( target=self.data_handler, args=[self.pika_connection, channel, method.delivery_tag, data]) thread.start() while thread.is_alive(): # Loop while the thread is processing self.pika_connection.sleep(1.0) logging.info('Message processed, thread terminated') # Data handling def data_handler(self, connection, channel, delivery_tag, data): job_id = data['job_id'] token = None if "token" in data: token = data["token"] try: content = base64.b64decode(data['content']).decode('utf-8') types = data['entities_types'] if 'entities_types' in data else [] update = data['update'] if 'update' in data else False if token: self.api.set_token(token) imported_data = self.api.stix2.import_bundle_from_json( content, update, types) self.api.set_token(self.opencti_token) if job_id is not None: messages = [] by_types = groupby(imported_data, key=lambda x: x['type']) for key, grp in by_types: messages.append(str(len(list(grp))) + ' imported ' + key) self.api.job.update_job(job_id, 'complete', messages) cb = functools.partial(self.ack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) return True except RequestException as re: logging.error('A connection error occurred: { ' + str(re) + ' }') logging.info('Message (delivery_tag=' + str(delivery_tag) + ') NOT acknowledged') cb = functools.partial(self.stop_consume, channel) connection.add_callback_threadsafe(cb) return False except Exception as e: logging.error('An unexpected error occurred: { ' + str(e) + ' }') cb = functools.partial(self.ack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) if job_id is not None: self.api.job.update_job(job_id, 'error', [str(e)]) return False def run(self): try: # Consume the queue logging.info('Thread for queue ' + self.queue_name + ' started') self.channel.basic_consume( queue=self.queue_name, on_message_callback=self._process_message) self.channel.start_consuming() finally: self.channel.stop_consuming() logging.info('Thread for queue ' + self.queue_name + ' terminated')
class Worker: def __init__(self): self.logs_all_queue = 'logs_all' self.consumer_threads = {} self.logger_threads = {} # Get configuration config_file_path = os.path.dirname( os.path.abspath(__file__)) + '/config.yml' config = yaml.load(open(config_file_path), Loader=yaml.FullLoader ) if os.path.isfile(config_file_path) else {} self.log_level = os.getenv( 'WORKER_LOG_LEVEL') or config['worker']['log_level'] self.opencti_url = os.getenv('OPENCTI_URL') or config['opencti']['url'] self.opencti_token = os.getenv( 'OPENCTI_TOKEN') or config['opencti']['token'] # Check if openCTI is available self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) # Configure logger numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: ' + self.log_level) logging.basicConfig(level=numeric_level) # Get logger config self.logger_config = self.api.get_logs_worker_config() # Initialize variables self.connectors = [] self.queues = [] # Start the main loop def start(self): while True: try: # Fetch queue configuration from API self.connectors = self.api.connector.list() self.queues = list( map(lambda x: x['config']['push'], self.connectors)) # Check if all queues are consumed for connector in self.connectors: queue = connector['config']['push'] if queue in self.consumer_threads: if not self.consumer_threads[queue].is_alive(): logging.info('Thread for queue ' + queue + ' not alive, creating a new one...') self.consumer_threads[queue] = Consumer( connector, self.api, self.opencti_token) self.consumer_threads[queue].start() else: self.consumer_threads[queue] = Consumer( connector, self.opencti_url, self.opencti_token) self.consumer_threads[queue].start() # Check logs queue is consumed if self.logs_all_queue in self.logger_threads: if not self.logger_threads[self.logs_all_queue].is_alive(): logging.info('Thread for queue ' + self.logs_all_queue + ' not alive, creating a new one...') self.logger_threads[self.logs_all_queue] = Logger( self.logger_config) self.logger_threads[self.logs_all_queue].start() else: self.logger_threads[self.logs_all_queue] = Logger( self.logger_config) self.logger_threads[self.logs_all_queue].start() # Check if some threads must be stopped for thread in list(self.consumer_threads): if thread not in self.queues: logging.info('Queue ' + thread + ' no longer exists, killing thread...') try: self.consumer_threads[thread].terminate() self.consumer_threads.pop(thread, None) except: logging.info( 'Unable to kill the thread for queue ' + thread + ', an operation is running, keep trying...') time.sleep(60) except KeyboardInterrupt: # Graceful stop for thread in self.consumer_threads.keys(): if thread not in self.queues: self.consumer_threads[thread].terminate() exit(0) except Exception as e: logging.error(e) time.sleep(60)
def api_client(): return OpenCTIApiClient( "https://demo.opencti.io", "31018a77-185f-4d86-9117-4f86a4a6be42", ssl_verify=True, )
class Consumer(threading.Thread): def __init__(self, connector, opencti_url, opencti_token): threading.Thread.__init__(self) self.opencti_url = opencti_url self.opencti_token = opencti_token self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token) self.queue_name = connector["config"]["push"] self.pika_credentials = pika.PlainCredentials( connector["config"]["connection"]["user"], connector["config"]["connection"]["pass"], ) self.pika_parameters = pika.ConnectionParameters( connector["config"]["connection"]["host"], connector["config"]["connection"]["port"], "/", self.pika_credentials, ) self.pika_connection = pika.BlockingConnection(self.pika_parameters) self.channel = self.pika_connection.channel() self.channel.basic_qos(prefetch_count=1) self.processing_count = 0 def get_id(self): if hasattr(self, "_thread_id"): return self._thread_id for id, thread in threading._active.items(): if thread is self: return id def terminate(self): thread_id = self.get_id() res = ctypes.pythonapi.PyThreadState_SetAsyncExc( thread_id, ctypes.py_object(SystemExit) ) if res > 1: ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0) logging.info("Unable to kill the thread") def nack_message(self, channel, delivery_tag): if channel.is_open: logging.info("Message (delivery_tag=" + str(delivery_tag) + ") rejected") channel.basic_nack(delivery_tag) else: logging.info( "Message (delivery_tag=" + str(delivery_tag) + ") NOT rejected (channel closed)" ) pass def ack_message(self, channel, delivery_tag): if channel.is_open: logging.info( "Message (delivery_tag=" + str(delivery_tag) + ") acknowledged" ) channel.basic_ack(delivery_tag) else: logging.info( "Message (delivery_tag=" + str(delivery_tag) + ") NOT acknowledged (channel closed)" ) pass def stop_consume(self, channel): if channel.is_open: channel.stop_consuming() # Callable for consuming a message def _process_message(self, channel, method, properties, body): data = json.loads(body) logging.info( "Processing a new message (delivery_tag=" + str(method.delivery_tag) + "), launching a thread..." ) thread = threading.Thread( target=self.data_handler, args=[self.pika_connection, channel, method.delivery_tag, data], ) thread.start() while thread.is_alive(): # Loop while the thread is processing self.pika_connection.sleep(0.05) logging.info("Message processed, thread terminated") # Data handling def data_handler(self, connection, channel, delivery_tag, data): # Set the API headers applicant_id = data["applicant_id"] self.api.set_applicant_id_header(applicant_id) work_id = data["work_id"] if "work_id" in data else None # Execute the import self.processing_count += 1 content = "Unparseable" try: content = base64.b64decode(data["content"]).decode("utf-8") types = ( data["entities_types"] if "entities_types" in data and len(data["entities_types"]) > 0 else None ) update = data["update"] if "update" in data else False processing_count = self.processing_count if self.processing_count == PROCESSING_COUNT: processing_count = None self.api.stix2.import_bundle_from_json( content, update, types, processing_count ) # Ack the message cb = functools.partial(self.ack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) if work_id is not None: self.api.work.report_expectation(work_id, None) self.processing_count = 0 return True except Timeout as te: logging.warn("A connection timeout occurred: { " + str(te) + " }") # Platform is under heavy load, wait for unlock & retry almost indefinitely sleep_jitter = round(random.uniform(10, 30), 2) time.sleep(sleep_jitter) self.data_handler(connection, channel, delivery_tag, data) return True except RequestException as re: logging.error("A connection error occurred: { " + str(re) + " }") time.sleep(60) logging.info( "Message (delivery_tag=" + str(delivery_tag) + ") NOT acknowledged" ) cb = functools.partial(self.nack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) self.processing_count = 0 return False except Exception as ex: error = str(ex) if "LockError" in error and self.processing_count < MAX_PROCESSING_COUNT: # Platform is under heavy load, wait for unlock & retry almost indefinitely sleep_jitter = round(random.uniform(10, 30), 2) time.sleep(sleep_jitter) self.data_handler(connection, channel, delivery_tag, data) elif ( "MissingReferenceError" in error and self.processing_count < PROCESSING_COUNT ): # In case of missing reference, wait & retry sleep_jitter = round(random.uniform(1, 3), 2) time.sleep(sleep_jitter) logging.info( "Message (delivery_tag=" + str(delivery_tag) + ") reprocess (retry nb: " + str(self.processing_count) + ")" ) self.data_handler(connection, channel, delivery_tag, data) else: # Platform does not know what to do and raises an error, fail and acknowledge the message logging.error(str(ex)) self.processing_count = 0 cb = functools.partial(self.ack_message, channel, delivery_tag) connection.add_callback_threadsafe(cb) if work_id is not None: self.api.work.report_expectation( work_id, {"error": str(ex), "source": content} ) return False def run(self): try: # Consume the queue logging.info("Thread for queue " + self.queue_name + " started") self.channel.basic_consume( queue=self.queue_name, on_message_callback=self._process_message ) self.channel.start_consuming() finally: self.channel.stop_consuming() logging.info("Thread for queue " + self.queue_name + " terminated")
class Worker: def __init__(self): # Get configuration config_file_path = os.path.dirname( os.path.abspath(__file__)) + '/config.yml' if os.path.isfile(config_file_path): config = yaml.load(open(config_file_path), Loader=yaml.FullLoader) self.type = config['worker']['type'] self.log_level = config['worker']['log_level'] self.opencti_url = config['opencti']['url'] self.opencti_token = config['opencti']['token'] self.rabbitmq_hostname = config['rabbitmq']['hostname'] self.rabbitmq_port = config['rabbitmq']['port'] self.rabbitmq_port_management = config['rabbitmq'][ 'port_management'] self.rabbitmq_management_ssl = config['rabbitmq']['management_ssl'] self.rabbitmq_username = config['rabbitmq']['username'] self.rabbitmq_password = config['rabbitmq']['password'] else: self.type = os.getenv('WORKER_TYPE', 'import') self.log_level = os.getenv('WORKER_LOG_LEVEL', 'info') self.opencti_url = os.getenv('OPENCTI_URL', 'http://localhost:4000') self.opencti_token = os.getenv('OPENCTI_TOKEN', 'ChangeMe') self.rabbitmq_hostname = os.getenv('RABBITMQ_HOSTNAME', 'localhost') self.rabbitmq_port = os.getenv('RABBITMQ_PORT', 5672) self.rabbitmq_port_management = os.getenv( 'RABBITMQ_PORT_MANAGEMENT', 15672) self.rabbitmq_management_ssl = os.getenv('RABBITMQ_MANAGEMENT_SSL', "false") == "true" self.rabbitmq_username = os.getenv('RABBITMQ_USERNAME', 'guest') self.rabbitmq_password = os.getenv('RABBITMQ_PASSWORD', 'guest') # Check configuration if self.type == 'import': self.DEFAULT_QUEUE_NAME = 'import-platform' self.DEFAULT_ROUTING_KEY = 'import.platform' self.CONNECTOR_QUEUE_PREFIX = 'import-connectors-' elif self.type == 'export': self.DEFAULT_QUEUE_NAME = 'export-platform' self.DEFAULT_ROUTING_KEY = 'export.platform' self.CONNECTOR_QUEUE_PREFIX = 'export-connectors-' else: raise ValueError('Type not supported: ' + self.type) if len( self.opencti_token ) == 0 or self.opencti_token == '<Must be the same as APP__ADMIN__TOKEN>': raise ValueError('Configuration not found') # Configure logger numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: ' + self.log_level) logging.basicConfig(level=numeric_level) # Initialize OpenCTI client self.opencti_api_client = OpenCTIApiClient(self.opencti_url, self.opencti_token) # Connect to RabbitMQ self.connection = self._connect() self.channel = self.connection.channel() self._create_exchange() self._create_default_queue() # Connect to RabbitMQ def _connect(self): try: credentials = pika.PlainCredentials(self.rabbitmq_username, self.rabbitmq_password) parameters = pika.ConnectionParameters(self.rabbitmq_hostname, self.rabbitmq_port, '/', credentials) return pika.BlockingConnection(parameters) except: logging.error( 'Unable to connect to RabbitMQ with the given parameters') return None # Create the exchange def _create_exchange(self): self.channel.exchange_declare(exchange=EXCHANGE_NAME, exchange_type='direct', durable=True) # Create the default queue for import coming from the platform def _create_default_queue(self): self.channel.queue_declare(self.DEFAULT_QUEUE_NAME, durable=True) self.channel.queue_bind(queue=self.DEFAULT_QUEUE_NAME, exchange=EXCHANGE_NAME, routing_key=self.DEFAULT_ROUTING_KEY) def _reconnect(self): self.connection = self._connect() self.channel = self.connection.channel() self._create_exchange() self._create_default_queue() logging.info('Successfully connected to RabbitMQ') # List all connectors queues def _list_connectors_queues(self): try: # Get all existing queues scheme = 'https' if self.rabbitmq_management_ssl else 'http' queues_request = requests.get( scheme + '://' + self.rabbitmq_hostname + ':' + str(self.rabbitmq_port_management) + '/api/queues', auth=HTTPBasicAuth(self.rabbitmq_username, self.rabbitmq_password)) queues_request.raise_for_status() queues = queues_request.json() queues_list = [] for queue in queues: if self.CONNECTOR_QUEUE_PREFIX in queue['name']: queues_list.append(queue['name']) return queues_list except: logging.error('Unable to list queues and bind them') return [] # Callable for consuming a message def _process_message(self, body): try: data = json.loads(body) logging.info('Received a new message of type "' + data['type'] + '"') if data['type'] == 'stix2-bundle' or data['type'] == 'stix2bundle': content = base64.b64decode(data['content']).decode('utf-8') self.opencti_api_client.stix2_import_bundle( content, True, data['entities_types'] if 'entities_types' in data else []) if data['type'] == 'stix2-bundle-simple': bundle = self.opencti_api_client.stix2_export_entity( data['entity_type'], data['entity_id'], 'simple') if bundle is not None: bundle = base64.b64encode( bytes(json.dumps(bundle, indent=4), 'utf-8')).decode('utf-8') self.opencti_api_client.push_stix_domain_entity_export( data['entity_id'], data['export_id'], bundle) if data["type"] == 'stix2-bundle-full': bundle = self.opencti_api_client.stix2_export_entity( data['entity_type'], data['entity_id'], 'full') if bundle is not None: bundle = base64.b64encode( bytes(json.dumps(bundle, indent=4), 'utf-8')).decode('utf-8') self.opencti_api_client.push_stix_domain_entity_export( data['entity_id'], data['export_id'], bundle) except Exception as e: logging.error('An unexpected error occurred: { ' + str(e) + ' }') return False # Consume the queues during 1 minute def _consume(self): timeout = time.time() + 60 queues = self._list_connectors_queues() queues.append(self.DEFAULT_QUEUE_NAME) logging.info('Worker has been loaded with type: ' + self.type) while True: if time.time() > timeout: break for queue in queues: method, header, body = self.channel.basic_get(queue=queue) if method: self._process_message(body) self.channel.basic_ack(delivery_tag=method.delivery_tag) time.sleep(1) # Start the main loop def start(self): self._reconnect() try: while True: self._consume() time.sleep(1) except KeyboardInterrupt: exit(0) except: raise ValueError('Unable to start the worker')
def api_client(): return OpenCTIApiClient( "https://demo.opencti.io", "d1b42111-a7fb-4830-846a-6a91c16b0084", ssl_verify=True, )
def api_client(): return OpenCTIApiClient( "https://demo.opencti.io", "e9613371-2fcc-4d7a-9ba1-ac93df589e5f", ssl_verify=True, )
def api_client(): return OpenCTIApiClient( "https://demo.opencti.io", "c7a8ab7d-2d49-46a2-af9b-89f7f4282a3c", ssl_verify=True, )
# coding: utf-8 from pycti import OpenCTIApiClient # Variables api_url = "https://demo.opencti.io" api_token = "YOUR_TOKEN" # OpenCTI initialization opencti_api_client = OpenCTIApiClient(api_url, api_token) process = opencti_api_client.stix_cyber_observable.create( observableData={ "type": "Process", "x_opencti_description": "A process", "cwd": "C:\Process.exe", "pid": "19000", "command_line": "--run exe", "x_opencti_score": 90, } ) print(process)
def api_client(): return OpenCTIApiClient( "https://demo.opencti.io", "01327cdd-9efd-4bea-b638-29bf840b8d48", ssl_verify=True, )