def __init__(self):
self.logs_all_queue = "logs_all"
self.consumer_threads = {}
self.logger_threads = {}
# Get configuration
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.log_level = os.getenv(
"WORKER_LOG_LEVEL") or config["worker"]["log_level"]
self.opencti_url = os.getenv("OPENCTI_URL") or config["opencti"]["url"]
self.opencti_token = os.getenv(
"OPENCTI_TOKEN") or config["opencti"]["token"]
# Check if openCTI is available
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
# Configure logger
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError("Invalid log level: " + self.log_level)
logging.basicConfig(level=numeric_level)
# Get logger config
self.logger_config = self.api.get_logs_worker_config()
# Initialize variables
self.connectors = []
self.queues = []
def __init__(
self,
connector: Dict[str, Any],
opencti_url: str,
opencti_token: str,
log_level: str,
) -> None:
threading.Thread.__init__(self)
self.api = OpenCTIApiClient(opencti_url, opencti_token, log_level)
self.queue_name = connector["config"]["push"]
self.pika_credentials = pika.PlainCredentials(
connector["config"]["connection"]["user"],
connector["config"]["connection"]["pass"],
)
self.pika_parameters = pika.ConnectionParameters(
connector["config"]["connection"]["host"],
connector["config"]["connection"]["port"],
"/",
self.pika_credentials,
ssl_options=pika.SSLOptions(create_ssl_context())
if connector["config"]["connection"]["use_ssl"] else None,
)
self.pika_connection = pika.BlockingConnection(self.pika_parameters)
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.processing_count: int = 0
def __init__(self):
self.logs_all_queue = 'logs_all'
self.consumer_threads = {}
self.logger_threads = {}
# Get configuration
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.log_level = os.getenv(
'WORKER_LOG_LEVEL') or config['worker']['log_level']
self.opencti_url = os.getenv('OPENCTI_URL') or config['opencti']['url']
self.opencti_token = os.getenv(
'OPENCTI_TOKEN') or config['opencti']['token']
# Check if openCTI is available
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
# Configure logger
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: ' + self.log_level)
logging.basicConfig(level=numeric_level)
# Get logger config
self.logger_config = self.api.get_logs_worker_config()
# Initialize variables
self.connectors = []
self.queues = []
def __post_init__(self) -> None:
super().__init__()
self.api = OpenCTIApiClient(
url=self.opencti_url,
token=self.opencti_token,
log_level=self.log_level,
ssl_verify=self.ssl_verify,
json_logging=self.json_logging,
)
self.queue_name = self.connector["config"]["push"]
self.pika_credentials = pika.PlainCredentials(
self.connector["config"]["connection"]["user"],
self.connector["config"]["connection"]["pass"],
)
self.pika_parameters = pika.ConnectionParameters(
self.connector["config"]["connection"]["host"],
self.connector["config"]["connection"]["port"],
"/",
self.pika_credentials,
ssl_options=pika.SSLOptions(create_ssl_context())
if self.connector["config"]["connection"]["use_ssl"] else None,
)
self.pika_connection = pika.BlockingConnection(self.pika_parameters)
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.processing_count: int = 0
def __init__(self):
self.consumer_threads = {}
# Get configuration
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
if os.path.isfile(config_file_path):
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader)
self.type = config['worker']['type']
self.log_level = config['worker']['log_level']
self.opencti_url = config['opencti']['url']
self.opencti_token = config['opencti']['token']
self.rabbitmq_hostname = config['rabbitmq']['hostname']
self.rabbitmq_port = config['rabbitmq']['port']
self.rabbitmq_port_management = config['rabbitmq'][
'port_management']
self.rabbitmq_management_ssl = config['rabbitmq']['management_ssl']
self.rabbitmq_username = config['rabbitmq']['username']
self.rabbitmq_password = config['rabbitmq']['password']
else:
self.type = os.getenv('WORKER_TYPE', 'import')
self.log_level = os.getenv('WORKER_LOG_LEVEL', 'info')
self.opencti_url = os.getenv('OPENCTI_URL',
'http://localhost:4000')
self.opencti_token = os.getenv('OPENCTI_TOKEN', 'ChangeMe')
self.rabbitmq_hostname = os.getenv('RABBITMQ_HOSTNAME',
'localhost')
self.rabbitmq_port = os.getenv('RABBITMQ_PORT', 5672)
self.rabbitmq_port_management = os.getenv(
'RABBITMQ_PORT_MANAGEMENT', 15672)
self.rabbitmq_management_ssl = os.getenv('RABBITMQ_MANAGEMENT_SSL',
"false") == "true"
self.rabbitmq_username = os.getenv('RABBITMQ_USERNAME', 'guest')
self.rabbitmq_password = os.getenv('RABBITMQ_PASSWORD', 'guest')
# Check configuration
if len(self.opencti_token) == 0 or self.opencti_token == 'ChangeMe':
raise ValueError(
'Token configuration must be the same as APP__ADMIN__TOKEN')
# Check if openCTI is available
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
if not self.api.health_check():
raise ValueError('OpenCTI API seems down')
if self.type == 'import':
self.DEFAULT_QUEUE_NAME = 'import-platform'
self.DEFAULT_ROUTING_KEY = 'import.platform'
self.CONNECTOR_QUEUE_PREFIX = 'import-connectors-'
elif self.type == 'export':
self.DEFAULT_QUEUE_NAME = 'export-platform'
self.DEFAULT_ROUTING_KEY = 'export.platform'
self.CONNECTOR_QUEUE_PREFIX = 'export-connectors-'
# Configure logger
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: ' + self.log_level)
logging.basicConfig(level=numeric_level)
def __init__(self, connector, opencti_url, opencti_token):
threading.Thread.__init__(self)
self.opencti_url = opencti_url
self.opencti_token = opencti_token
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
self.queue_name = connector['config']['push']
self.pika_connection = pika.BlockingConnection(pika.URLParameters(connector['config']['uri']))
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
def __init__(self):
logging.getLogger("pika").setLevel(logging.ERROR)
welcome_art = text2art("OpenCTI migrator")
print(welcome_art)
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
self.config = (yaml.load(open(config_file_path),
Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.config.update({
key.lower(): value
for key, value in os.environ.items() if key.startswith("OPENCTI")
})
for config in mandatory_configs:
if config not in config:
raise ValueError("Missing configuration parameter: " + config)
# Test connection to the V3 API
self.opencti_api_client = OpenCTIApiClient(
self.config["opencti_v3_url"], self.config["opencti_v3_token"],
"error")
print("Checking access to OpenCTI version 3.3.2 instance... OK")
# Check RabbitMQ
self.connection = None
self.channel = None
self._rabbitmq_connect()
print("Checking access to the OpenCTI versio 4.X.X RabbitMQ... OK")
# Check if state already here or is creatable
self.state_file = os.path.dirname(
os.path.abspath(__file__)) + "/state.json"
print("Checking if the state file is writtable... OK")
self.get_state()
def get_client(opencti_url, opencti_token, ssl_verify, http_proxies):
client = OpenCTIApiClient(
opencti_url,
opencti_token,
log_level=opencti_config["log_level"],
ssl_verify=ssl_verify,
proxies=http_proxies,
)
return client
def __init__(self, connector, opencti_url, opencti_token, log_level):
threading.Thread.__init__(self)
self.api = OpenCTIApiClient(opencti_url, opencti_token, log_level)
self.queue_name = connector["config"]["push"]
self.pika_credentials = pika.PlainCredentials(
connector["config"]["connection"]["user"],
connector["config"]["connection"]["pass"],
)
self.pika_parameters = pika.ConnectionParameters(
connector["config"]["connection"]["host"],
connector["config"]["connection"]["port"],
"/",
self.pika_credentials,
)
self.pika_connection = pika.BlockingConnection(self.pika_parameters)
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.processing_count = 0
def __init__( # pylint: disable=too-many-arguments
self,
source_url,
source_token,
target_url,
target_token,
consuming_count,
start_timestamp,
live_stream_id=None,
):
self.source_url = source_url
self.source_token = source_token
self.target_url = target_url
self.target_token = target_token
self.live_stream_id = live_stream_id
self.count_number = 0
self.consuming_count = consuming_count
self.start_timestamp = start_timestamp
self.stream = None
# Source
config = {
"id": "673ba380-d229-4160-9213-ac5afdaabf96",
"type": "STREAM",
"name": "Synchronizer",
"scope": "synchronizer",
"confidence_level": 15,
"live_stream_id": self.live_stream_id,
"log_level": "info",
}
self.opencti_source_client = OpenCTIApiClient(source_url, source_token)
self.opencti_source_helper = OpenCTIConnectorHelper(
{
"opencti": {"url": self.source_url, "token": self.source_token},
"connector": config,
}
)
# Target
self.opencti_target_client = OpenCTIApiClient(target_url, target_token)
self.opencti_target_helper = OpenCTIConnectorHelper(
{
"opencti": {"url": self.target_url, "token": self.target_token},
"connector": config,
}
)
def main():
params = demisto.params()
args = demisto.args()
credentials = params.get('credentials', {})
api_key = credentials.get('password')
base_url = params.get('base_url').strip('/')
indicator_types = params.get('indicator_types', ['ALL'])
max_fetch = params.get('max_indicator_to_fetch')
tlp_color = params.get('tlp_color')
tags = argToList(params.get('feedTags'))
if max_fetch:
max_fetch = arg_to_number(max_fetch)
else:
max_fetch = 500
start = arg_to_number(args.get('score_start', 1))
end = arg_to_number(args.get('score_end', 100)) + 1 # type:ignore
score = None
if start or end:
score = [str(i) for i in range(start, end)] # type:ignore
try:
client = OpenCTIApiClient(base_url,
api_key,
ssl_verify=params.get('insecure'),
log_level='error')
command = demisto.command()
demisto.info(f"Command being called is {command}")
# Switch case
if command == "fetch-indicators":
fetch_indicators_command(client,
indicator_types,
max_fetch,
tlp_color=tlp_color,
score=score,
tags=tags)
elif command == "test-module":
'''When setting up an OpenCTI Client it is checked that it is valid and allows requests to be sent.
and if not he immediately sends an error'''
fetch_indicators_command(client,
indicator_types,
max_fetch,
is_test=True)
return_results('ok')
elif command == "opencti-get-indicators":
return_results(get_indicators_command(client, args))
elif command == "opencti-reset-fetch-indicators":
return_results(reset_last_run())
except Exception as e:
demisto.error(traceback.format_exc()) # print the traceback
return_error(f"Error:\n [{e}]")
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.lastinfosec_cti_url = "https://api.client.lastinfosec.com/v2/stix21/getbyminutes/{}?api_key={}&headers=false&platform=opencti"
self.lastinfosec_cve_url = "https://api.client.lastinfosec.com/v2/stix21/vulnerabilities/getlasthour?api_key={}&headers=false&platform=opencti"
self.lastinfosec_tactic_url = "https://api.client.lastinfosec.com/v2/stix21/tactic/getlast24hour?api_key={}&headers=false&platform=opencti"
self.lastinfosec_apikey = get_config_variable(
"CONFIG_LIS_APIKEY", ["lastinfosec", "api_key"], config)
self.lastinfosec_cti_enabled = get_config_variable(
"CONFIG_LIS_CTI_ENABLED", ["lastinfosec", "cti", "is_enabled"],
config)
self.lastinfosec_cti_interval = get_config_variable(
"CONFIG_LIS_CTI_INTERVAL", ["lastinfosec", "cti", "interval"],
config)
self.lastinfosec_cve_enabled = get_config_variable(
"CONFIG_LIS_CVE_ENABLED", ["lastinfosec", "cve", "is_enabled"],
config)
self.lastinfosec_tactic_enabled = get_config_variable(
"CONFIG_LIS_TACTIC_ENABLED",
["lastinfosec", "tactic", "is_enabled"], config)
self.opencti_url = get_config_variable("OPENCTI_URL",
["opencti", "url"], config)
self.opencti_id = get_config_variable("OPENCTI_TOKEN",
["opencti", "token"], config)
self.update_existing_data = get_config_variable(
"OPENCTI_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"], config)
self.proxy_http = get_config_variable("PROXY_HTTP",
["opencti", "proxy_http"],
config)
self.proxy_https = get_config_variable("PROXY_HTTPS",
["opencti", "proxy_https"],
config)
total_enabled = 0
if self.lastinfosec_cti_enabled:
total_enabled += 1
if self.lastinfosec_cve_enabled:
total_enabled += 1
if self.lastinfosec_tactic_enabled:
total_enabled += 1
if total_enabled == 0:
raise Exception("You must enable one feed")
elif total_enabled > 1:
raise Exception("You can enable only one feed per connector")
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_id)
def main():
params = demisto.params()
args = demisto.args()
api_key = params.get('apikey')
base_url = params.get('base_url')
if base_url.endswith('/'):
base_url = base_url[:-1]
indicator_types = params.get('indicator_types')
max_fetch = params.get('max_indicator_to_fetch')
tlp_color = params.get('tlp_color')
if max_fetch:
max_fetch = int(max_fetch)
else:
max_fetch = 500
try:
client = OpenCTIApiClient(base_url,
api_key,
ssl_verify=params.get('insecure'))
command = demisto.command()
demisto.info("Command being called is {}".format(command))
# Switch case
if command == "fetch-indicators":
indicators = fetch_indicators_command(client,
indicator_types,
max_fetch,
tlp_color=tlp_color)
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
elif command == "test-module":
'''When setting up an OpenCTI Client it is checked that it is valid and allows requests to be sent.
and if not he immediately sends an error'''
fetch_indicators_command(client,
indicator_types,
max_fetch,
is_test=True)
return_outputs('ok')
elif command == "opencti-get-indicators":
return_results(get_indicators_command(client, args))
elif command == "opencti-reset-fetch-indicators":
return_results(reset_last_run())
except Exception as e:
return_error(f"Error [{e}]")
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.lastinfosec_url = get_config_variable("CONFIG_LIS_URL",
["lastinfosec", "api_url"],
config)
self.lastinfosec_apikey = get_config_variable(
"CONFIG_LIS_APIKEY", ["lastinfosec", "api_key"], config)
self.opencti_url = get_config_variable("OPENCTI_URL",
["opencti", "url"], config)
self.opencti_id = get_config_variable("OPENCTI_TOKEN",
["opencti", "token"], config)
self.update_existing_data = True
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_id)
def __post_init__(self) -> None:
# Get configuration
config_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "config.yml")
if os.path.isfile(config_file_path):
with open(config_file_path, "r") as f:
config = yaml.load(f, Loader=yaml.FullLoader)
else:
config = {}
# Load API config
self.opencti_url = get_config_variable("OPENCTI_URL",
["opencti", "url"], config)
self.opencti_token = get_config_variable("OPENCTI_TOKEN",
["opencti", "token"], config)
self.opencti_ssl_verify = get_config_variable(
"OPENCTI_SSL_VERIFY", ["opencti", "ssl_verify"], config, False,
False)
self.opencti_json_logging = get_config_variable(
"OPENCTI_JSON_LOGGING", ["opencti", "json_logging"], config)
# Load worker config
self.log_level = get_config_variable("WORKER_LOG_LEVEL",
["worker", "log_level"], config)
# Check if openCTI is available
self.api = OpenCTIApiClient(
url=self.opencti_url,
token=self.opencti_token,
log_level=self.log_level,
ssl_verify=self.opencti_ssl_verify,
json_logging=self.opencti_json_logging,
)
# Configure logger
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f"Invalid log level: {self.log_level}")
logging.basicConfig(level=numeric_level)
# Initialize variables
self.connectors: List[Any] = []
self.queues: List[Any] = []
def __init__(self):
Analyzer.__init__(self)
self.service = self.get_param('config.service', "search_observable",
None)
ssl = self.get_param('config.cert_check', True)
names = self.get_param('config.name', None,
'No OpenCTI instance name given.')
urls = self.get_param('config.url', None, 'No OpenCTI url given.')
keys = self.get_param('config.key', None, 'No OpenCTI api key given.')
if len(names) != len(urls) or len(urls) != len(keys):
self.error(
"Config error: please add a name, an url and a key for each OpenCTI instance."
)
else:
try:
self.openctis = []
for i in range(len(names)):
self.openctis.append({
"name":
names[i],
"url":
urls[i],
"api_client":
OpenCTIApiClient(
urls[i],
keys[i],
"error",
ssl,
)
})
except Exception as e:
self.error(str(e))
# coding: utf-8
import os
import yaml
from pycti import OpenCTIApiClient
# Load configuration
config = yaml.load(open(os.path.dirname(__file__) + '/../config.yml'))
# OpenCTI initialization
opencti_api_client = OpenCTIApiClient(config['opencti']['url'], config['opencti']['token'], 'info')
# Get observables and their context
observables = opencti_api_client.get_stix_observables(10)
for observable in observables:
observable_value = observable['observable_value']
for relation in observable['stixRelations']:
first_seen = relation['first_seen']
last_seen = relation['last_seen']
print('Observable with value "' + observable_value + '" (first_seen: ' + first_seen + ', last_seen: ' + last_seen + ')')
class Consumer(threading.Thread):
def __init__(
self,
connector: Dict[str, Any],
opencti_url: str,
opencti_token: str,
log_level: str,
) -> None:
threading.Thread.__init__(self)
self.api = OpenCTIApiClient(opencti_url, opencti_token, log_level)
self.queue_name = connector["config"]["push"]
self.pika_credentials = pika.PlainCredentials(
connector["config"]["connection"]["user"],
connector["config"]["connection"]["pass"],
)
self.pika_parameters = pika.ConnectionParameters(
connector["config"]["connection"]["host"],
connector["config"]["connection"]["port"],
"/",
self.pika_credentials,
ssl_options=pika.SSLOptions(create_ssl_context())
if connector["config"]["connection"]["use_ssl"] else None,
)
self.pika_connection = pika.BlockingConnection(self.pika_parameters)
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.processing_count: int = 0
def get_id(self) -> Any: # pylint: disable=inconsistent-return-statements
if hasattr(self, "_thread_id"):
return self._thread_id # type: ignore # pylint: disable=no-member
# pylint: disable=protected-access,redefined-builtin
for id, thread in threading._active.items(): # type: ignore
if thread is self:
return id
def terminate(self) -> None:
thread_id = self.get_id()
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
thread_id, ctypes.py_object(SystemExit))
if res > 1:
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0)
logging.info("Unable to kill the thread")
def nack_message(self, channel: BlockingChannel,
delivery_tag: int) -> None:
if channel.is_open:
logging.info("%s",
f"Message (delivery_tag={delivery_tag}) rejected")
channel.basic_nack(delivery_tag)
else:
logging.info(
"%s",
f"Message (delivery_tag={delivery_tag}) NOT rejected (channel closed)",
)
def ack_message(self, channel: BlockingChannel, delivery_tag: int) -> None:
if channel.is_open:
logging.info(
"%s", f"Message (delivery_tag={delivery_tag}) acknowledged")
channel.basic_ack(delivery_tag)
else:
logging.info(
"%s",
(f"Message (delivery_tag={delivery_tag}) "
"NOT acknowledged (channel closed)"),
)
def stop_consume(self, channel: BlockingChannel) -> None:
if channel.is_open:
channel.stop_consuming()
# Callable for consuming a message
def _process_message(
self,
channel: BlockingChannel,
method: Any,
properties: None, # pylint: disable=unused-argument
body: str,
) -> None:
data = json.loads(body)
logging.info(
"%s",
(f"Processing a new message (delivery_tag={method.delivery_tag})"
", launching a thread..."),
)
thread = threading.Thread(
target=self.data_handler,
args=[self.pika_connection, channel, method.delivery_tag, data],
)
thread.start()
while thread.is_alive(): # Loop while the thread is processing
self.pika_connection.sleep(0.05)
logging.info("Message processed, thread terminated")
# Data handling
def data_handler( # pylint: disable=too-many-statements,too-many-locals
self,
connection: Any,
channel: BlockingChannel,
delivery_tag: str,
data: Dict[str, Any],
) -> Optional[bool]:
# Set the API headers
applicant_id = data["applicant_id"]
self.api.set_applicant_id_header(applicant_id)
work_id = data["work_id"] if "work_id" in data else None
# Execute the import
self.processing_count += 1
content = "Unparseable"
try:
content = base64.b64decode(data["content"]).decode("utf-8")
types = (data["entities_types"] if "entities_types" in data
and len(data["entities_types"]) > 0 else None)
update = data["update"] if "update" in data else False
processing_count = self.processing_count
if self.processing_count == PROCESSING_COUNT:
processing_count = None # type: ignore
self.api.stix2.import_bundle_from_json(content, update, types,
processing_count)
# Ack the message
cb = functools.partial(self.ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
if work_id is not None:
self.api.work.report_expectation(work_id, None)
self.processing_count = 0
return True
except Timeout as te:
logging.warning("%s", f"A connection timeout occurred: {{ {te} }}")
# Platform is under heavy load: wait for unlock & retry almost indefinitely.
sleep_jitter = round(random.uniform(10, 30), 2)
time.sleep(sleep_jitter)
self.data_handler(connection, channel, delivery_tag, data)
return True
except RequestException as re:
logging.error("%s", f"A connection error occurred: {{ {re} }}")
time.sleep(60)
logging.info(
"%s",
f"Message (delivery_tag={delivery_tag}) NOT acknowledged")
cb = functools.partial(self.nack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
self.processing_count = 0
return False
except Exception as ex: # pylint: disable=broad-except
error = str(ex)
if "LockError" in error and self.processing_count < MAX_PROCESSING_COUNT:
# Platform is under heavy load:
# wait for unlock & retry almost indefinitely.
sleep_jitter = round(random.uniform(10, 30), 2)
time.sleep(sleep_jitter)
self.data_handler(connection, channel, delivery_tag, data)
elif ("MissingReferenceError" in error
and self.processing_count < PROCESSING_COUNT):
# In case of missing reference, wait & retry
sleep_jitter = round(random.uniform(1, 3), 2)
time.sleep(sleep_jitter)
logging.info(
"%s",
(f"Message (delivery_tag={delivery_tag}) "
f"reprocess (retry nb: {self.processing_count})"),
)
self.data_handler(connection, channel, delivery_tag, data)
else:
# Platform does not know what to do and raises an error:
# fail and acknowledge the message.
logging.error(error)
self.processing_count = 0
cb = functools.partial(self.ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
if work_id is not None:
self.api.work.report_expectation(work_id, {
"error": error,
"source": content
})
return False
return None
def run(self) -> None:
try:
# Consume the queue
logging.info("%s", f"Thread for queue {self.queue_name} started")
self.channel.basic_consume(
queue=self.queue_name,
on_message_callback=self._process_message,
)
self.channel.start_consuming()
finally:
self.channel.stop_consuming()
logging.info("%s",
f"Thread for queue {self.queue_name} terminated")
# coding: utf-8 from pycti import OpenCTIApiClient # Variables api_url = "https://demo.opencti.io" api_token = "2b4f29e3-5ea8-4890-8cf5-a76f61f1e2b2" # OpenCTI initialization opencti_api_client = OpenCTIApiClient(api_url, api_token) # Upload the file file = opencti_api_client.upload_file(file_name="./file.pdf", ) print(file)
def api_client():
return OpenCTIApiClient(
"https://demo.opencti.io",
"681b01f9-542d-4c8c-be0c-b6c850b087c8",
ssl_verify=True,
)
class Consumer(threading.Thread):
def __init__(self, connector, opencti_url, opencti_token):
threading.Thread.__init__(self)
self.opencti_url = opencti_url
self.opencti_token = opencti_token
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
self.queue_name = connector['config']['push']
self.pika_connection = pika.BlockingConnection(
pika.URLParameters(connector['config']['uri']))
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
def get_id(self):
if hasattr(self, '_thread_id'):
return self._thread_id
for id, thread in threading._active.items():
if thread is self:
return id
def terminate(self):
thread_id = self.get_id()
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
thread_id, ctypes.py_object(SystemExit))
if res > 1:
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0)
logging.info('Unable to kill the thread')
def ack_message(self, channel, delivery_tag):
if channel.is_open:
logging.info('Message (delivery_tag=' + str(delivery_tag) +
') acknowledged')
channel.basic_ack(delivery_tag)
else:
logging.info('Message (delivery_tag=' + str(delivery_tag) +
') NOT acknowledged (channel closed)')
pass
def stop_consume(self, channel):
if channel.is_open:
channel.stop_consuming()
# Callable for consuming a message
def _process_message(self, channel, method, properties, body):
data = json.loads(body)
logging.info('Processing a new message (delivery_tag=' +
str(method.delivery_tag) + '), launching a thread...')
thread = threading.Thread(
target=self.data_handler,
args=[self.pika_connection, channel, method.delivery_tag, data])
thread.start()
while thread.is_alive(): # Loop while the thread is processing
self.pika_connection.sleep(1.0)
logging.info('Message processed, thread terminated')
# Data handling
def data_handler(self, connection, channel, delivery_tag, data):
job_id = data['job_id']
token = None
if "token" in data:
token = data["token"]
try:
content = base64.b64decode(data['content']).decode('utf-8')
types = data['entities_types'] if 'entities_types' in data else []
update = data['update'] if 'update' in data else False
if token:
self.api.set_token(token)
imported_data = self.api.stix2.import_bundle_from_json(
content, update, types)
self.api.set_token(self.opencti_token)
if job_id is not None:
messages = []
by_types = groupby(imported_data, key=lambda x: x['type'])
for key, grp in by_types:
messages.append(str(len(list(grp))) + ' imported ' + key)
self.api.job.update_job(job_id, 'complete', messages)
cb = functools.partial(self.ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
return True
except RequestException as re:
logging.error('A connection error occurred: { ' + str(re) + ' }')
logging.info('Message (delivery_tag=' + str(delivery_tag) +
') NOT acknowledged')
cb = functools.partial(self.stop_consume, channel)
connection.add_callback_threadsafe(cb)
return False
except Exception as e:
logging.error('An unexpected error occurred: { ' + str(e) + ' }')
cb = functools.partial(self.ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
if job_id is not None:
self.api.job.update_job(job_id, 'error', [str(e)])
return False
def run(self):
try:
# Consume the queue
logging.info('Thread for queue ' + self.queue_name + ' started')
self.channel.basic_consume(
queue=self.queue_name,
on_message_callback=self._process_message)
self.channel.start_consuming()
finally:
self.channel.stop_consuming()
logging.info('Thread for queue ' + self.queue_name + ' terminated')
class Worker:
def __init__(self):
self.logs_all_queue = 'logs_all'
self.consumer_threads = {}
self.logger_threads = {}
# Get configuration
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.log_level = os.getenv(
'WORKER_LOG_LEVEL') or config['worker']['log_level']
self.opencti_url = os.getenv('OPENCTI_URL') or config['opencti']['url']
self.opencti_token = os.getenv(
'OPENCTI_TOKEN') or config['opencti']['token']
# Check if openCTI is available
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
# Configure logger
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: ' + self.log_level)
logging.basicConfig(level=numeric_level)
# Get logger config
self.logger_config = self.api.get_logs_worker_config()
# Initialize variables
self.connectors = []
self.queues = []
# Start the main loop
def start(self):
while True:
try:
# Fetch queue configuration from API
self.connectors = self.api.connector.list()
self.queues = list(
map(lambda x: x['config']['push'], self.connectors))
# Check if all queues are consumed
for connector in self.connectors:
queue = connector['config']['push']
if queue in self.consumer_threads:
if not self.consumer_threads[queue].is_alive():
logging.info('Thread for queue ' + queue +
' not alive, creating a new one...')
self.consumer_threads[queue] = Consumer(
connector, self.api, self.opencti_token)
self.consumer_threads[queue].start()
else:
self.consumer_threads[queue] = Consumer(
connector, self.opencti_url, self.opencti_token)
self.consumer_threads[queue].start()
# Check logs queue is consumed
if self.logs_all_queue in self.logger_threads:
if not self.logger_threads[self.logs_all_queue].is_alive():
logging.info('Thread for queue ' +
self.logs_all_queue +
' not alive, creating a new one...')
self.logger_threads[self.logs_all_queue] = Logger(
self.logger_config)
self.logger_threads[self.logs_all_queue].start()
else:
self.logger_threads[self.logs_all_queue] = Logger(
self.logger_config)
self.logger_threads[self.logs_all_queue].start()
# Check if some threads must be stopped
for thread in list(self.consumer_threads):
if thread not in self.queues:
logging.info('Queue ' + thread +
' no longer exists, killing thread...')
try:
self.consumer_threads[thread].terminate()
self.consumer_threads.pop(thread, None)
except:
logging.info(
'Unable to kill the thread for queue ' +
thread +
', an operation is running, keep trying...')
time.sleep(60)
except KeyboardInterrupt:
# Graceful stop
for thread in self.consumer_threads.keys():
if thread not in self.queues:
self.consumer_threads[thread].terminate()
exit(0)
except Exception as e:
logging.error(e)
time.sleep(60)
def api_client():
return OpenCTIApiClient(
"https://demo.opencti.io",
"31018a77-185f-4d86-9117-4f86a4a6be42",
ssl_verify=True,
)
class Consumer(threading.Thread):
def __init__(self, connector, opencti_url, opencti_token):
threading.Thread.__init__(self)
self.opencti_url = opencti_url
self.opencti_token = opencti_token
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_token)
self.queue_name = connector["config"]["push"]
self.pika_credentials = pika.PlainCredentials(
connector["config"]["connection"]["user"],
connector["config"]["connection"]["pass"],
)
self.pika_parameters = pika.ConnectionParameters(
connector["config"]["connection"]["host"],
connector["config"]["connection"]["port"],
"/",
self.pika_credentials,
)
self.pika_connection = pika.BlockingConnection(self.pika_parameters)
self.channel = self.pika_connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.processing_count = 0
def get_id(self):
if hasattr(self, "_thread_id"):
return self._thread_id
for id, thread in threading._active.items():
if thread is self:
return id
def terminate(self):
thread_id = self.get_id()
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
thread_id, ctypes.py_object(SystemExit)
)
if res > 1:
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0)
logging.info("Unable to kill the thread")
def nack_message(self, channel, delivery_tag):
if channel.is_open:
logging.info("Message (delivery_tag=" + str(delivery_tag) + ") rejected")
channel.basic_nack(delivery_tag)
else:
logging.info(
"Message (delivery_tag="
+ str(delivery_tag)
+ ") NOT rejected (channel closed)"
)
pass
def ack_message(self, channel, delivery_tag):
if channel.is_open:
logging.info(
"Message (delivery_tag=" + str(delivery_tag) + ") acknowledged"
)
channel.basic_ack(delivery_tag)
else:
logging.info(
"Message (delivery_tag="
+ str(delivery_tag)
+ ") NOT acknowledged (channel closed)"
)
pass
def stop_consume(self, channel):
if channel.is_open:
channel.stop_consuming()
# Callable for consuming a message
def _process_message(self, channel, method, properties, body):
data = json.loads(body)
logging.info(
"Processing a new message (delivery_tag="
+ str(method.delivery_tag)
+ "), launching a thread..."
)
thread = threading.Thread(
target=self.data_handler,
args=[self.pika_connection, channel, method.delivery_tag, data],
)
thread.start()
while thread.is_alive(): # Loop while the thread is processing
self.pika_connection.sleep(0.05)
logging.info("Message processed, thread terminated")
# Data handling
def data_handler(self, connection, channel, delivery_tag, data):
# Set the API headers
applicant_id = data["applicant_id"]
self.api.set_applicant_id_header(applicant_id)
work_id = data["work_id"] if "work_id" in data else None
# Execute the import
self.processing_count += 1
content = "Unparseable"
try:
content = base64.b64decode(data["content"]).decode("utf-8")
types = (
data["entities_types"]
if "entities_types" in data and len(data["entities_types"]) > 0
else None
)
update = data["update"] if "update" in data else False
processing_count = self.processing_count
if self.processing_count == PROCESSING_COUNT:
processing_count = None
self.api.stix2.import_bundle_from_json(
content, update, types, processing_count
)
# Ack the message
cb = functools.partial(self.ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
if work_id is not None:
self.api.work.report_expectation(work_id, None)
self.processing_count = 0
return True
except Timeout as te:
logging.warn("A connection timeout occurred: { " + str(te) + " }")
# Platform is under heavy load, wait for unlock & retry almost indefinitely
sleep_jitter = round(random.uniform(10, 30), 2)
time.sleep(sleep_jitter)
self.data_handler(connection, channel, delivery_tag, data)
return True
except RequestException as re:
logging.error("A connection error occurred: { " + str(re) + " }")
time.sleep(60)
logging.info(
"Message (delivery_tag=" + str(delivery_tag) + ") NOT acknowledged"
)
cb = functools.partial(self.nack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
self.processing_count = 0
return False
except Exception as ex:
error = str(ex)
if "LockError" in error and self.processing_count < MAX_PROCESSING_COUNT:
# Platform is under heavy load, wait for unlock & retry almost indefinitely
sleep_jitter = round(random.uniform(10, 30), 2)
time.sleep(sleep_jitter)
self.data_handler(connection, channel, delivery_tag, data)
elif (
"MissingReferenceError" in error
and self.processing_count < PROCESSING_COUNT
):
# In case of missing reference, wait & retry
sleep_jitter = round(random.uniform(1, 3), 2)
time.sleep(sleep_jitter)
logging.info(
"Message (delivery_tag="
+ str(delivery_tag)
+ ") reprocess (retry nb: "
+ str(self.processing_count)
+ ")"
)
self.data_handler(connection, channel, delivery_tag, data)
else:
# Platform does not know what to do and raises an error, fail and acknowledge the message
logging.error(str(ex))
self.processing_count = 0
cb = functools.partial(self.ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
if work_id is not None:
self.api.work.report_expectation(
work_id, {"error": str(ex), "source": content}
)
return False
def run(self):
try:
# Consume the queue
logging.info("Thread for queue " + self.queue_name + " started")
self.channel.basic_consume(
queue=self.queue_name, on_message_callback=self._process_message
)
self.channel.start_consuming()
finally:
self.channel.stop_consuming()
logging.info("Thread for queue " + self.queue_name + " terminated")
class Worker:
def __init__(self):
# Get configuration
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
if os.path.isfile(config_file_path):
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader)
self.type = config['worker']['type']
self.log_level = config['worker']['log_level']
self.opencti_url = config['opencti']['url']
self.opencti_token = config['opencti']['token']
self.rabbitmq_hostname = config['rabbitmq']['hostname']
self.rabbitmq_port = config['rabbitmq']['port']
self.rabbitmq_port_management = config['rabbitmq'][
'port_management']
self.rabbitmq_management_ssl = config['rabbitmq']['management_ssl']
self.rabbitmq_username = config['rabbitmq']['username']
self.rabbitmq_password = config['rabbitmq']['password']
else:
self.type = os.getenv('WORKER_TYPE', 'import')
self.log_level = os.getenv('WORKER_LOG_LEVEL', 'info')
self.opencti_url = os.getenv('OPENCTI_URL',
'http://localhost:4000')
self.opencti_token = os.getenv('OPENCTI_TOKEN', 'ChangeMe')
self.rabbitmq_hostname = os.getenv('RABBITMQ_HOSTNAME',
'localhost')
self.rabbitmq_port = os.getenv('RABBITMQ_PORT', 5672)
self.rabbitmq_port_management = os.getenv(
'RABBITMQ_PORT_MANAGEMENT', 15672)
self.rabbitmq_management_ssl = os.getenv('RABBITMQ_MANAGEMENT_SSL',
"false") == "true"
self.rabbitmq_username = os.getenv('RABBITMQ_USERNAME', 'guest')
self.rabbitmq_password = os.getenv('RABBITMQ_PASSWORD', 'guest')
# Check configuration
if self.type == 'import':
self.DEFAULT_QUEUE_NAME = 'import-platform'
self.DEFAULT_ROUTING_KEY = 'import.platform'
self.CONNECTOR_QUEUE_PREFIX = 'import-connectors-'
elif self.type == 'export':
self.DEFAULT_QUEUE_NAME = 'export-platform'
self.DEFAULT_ROUTING_KEY = 'export.platform'
self.CONNECTOR_QUEUE_PREFIX = 'export-connectors-'
else:
raise ValueError('Type not supported: ' + self.type)
if len(
self.opencti_token
) == 0 or self.opencti_token == '<Must be the same as APP__ADMIN__TOKEN>':
raise ValueError('Configuration not found')
# Configure logger
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: ' + self.log_level)
logging.basicConfig(level=numeric_level)
# Initialize OpenCTI client
self.opencti_api_client = OpenCTIApiClient(self.opencti_url,
self.opencti_token)
# Connect to RabbitMQ
self.connection = self._connect()
self.channel = self.connection.channel()
self._create_exchange()
self._create_default_queue()
# Connect to RabbitMQ
def _connect(self):
try:
credentials = pika.PlainCredentials(self.rabbitmq_username,
self.rabbitmq_password)
parameters = pika.ConnectionParameters(self.rabbitmq_hostname,
self.rabbitmq_port, '/',
credentials)
return pika.BlockingConnection(parameters)
except:
logging.error(
'Unable to connect to RabbitMQ with the given parameters')
return None
# Create the exchange
def _create_exchange(self):
self.channel.exchange_declare(exchange=EXCHANGE_NAME,
exchange_type='direct',
durable=True)
# Create the default queue for import coming from the platform
def _create_default_queue(self):
self.channel.queue_declare(self.DEFAULT_QUEUE_NAME, durable=True)
self.channel.queue_bind(queue=self.DEFAULT_QUEUE_NAME,
exchange=EXCHANGE_NAME,
routing_key=self.DEFAULT_ROUTING_KEY)
def _reconnect(self):
self.connection = self._connect()
self.channel = self.connection.channel()
self._create_exchange()
self._create_default_queue()
logging.info('Successfully connected to RabbitMQ')
# List all connectors queues
def _list_connectors_queues(self):
try:
# Get all existing queues
scheme = 'https' if self.rabbitmq_management_ssl else 'http'
queues_request = requests.get(
scheme + '://' + self.rabbitmq_hostname + ':' +
str(self.rabbitmq_port_management) + '/api/queues',
auth=HTTPBasicAuth(self.rabbitmq_username,
self.rabbitmq_password))
queues_request.raise_for_status()
queues = queues_request.json()
queues_list = []
for queue in queues:
if self.CONNECTOR_QUEUE_PREFIX in queue['name']:
queues_list.append(queue['name'])
return queues_list
except:
logging.error('Unable to list queues and bind them')
return []
# Callable for consuming a message
def _process_message(self, body):
try:
data = json.loads(body)
logging.info('Received a new message of type "' + data['type'] +
'"')
if data['type'] == 'stix2-bundle' or data['type'] == 'stix2bundle':
content = base64.b64decode(data['content']).decode('utf-8')
self.opencti_api_client.stix2_import_bundle(
content, True,
data['entities_types'] if 'entities_types' in data else [])
if data['type'] == 'stix2-bundle-simple':
bundle = self.opencti_api_client.stix2_export_entity(
data['entity_type'], data['entity_id'], 'simple')
if bundle is not None:
bundle = base64.b64encode(
bytes(json.dumps(bundle, indent=4),
'utf-8')).decode('utf-8')
self.opencti_api_client.push_stix_domain_entity_export(
data['entity_id'], data['export_id'], bundle)
if data["type"] == 'stix2-bundle-full':
bundle = self.opencti_api_client.stix2_export_entity(
data['entity_type'], data['entity_id'], 'full')
if bundle is not None:
bundle = base64.b64encode(
bytes(json.dumps(bundle, indent=4),
'utf-8')).decode('utf-8')
self.opencti_api_client.push_stix_domain_entity_export(
data['entity_id'], data['export_id'], bundle)
except Exception as e:
logging.error('An unexpected error occurred: { ' + str(e) + ' }')
return False
# Consume the queues during 1 minute
def _consume(self):
timeout = time.time() + 60
queues = self._list_connectors_queues()
queues.append(self.DEFAULT_QUEUE_NAME)
logging.info('Worker has been loaded with type: ' + self.type)
while True:
if time.time() > timeout:
break
for queue in queues:
method, header, body = self.channel.basic_get(queue=queue)
if method:
self._process_message(body)
self.channel.basic_ack(delivery_tag=method.delivery_tag)
time.sleep(1)
# Start the main loop
def start(self):
self._reconnect()
try:
while True:
self._consume()
time.sleep(1)
except KeyboardInterrupt:
exit(0)
except:
raise ValueError('Unable to start the worker')
def api_client():
return OpenCTIApiClient(
"https://demo.opencti.io",
"d1b42111-a7fb-4830-846a-6a91c16b0084",
ssl_verify=True,
)
def api_client():
return OpenCTIApiClient(
"https://demo.opencti.io",
"e9613371-2fcc-4d7a-9ba1-ac93df589e5f",
ssl_verify=True,
)
def api_client():
return OpenCTIApiClient(
"https://demo.opencti.io",
"c7a8ab7d-2d49-46a2-af9b-89f7f4282a3c",
ssl_verify=True,
)
# coding: utf-8
from pycti import OpenCTIApiClient
# Variables
api_url = "https://demo.opencti.io"
api_token = "YOUR_TOKEN"
# OpenCTI initialization
opencti_api_client = OpenCTIApiClient(api_url, api_token)
process = opencti_api_client.stix_cyber_observable.create(
observableData={
"type": "Process",
"x_opencti_description": "A process",
"cwd": "C:\Process.exe",
"pid": "19000",
"command_line": "--run exe",
"x_opencti_score": 90,
}
)
print(process)
def api_client():
return OpenCTIApiClient(
"https://demo.opencti.io",
"01327cdd-9efd-4bea-b638-29bf840b8d48",
ssl_verify=True,
)