class Mitre:
def __init__(self, config):
# Initialize config
self.config = config
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/mitre.log',
True
)
def set_config(self, config):
self.config = config
def get_config(self):
return self.config['mitre']
def run(self):
http = urllib3.PoolManager()
with http.request('GET', self.config['mitre']['enterprise_file_url'], preload_content=False) as r, open('./enterprise.json', 'wb') as out_file:
shutil.copyfileobj(r, out_file)
self.opencti.stix2_import_bundle_from_file('./enterprise.json', True, self.config['mitre']['entities'])
os.remove('./enterprise.json')
def __init__(self, verbose=True):
# Initialize logger
self.logger = Logger(
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log')
# Load configuration
self.config = yaml.load(
open(os.path.dirname(os.path.abspath(__file__)) + '/config.yml'))
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log',
self.config['opencti']['verbose'])
# Initialize the RabbitMQ connection
credentials = pika.PlainCredentials(
self.config['rabbitmq']['username'],
self.config['rabbitmq']['password'])
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self.config['rabbitmq']['hostname'],
port=self.config['rabbitmq']['port'],
virtual_host='/',
credentials=credentials))
self.channel = connection.channel()
self.channel.exchange_declare(exchange='opencti',
exchange_type='topic',
durable=True)
self.channel.queue_declare('opencti-export', durable=True)
self.channel.queue_bind(exchange='opencti',
queue='opencti-export',
routing_key='export.*.*')
def __init__(self, config):
# Initialize config
self.config = config
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/openctidata.log',
True
)
class WorkerImport:
def __init__(self, verbose=True):
# Initialize logger
self.logger = Logger(
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log')
# Load configuration
self.config = yaml.load(
open(os.path.dirname(os.path.abspath(__file__)) + '/config.yml'))
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log',
self.config['opencti']['verbose'])
# Initialize the RabbitMQ connection
credentials = pika.PlainCredentials(
self.config['rabbitmq']['username'],
self.config['rabbitmq']['password'])
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self.config['rabbitmq']['hostname'],
port=self.config['rabbitmq']['port'],
virtual_host='/',
credentials=credentials))
self.channel = connection.channel()
self.channel.exchange_declare(exchange='opencti',
exchange_type='topic',
durable=True)
self.channel.queue_declare('opencti-import', durable=True)
self.channel.queue_bind(exchange='opencti',
queue='opencti-import',
routing_key='import.*.*')
def import_action(self, ch, method, properties, body):
try:
data = json.loads(body)
self.logger.log('Receiving new action of type: { ' + data['type'] +
' }')
if data['type'] == 'import.stix2.bundle':
self.opencti.stix2_import_bundle(
base64.b64decode(data['content']).decode('utf-8'))
except Exception as e:
self.logger.log('An unexpected error occurred: { ' + str(e) + ' }')
return False
def consume(self):
self.channel.basic_consume(queue='opencti-import',
on_message_callback=self.import_action,
auto_ack=True)
self.channel.start_consuming()
def __init__(self, config):
# Initialize config
self.config = config
# Initialize MISP
self.misp = PyMISP(self.config['misp']['url'], self.config['misp']['key'], False, 'json')
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/misp.log',
True
)
def __init__(self, verbose=True):
# Initialize logger
self.logger = Logger(
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log')
# Load configuration
self.config = yaml.load(
open(os.path.dirname(os.path.abspath(__file__)) + '/config.yml'))
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log',
self.config['opencti']['verbose'])
class Openctidata:
def __init__(self, config):
# Initialize config
self.config = config
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/openctidata.log',
True
)
def set_config(self, config):
self.config = config
def get_config(self):
return self.config['openctidata']
def run(self):
if 'sector' in self.config['openctidata']['entities']:
sectors_data = urllib.request.urlopen(self.config['openctidata']['sectors_file_url']).read()
sectors = json.loads(sectors_data)
for sector in sectors:
sector_id = self.opencti.create_identity_if_not_exists(
'Sector',
sector['name'],
sector['description']
)['id']
self.opencti.update_stix_domain_entity_field(
sector_id,
'name', sector['name']
)
self.opencti.update_stix_domain_entity_field(
sector_id,
'description',
sector['description']
)
for subsector in sector['subsectors']:
subsector_id = self.opencti.create_identity_if_not_exists(
'Sector',
subsector['name'],
subsector['description']
)['id']
self.opencti.update_stix_domain_entity_field(
subsector_id,
'name',
subsector['name']
)
self.opencti.update_stix_domain_entity_field(
subsector_id,
'description',
subsector['description']
)
# Temporary for fixing multiple relations of previous version
old_relations = self.opencti.get_stix_relations(sector_id, subsector_id)
for old_relation in old_relations:
self.opencti.delete_relation(old_relation['id'])
self.opencti.create_relation_if_not_exists(
sector_id,
'sector',
subsector_id,
'sector',
'gathering',
'Subsector from OpenCTI connector',
'1900-01-01T00:00:00.000Z',
'1900-01-01T00:00:00.000Z',
5
)
class Misp:
def __init__(self, config):
# Initialize config
self.config = config
# Initialize MISP
self.misp = PyMISP(self.config['misp']['url'], self.config['misp']['key'], False, 'json')
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/misp.log',
True
)
def set_config(self, config):
self.config = config
def get_config(self):
return self.config['misp']
def run(self):
result = self.misp.search('events', tags=[self.config['misp']['tag']])
for event in result['response']:
# Default values
author_id = self.opencti.create_identity_if_not_exists(
'Organization',
event['Event']['Orgc']['name'],
''
)['id']
event_threats = self.prepare_threats(event['Event']['Galaxy'])
event_markings = self.resolve_markings(event['Event']['Tag'])
# Create the external reference of the event
external_reference_id = self.opencti.create_external_reference_if_not_exists(
self.config['misp']['name'],
self.config['misp']['url'] + '/events/view/' + event['Event']['uuid'],
event['Event']['uuid'])['id']
# Create the report of the event
report_id = self.opencti.create_report_if_not_exists_from_external_reference(
external_reference_id,
event['Event']['info'],
event['Event']['info'],
parse(event['Event']['date']).strftime('%Y-%m-%dT%H:%M:%SZ'),
'external'
)['id']
self.opencti.update_stix_domain_entity_created_by_ref(report_id, author_id)
# Add markings to report
for marking in event_markings:
self.opencti.add_marking_definition_if_not_exists(report_id, marking)
# Add entities to report
for threat in event_threats:
self.opencti.add_object_ref_to_report_if_not_exists(report_id, threat['id'])
# Get all attributes
for attribute in event['Event']['Attribute']:
self.process_attribute(report_id, author_id, event_threats, event_markings, attribute)
# get all attributes of object
for object in event['Event']['Object']:
for attribute in object['Attribute']:
self.process_attribute(report_id, author_id, event_threats, event_markings, attribute)
self.misp.tag(event['Event']['uuid'], 'OpenCTI: Imported')
self.misp.untag(event['Event']['uuid'], self.config['misp']['tag'])
def process_attribute(self, report_id, author_id, event_threats, event_markings, attribute):
type = self.resolve_type(attribute['type'], attribute['value'])
if type is not None:
# Default values
attribute_threats = self.prepare_threats(attribute['Galaxy'])
if 'Tag' in attribute:
attribute_markings = self.resolve_markings(attribute['Tag'])
else:
attribute_markings = []
# Check necessary threats
if len(event_threats) == 0 and len(attribute_threats) == 0:
attribute_threats.append({'type': 'Threat-Actor', 'id': self.opencti.create_threat_actor_if_not_exists(
'Unknown threats',
'All unknown threats are representing by this pseudo threat actors.'
)['id']})
# Create observable
observable_id = self.opencti.create_stix_observable_if_not_exists(
type,
attribute['value'],
attribute['comment']
)['id']
self.opencti.update_stix_observable_created_by_ref(observable_id, author_id)
# Add observable to report
self.opencti.add_object_ref_to_report_if_not_exists(report_id, observable_id)
# Add threats to reports
for threat in attribute_threats:
self.opencti.add_object_ref_to_report_if_not_exists(report_id, threat['id'])
# Add threats to observables
for threat in event_threats:
relation_id = self.opencti.create_relation_if_not_exists(
observable_id,
'Observable',
threat['id'],
threat['type'],
'indicates',
attribute['comment'],
datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime('%Y-%m-%dT%H:%M:%SZ'),
datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime('%Y-%m-%dT%H:%M:%SZ'),
2
)['id']
self.opencti.add_object_ref_to_report_if_not_exists(report_id, relation_id)
for threat in attribute_threats:
relation_id = self.opencti.create_relation_if_not_exists(
observable_id,
'Observable',
threat['id'],
threat['type'],
'indicates',
attribute['comment'],
datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime('%Y-%m-%dT%H:%M:%SZ'),
datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime('%Y-%m-%dT%H:%M:%SZ'),
2
)['id']
self.opencti.add_object_ref_to_report_if_not_exists(report_id, relation_id)
# Add markings to observable
if len(attribute_markings) > 0:
for marking in attribute_markings:
self.opencti.add_marking_definition_if_not_exists(observable_id, marking)
self.opencti.add_marking_definition_if_not_exists(observable_id, marking)
else:
for marking in event_markings:
self.opencti.add_marking_definition_if_not_exists(observable_id, marking)
def prepare_threats(self, galaxies):
threats = []
for galaxy in galaxies:
if galaxy['name'] == 'Intrusion Set':
for galaxy_entity in galaxy['GalaxyCluster']:
threats.append({
'type': 'Intrusion-Set',
'id': self.opencti.create_intrusion_set_if_not_exists(
galaxy_entity['value'],
galaxy_entity['description']
)['id']
})
if galaxy['name'] == 'Threat Actor':
for galaxy_entity in galaxy['GalaxyCluster']:
threats.append({
'type': 'Intrusion-Set',
'id': self.opencti.create_intrusion_set_if_not_exists(
galaxy_entity['value'],
galaxy_entity['description']
)['id']
})
if galaxy['name'] == 'Malware':
for galaxy_entity in galaxy['GalaxyCluster']:
threats.append({
'type': 'Malware',
'id': self.opencti.create_malware_if_not_exists(
galaxy_entity['value'],
galaxy_entity['description']
)['id']
})
if galaxy['name'] == 'Tool':
for galaxy_entity in galaxy['GalaxyCluster']:
threats.append({
'type': 'Tool',
'id': self.opencti.create_tool_if_not_exists(
galaxy_entity['value'],
galaxy_entity['description']
)['id']
})
return threats
def resolve_type(self, type, value):
types = {
'ip-src': 'IPv4-Addr',
'ip-dst': 'IPv4-Addr',
'domain': 'Domain',
'hostname': 'Domain',
'url': 'URL',
'md5': 'File-MD5',
'sha1': 'File-SHA1',
'sha256': 'File-SHA256'
}
if type in types:
resolved_type = types[type]
if resolved_type == 'IPv4-Addr' and len(value) > 16:
return 'IPv6-Addr'
else:
return resolved_type
else:
return None
def resolve_markings(self, tags):
markings = []
for tag in tags:
if tag['name'] == 'tlp:white':
markings.append(self.opencti.get_marking_definition_by_definition('TLP', 'TLP:WHITE')['id'])
if tag['name'] == 'tlp:green':
markings.append(self.opencti.get_marking_definition_by_definition('TLP', 'TLP:GREEN')['id'])
if tag['name'] == 'tlp:amber':
markings.append(self.opencti.get_marking_definition_by_definition('TLP', 'TLP:AMBER')['id'])
if tag['name'] == 'tlp:red':
markings.append(self.opencti.get_marking_definition_by_definition('TLP', 'TLP:RED')['id'])
return markings
class ConnectorsScheduler:
def __init__(self, verbose=True):
# Initialize connectors object
self.connectors = {}
# Initialize logger
self.logger = Logger(
os.path.dirname(os.path.abspath(__file__)) + '/scheduler.log')
# Load configuration
self.config = yaml.load(
open(os.path.dirname(os.path.abspath(__file__)) + '/config.yml'))
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/scheduler.log',
self.config['opencti']['verbose'])
def send_stix2_bundle(self, bundle):
self.logger.log('Sending a message to the import workers')
# Prepare
message = {
'type': 'import.stix2.bundle',
'content': base64.b64encode(bundle.encode('utf-8')).decode('utf-8')
}
# Initialize the RabbitMQ connection
credentials = pika.PlainCredentials(
self.config['rabbitmq']['username'],
self.config['rabbitmq']['password'])
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self.config['rabbitmq']['hostname'],
port=self.config['rabbitmq']['port'],
virtual_host='/',
credentials=credentials))
channel = connection.channel()
channel.exchange_declare(exchange='opencti',
exchange_type='topic',
durable=True)
channel.basic_publish('opencti', 'import.stix2.bundle',
json.dumps(message))
connection.close()
def init_connectors(self):
self.logger.log('Configuring connectors')
connectors = self.opencti.get_connectors()
for connector in connectors:
try:
if connector['config'] is not None:
connector_config = json.loads(
base64.b64decode(connector['config']))
config = self.config
# First time, load code, create instance
if connector['identifier'] not in self.connectors:
config[connector['identifier']] = connector_config
connector_module = importlib.import_module(
'connectors.' + connector['identifier'] + '.' +
connector['identifier'])
connector_class = getattr(
connector_module,
connector['identifier'].capitalize())
self.connectors[connector['identifier']] = {
"config": config,
"instance": connector_class(config, self)
}
self.logger.log('Connector ' +
connector['identifier'] +
' initialized')
self.schedule_connectors()
# Code is already there, just reconfigure
else:
# If cron changed, reschedule
current_config = self.connectors[
connector['identifier']]['instance'].get_config()
if connector_config['cron'] != current_config['cron']:
self.logger.log('Connector ' +
connector['identifier'] +
' has to be rescheduled')
self.schedule_connectors()
# Reconfigure
config[connector['identifier']] = connector_config
self.connectors[connector['identifier']][
'instance'].set_config(config)
self.connectors[
connector['identifier']]['config'] = config
self.logger.log('Connector ' +
connector['identifier'] +
' configured')
# Manual trigger of the connector
if 'triggered' in connector_config and connector_config[
'triggered'] is True:
connector_config['triggered'] = False
self.opencti.update_connector_config(
connector['identifier'], connector_config)
self.run_connector(connector['identifier'])
except Exception as e:
self.logger.log('Unable to initialize ' +
connector['identifier'] + ': {' + str(e) + '}')
def run_connector(self, identifier):
try:
if 'active' in self.connectors[identifier]['config'][
identifier] and self.connectors[identifier]['config'][
identifier]['active'] is True:
self.logger.log('Running ' + identifier)
self.connectors[identifier]['instance'].run()
except Exception as e:
self.logger.log('Unable to run ' + identifier + ': {' + str(e) +
'}')
def schedule_connectors(self):
schedule.clear()
self.logger.log('Scheduling connectors')
schedule.every(1).minutes.do(self.init_connectors)
for identifier, connector in self.connectors.items():
connector_config = connector['instance'].get_config()
if connector_config['cron'] == 'realtime':
schedule.every(1).minutes.do(self.run_connector,
identifier=identifier)
elif connector_config['cron'] == 'hourly':
schedule.every(1).hours.do(self.run_connector,
identifier=identifier)
elif connector_config['cron'] == 'daily':
schedule.every().day.at("02:30").do(self.run_connector,
identifier=identifier)
elif connector_config['cron'] == 'weekly':
schedule.every().wednesday.at("04:30").do(
self.run_connector, identifier=identifier)
elif connector_config['cron'] == 'monthly':
schedule.every(30).day.at("04:30").do(self.run_connector,
identifier=identifier)
def run(self):
while True:
schedule.run_pending()
time.sleep(1)
def init(self):
self.init_connectors()
self.schedule_connectors()
self.run()
class WorkerExport:
def __init__(self, verbose=True):
# Initialize logger
self.logger = Logger(
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log')
# Load configuration
self.config = yaml.load(
open(os.path.dirname(os.path.abspath(__file__)) + '/config.yml'))
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/logs/worker.log',
self.config['opencti']['verbose'])
def export_action(self, ch, method, properties, body):
try:
data = json.loads(body)
self.logger.log('Receiving new action of type: { ' + data['type'] +
' }')
bundle = None
if data['type'] == 'export.stix2.simple':
bundle = self.opencti.stix2_export_entity(
data['entity_type'], data['entity_id'], 'simple')
if data["type"] == 'export.stix2.full':
bundle = self.opencti.stix2_export_entity(
data['entity_type'], data['entity_id'], 'full')
if bundle is not None:
bundle = base64.b64encode(
bytes(json.dumps(bundle, indent=4),
'utf-8')).decode('utf-8')
self.opencti.push_stix_domain_entity_export(
data['entity_id'], data['export_id'], bundle)
except Exception as e:
self.logger.log('An unexpected error occurred: { ' + str(e) + ' }')
return False
def consume(self):
# Initialize the RabbitMQ connection
credentials = pika.PlainCredentials(
self.config['rabbitmq']['username'],
self.config['rabbitmq']['password'])
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self.config['rabbitmq']['hostname'],
port=self.config['rabbitmq']['port'],
virtual_host='/',
credentials=credentials))
channel = connection.channel()
channel.exchange_declare(exchange='opencti',
exchange_type='topic',
durable=True)
channel.queue_declare('opencti-export', durable=True)
channel.queue_bind(exchange='opencti',
queue='opencti-export',
routing_key='export.*.*')
channel.basic_consume(queue='opencti-export',
on_message_callback=self.export_action,
auto_ack=True)
channel.start_consuming()
class ConnectorsScheduler:
def __init__(self, verbose=True):
# Initialize connectors object
self.connectors = {}
# Initialize logger
self.logger = Logger(os.path.dirname(os.path.abspath(__file__)) + '/scheduler.log')
# Load configuration
self.config = yaml.load(open(os.path.dirname(os.path.abspath(__file__)) + '/config.yml'))
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/scheduler.log',
self.config['opencti']['verbose']
)
def init_connectors(self):
self.logger.log('Configuring connectors')
connectors = self.opencti.get_connectors()
for connector in connectors:
if connector['config'] is not None:
connector_config = json.loads(base64.b64decode(connector['config']))
config = self.config
config[connector['identifier']] = connector_config
if connector['identifier'] not in self.connectors:
connector_module = importlib.import_module('connectors.' + connector['identifier'] + '.' + connector['identifier'])
connector_class = getattr(connector_module, connector['identifier'].capitalize())
self.connectors[connector['identifier']] = {"config": config, "instance": connector_class(config)}
self.logger.log('Connector ' + connector['identifier'] + ' initialized')
else:
self.connectors[connector['identifier']]['instance'].set_config(config)
self.connectors[connector['identifier']]['config'] = config
self.logger.log('Connector ' + connector['identifier'] + ' configured')
if 'triggered' in connector_config and connector_config['triggered'] is True:
connector_config['triggered'] = False
self.opencti.update_connector_config(connector['identifier'], connector_config)
self.run_connector(connector['identifier'])
def run_connector(self, identifier):
try:
if 'active' in self.connectors[identifier]['config'][identifier] and self.connectors[identifier]['config'][identifier]['active'] is True:
self.logger.log('Running ' + identifier)
self.connectors[identifier]['instance'].run()
except Exception as e:
self.logger.log('Unable to run ' + identifier + ': {' + str(e) + '}')
def run_connectors(self):
self.logger.log('Starting connectors')
schedule.every(1).minutes.do(self.init_connectors)
for identifier, connector in self.connectors.items():
connector_config = connector['instance'].get_config()
if connector_config['cron'] == 'realtime':
schedule.every(1).minutes.do(self.run_connector, identifier=identifier)
elif connector_config['cron'] == 'daily':
schedule.every().day.at("02:30").do(self.run_connector, identifier=identifier)
elif connector_config['cron'] == 'weekly':
schedule.every().wednesday.at("04:30").do(self.run_connector, identifier=identifier)
elif connector_config['cron'] == 'monthly':
schedule.every(30).day.at("04:30").do(self.run_connector, identifier=identifier)
while True:
schedule.run_pending()
time.sleep(1)
def init(self):
self.init_connectors()
self.run_connectors()
