class ExportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
entity_id = data['entity_id']
file_name = data['file_name']
entity_type = data['entity_type']
export_type = data['export_type']
self.helper.log_info('Exporting: ' + entity_type + '/' + export_type +
'(' + entity_id + ') to ' + file_name)
bundle = self.helper.api.stix2_export_entity(entity_type, entity_id,
export_type)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info('Uploading: ' + entity_type + '/' + export_type +
'(' + entity_id + ') to ' + file_name)
self.helper.api.push_stix_domain_entity_export(entity_id, file_name,
json_bundle)
self.helper.log_info('Export done: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
return ['Export done']
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ImportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
file_path = data['file_path']
update = data['update']
file_uri = self.helper.opencti_url + file_path
self.helper.log_info('Importing the file ' + file_uri)
file_content = self.helper.api.fetch_opencti_file(file_uri)
bundles_sent = self.helper.send_stix2_bundle(file_content, None,
update)
return [
'Sent ' + str(len(bundles_sent)) +
' stix bundle(s) for worker import'
]
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ImportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
old_token = self.helper.api.get_token()
token = None
if "token" in data:
token = data["token"]
file_path = data["file_path"]
update = data["update"]
file_uri = self.helper.opencti_url + file_path
self.helper.log_info("Importing the file " + file_uri)
file_content = self.helper.api.fetch_opencti_file(file_uri)
if token:
self.helper.api.set_token(token)
bundles_sent = self.helper.send_stix2_bundle(file_content, None,
update)
self.helper.api.set_token(old_token)
return [
"Sent " + str(len(bundles_sent)) +
" stix bundle(s) for worker import"
]
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ImportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
self.helper.log_info("Importing the file " + file_uri)
file_content = self.helper.api.fetch_opencti_file(file_uri)
if data["file_mime"] == "text/xml":
initialize_options()
file_content = elevate(file_content)
bundles_sent = self.helper.send_stix2_bundle(file_content)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class VirustotalReference:
def __init__(self):
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
def create_reference(self, data):
virus_ref = self.helper.api.external_reference.create(
source_name="Virustotal " + data,
url="https://www.virustotal.com/gui/search/" + data,
)
return virus_ref
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_observable.read(id=entity_id)
observable_value = observable["observable_value"]
# self.helper.log_info("Creating virustotal reference for {}".format(observable_value))
created_reference = self.create_reference(observable_value)
# self.helper.log_info("External reference created with id {}".format(created_reference["id"]))
# self.helper.log_info("Attaching the reference to {}".format(entity_id))
self.helper.api.stix_entity.add_external_reference(
id=entity_id, external_reference_id=created_reference["id"]
)
def start(self):
self.helper.listen(self._process_message)
class InternalImportConnector:
def __init__(self, config_file_path: str, api_client: OpenCTIApiClient,
data: Dict):
# set OPENCTI settings from fixture
os.environ["OPENCTI_URL"] = api_client.api_url
os.environ["OPENCTI_TOKEN"] = api_client.api_token
os.environ["OPENCTI_SSL_VERIFY"] = str(api_client.ssl_verify)
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.data = data
def _process_message(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
# Downloading and saving file to connector
self.helper.log_info("Importing the file " + file_uri)
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=self.data["simple_observable_key"],
value=self.data["simple_observable_value"],
)
bundle_objects = [observable]
entity_id = data.get("entity_id", None)
report = self.helper.api.report.read(id=entity_id)
report = Report(
id=report["standard_id"],
name=report["name"],
description=report["description"],
published=self.helper.api.stix2.format_date(report["published"]),
report_types=report["report_types"],
object_refs=bundle_objects,
)
bundle_objects.append(report)
# create stix bundle
bundle = Bundle(objects=bundle_objects).serialize()
# send data
self.helper.send_stix2_bundle(bundle=bundle)
return "foo"
def stop(self):
self.helper.stop()
def start(self):
try:
self.helper.listen(self._process_message)
except pika.exceptions.AMQPConnectionError:
self.stop()
raise ValueError(
"Connector was not able to establish the connection to RabbitMQ"
)
class ExportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
entity_id = data["entity_id"]
entity_type = data["entity_type"]
file_name = data["file_name"]
file_context = data["file_context"]
export_type = data["export_type"]
list_args = data["list_args"]
max_marking_definition = data["max_marking_definition"]
if entity_id is not None:
self.helper.log_info("Exporting: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
bundle = self.helper.api.stix2.export_entity(
entity_type, entity_id, export_type, max_marking_definition)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
self.helper.api.stix_domain_entity.push_entity_export(
entity_id, file_name, json_bundle)
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
else:
self.helper.log_info("Exporting list: " + entity_type + "/" +
export_type + " to " + file_name)
bundle = self.helper.api.stix2.export_list(
entity_type.lower(),
list_args["search"],
list_args["filters"],
list_args["orderBy"],
list_args["orderMode"],
max_marking_definition,
list_args["types"] if "types" in list_args else None,
)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + " to " + file_name)
self.helper.api.stix_domain_entity.push_list_export(
entity_type, file_name, json_bundle, file_context,
json.dumps(list_args))
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + " to " + file_name)
return ["Export done"]
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ExportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
file_name = data["file_name"]
export_scope = data["export_scope"] # single or list
export_type = data["export_type"] # Simple or Full
max_marking = data["max_marking"]
entity_type = data["entity_type"]
if export_scope == "single":
entity_id = data["entity_id"]
self.helper.log_info("Exporting: " + entity_id + "(" +
export_type + ") to " + file_name)
bundle = self.helper.api.stix2.export_entity(
entity_type, entity_id, export_type, max_marking)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info("Uploading: " + entity_id + "(" +
export_type + ") to " + file_name)
self.helper.api.stix_domain_object.push_entity_export(
entity_id, file_name, json_bundle)
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
else:
list_params = data["list_params"]
self.helper.log_info("Exporting list: " + entity_type + "/" +
export_type + " to " + file_name)
bundle = self.helper.api.stix2.export_list(
entity_type,
list_params["search"],
list_params["filters"],
list_params["orderBy"],
list_params["orderMode"],
max_marking,
list_params.get("types"),
)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + " to " + file_name)
self.helper.api.stix_domain_object.push_list_export(
entity_type, file_name, json_bundle, json.dumps(list_params))
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + " to " + file_name)
return "Export done"
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ExportFileStix:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
entity_id = data['entity_id']
entity_type = data['entity_type']
file_name = data['file_name']
file_context = data['file_context']
export_type = data['export_type']
list_args = data['list_args']
max_marking_definition = data['max_marking_definition']
if entity_id is not None:
self.helper.log_info('Exporting: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
bundle = self.helper.api.stix2.export_entity(
entity_type, entity_id, export_type, max_marking_definition)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info('Uploading: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
self.helper.api.stix_domain_entity.push_entity_export(
entity_id, file_name, json_bundle)
self.helper.log_info('Export done: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
else:
self.helper.log_info('Exporting list: ' + entity_type + '/' +
export_type + ' to ' + file_name)
bundle = self.helper.api.stix2.export_list(entity_type.lower(),
list_args['search'],
list_args['filters'],
list_args['orderBy'],
list_args['orderMode'],
max_marking_definition)
json_bundle = json.dumps(bundle, indent=4)
self.helper.log_info('Uploading: ' + entity_type + '/' +
export_type + ' to ' + file_name)
self.helper.api.stix_domain_entity.push_list_export(
entity_type, file_name, json_bundle, file_context,
json.dumps(list_args))
self.helper.log_info('Export done: ' + entity_type + '/' +
export_type + ' to ' + file_name)
return ['Export done']
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class SightingConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
print(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class InternalEnrichmentConnector:
def __init__(self, config_file_path: str, api_client: OpenCTIApiClient,
data: Dict):
# set OPENCTI settings from fixture
os.environ["OPENCTI_URL"] = api_client.api_url
os.environ["OPENCTI_TOKEN"] = api_client.api_token
os.environ["OPENCTI_SSL_VERIFY"] = str(api_client.ssl_verify)
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def _process_message(self, data: Dict) -> str:
# set score to 100
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
self.helper.api.stix_cyber_observable.update_field(
id=observable["standard_id"],
input={
"key": "x_opencti_score",
"value": ["100"]
},
)
# now = datetime.utcfromtimestamp(time.time())
# now_time = now.strftime("%Y-%m-%d %H:%M:%S")
# friendly_name = f"{self.helper.connect_name} run @ {now_time}"
# work_id = self.helper.api.work.initiate_work(
# self.helper.connect_id, friendly_name
# )
#
# # set score to 100
# entity_id = data["entity_id"]
# observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
#
# stix_observable = SimpleObservable(
# id=OpenCTIStix2Utils.generate_random_stix_id("x-opencti-simple-observable"),
# key="IPv4-Addr.value",
# value=observable['value'],
# x_opencti_score=100
# )
# bundle_objects = [stix_observable]
# # create stix bundle
# bundle = Bundle(objects=bundle_objects).serialize()
# # send data
# self.helper.send_stix2_bundle(
# bundle=bundle,
# update=True,
# )
#
# message = "Connector successfully run, storing last_run as " + str(now_time)
# self.helper.api.work.to_processed(work_id, message)
return "Finished"
def stop(self):
self.helper.stop()
def start(self):
try:
self.helper.listen(self._process_message)
except pika.exceptions.AMQPConnectionError:
self.stop()
raise ValueError(
"Connector was not able to establish the connection to RabbitMQ"
)
class ReportImporter:
def __init__(self) -> None:
# Instantiate the connector helper from config
base_path = os.path.dirname(os.path.abspath(__file__))
config_file_path = base_path + "/../config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.create_indicator = get_config_variable(
"IMPORT_DOCUMENT_CREATE_INDICATOR",
["import_document", "create_indicator"],
config,
)
# Load Entity and Observable configs
observable_config_file = base_path + "/config/observable_config.ini"
entity_config_file = base_path + "/config/entity_config.ini"
if os.path.isfile(observable_config_file) and os.path.isfile(
entity_config_file):
self.observable_config = self._parse_config(
observable_config_file, Observable)
else:
raise FileNotFoundError(f"{observable_config_file} was not found")
if os.path.isfile(entity_config_file):
self.entity_config = self._parse_config(entity_config_file,
EntityConfig)
else:
raise FileNotFoundError(f"{entity_config_file} was not found")
def _process_message(self, data: Dict) -> str:
self.helper.log_info("Processing new message")
file_name = self._download_import_file(data)
entity_id = data.get("entity_id", None)
bypass_validation = data.get("bypass_validation", False)
entity = (self.helper.api.stix_domain_object.read(
id=entity_id) if entity_id is not None else None)
if self.helper.get_only_contextual() and entity is None:
return "Connector is only contextual and entity is not defined. Nothing was imported"
# Retrieve entity set from OpenCTI
entity_indicators = self._collect_stix_objects(self.entity_config)
# Parse report
parser = ReportParser(self.helper, entity_indicators,
self.observable_config)
parsed = parser.run_parser(file_name, data["file_mime"])
os.remove(file_name)
if not parsed:
return "No information extracted from report"
# Process parsing results
self.helper.log_debug("Results: {}".format(parsed))
observables, entities = self._process_parsing_results(parsed, entity)
# Send results to OpenCTI
observable_cnt = self._process_parsed_objects(entity, observables,
entities,
bypass_validation,
file_name)
entity_cnt = len(entities)
if self.helper.get_validate_before_import() and not bypass_validation:
return "Generated bundle sent for validation"
else:
return (
f"Sent {observable_cnt} observables, 1 report update and {entity_cnt} entity connections as stix "
f"bundle for worker import ")
def start(self) -> None:
self.helper.listen(self._process_message)
def _download_import_file(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
# Downloading and saving file to connector
self.helper.log_info("Importing the file " + file_uri)
file_name = os.path.basename(file_fetch)
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
with open(file_name, "wb") as f:
f.write(file_content)
return file_name
def _collect_stix_objects(
self, entity_config_list: List[EntityConfig]) -> List[Entity]:
base_func = self.helper.api
entity_list = []
for entity_config in entity_config_list:
func_format = entity_config.stix_class
try:
custom_function = getattr(base_func, func_format)
entries = custom_function.list(
getAll=True,
filters=entity_config.filter,
customAttributes=entity_config.custom_attributes,
)
entity_list += entity_config.convert_to_entity(
entries, self.helper)
except AttributeError:
e = "Selected parser format is not supported: {}".format(
func_format)
raise NotImplementedError(e)
return entity_list
@staticmethod
def _parse_config(config_file: str,
file_class: Callable) -> List[BaseModel]:
config = MyConfigParser()
config.read(config_file)
config_list = []
for section, content in config.as_dict().items():
content["name"] = section
config_object = file_class(**content)
config_list.append(config_object)
return config_list
def _process_parsing_results(
self, parsed: List[Dict],
context_entity: Dict) -> (List[SimpleObservable], List[str]):
observables = []
entities = []
if context_entity is not None:
object_markings = [
x["standard_id"]
for x in context_entity.get("objectMarking", [])
]
# external_references = [x['standard_id'] for x in report.get('externalReferences', [])]
# labels = [x['standard_id'] for x in report.get('objectLabel', [])]
author = context_entity.get("createdBy")
else:
object_markings = []
author = None
if author is not None:
author = author.get("standard_id", None)
for match in parsed:
if match[RESULT_FORMAT_TYPE] == OBSERVABLE_CLASS:
if match[RESULT_FORMAT_CATEGORY] == "Vulnerability.name":
entity = self.helper.api.vulnerability.read(
filters={
"key": "name",
"values": [match[RESULT_FORMAT_MATCH]]
})
if entity is None:
self.helper.log_info(
f"Vulnerability with name '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the CVE Connector activated?")
continue
entities.append(entity["standard_id"])
elif match[
RESULT_FORMAT_CATEGORY] == "Attack-Pattern.x_mitre_id":
entity = self.helper.api.attack_pattern.read(
filters={
"key": "x_mitre_id",
"values": [match[RESULT_FORMAT_MATCH]],
})
if entity is None:
self.helper.log_info(
f"AttackPattern with MITRE ID '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the MITRE Connector activated?")
continue
entities.append(entity["standard_id"])
else:
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=match[RESULT_FORMAT_CATEGORY],
value=match[RESULT_FORMAT_MATCH],
x_opencti_create_indicator=self.create_indicator,
object_marking_refs=object_markings,
created_by_ref=author,
# labels=labels,
# external_references=external_references
)
observables.append(observable)
elif match[RESULT_FORMAT_TYPE] == ENTITY_CLASS:
entities.append(match[RESULT_FORMAT_MATCH])
else:
self.helper.log_info("Odd data received: {}".format(match))
return observables, entities
def _process_parsed_objects(
self,
entity: Dict,
observables: List,
entities: List,
bypass_validation: bool,
file_name: str,
) -> int:
if len(observables) == 0 and len(entities) == 0:
return 0
if entity is not None and entity["entity_type"] == "Report":
report = Report(
id=entity["standard_id"],
name=entity["name"],
description=entity["description"],
published=self.helper.api.stix2.format_date(entity["created"]),
report_types=entity["report_types"],
object_refs=observables + entities,
allow_custom=True,
)
observables.append(report)
elif entity is not None:
# TODO, relate all object to the entity
entity_stix_bundle = self.helper.api.stix2.export_entity(
entity["entity_type"], entity["id"])
observables = observables + entity_stix_bundle["objects"]
else:
timestamp = int(time.time())
now = datetime.utcfromtimestamp(timestamp)
report = Report(
name=file_name,
description="Automatic import",
published=now,
report_types=["threat-report"],
object_refs=observables + entities,
allow_custom=True,
)
observables.append(report)
bundles_sent = []
if len(observables) > 0:
bundle = Bundle(objects=observables, allow_custom=True).serialize()
bundles_sent = self.helper.send_stix2_bundle(
bundle=bundle,
update=True,
bypass_validation=bypass_validation,
file_name=file_name + ".json",
entity_id=entity["id"] if entity is not None else None,
)
# len() - 1 because the report update increases the count by one
return len(bundles_sent) - 1
class VirusTotalConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.token = get_config_variable("VIRUSTOTAL_TOKEN",
["virustotal", "token"], config)
self.max_tlp = get_config_variable("VIRUSTOTAL_MAX_TLP",
["virustotal", "max_tlp"], config)
self.api_url = "https://www.virustotal.com/api/v3"
self.headers = {
"x-apikey": self.token,
"accept": "application/json",
"content-type": "application/json",
}
self._CONNECTOR_RUN_INTERVAL_SEC = 60 * 60
def _process_file(self, observable):
response = requests.request(
"GET",
self.api_url + "/files/" + observable["observable_value"],
headers=self.headers,
)
json_data = json.loads(response.text)
if "error" in json_data:
if json_data["error"]["message"] == "Quota exceeded":
self.helper.log_info("Quota reached, waiting 1 hour.")
sleep(self._CONNECTOR_RUN_INTERVAL_SEC)
elif "not found" in json_data["error"]["message"]:
self.helper.log_info("File not found on VirusTotal.")
return "File not found on VirusTotal."
else:
raise ValueError(json_data["error"]["message"])
if "data" in json_data:
data = json_data["data"]
attributes = data["attributes"]
# Update the current observable
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=observable["id"], key="hashes.MD5", value=attributes["md5"])
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.SHA-1",
value=attributes["sha1"])
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.SHA-256",
value=attributes["sha256"],
)
if observable["entity_type"] == "StixFile":
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="size",
value=str(attributes["size"]),
)
if observable["name"] is None and len(attributes["names"]) > 0:
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="name",
value=attributes["names"][0])
del attributes["names"][0]
if len(attributes["names"]) > 0:
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="x_opencti_additional_names",
value=attributes["names"],
)
# Create external reference
external_reference = self.helper.api.external_reference.create(
source_name="VirusTotal",
url="https://www.virustotal.com/gui/file/" +
attributes["sha256"],
description=attributes["magic"],
)
# Create tags
for tag in attributes["tags"]:
tag_vt = self.helper.api.label.create(value=tag,
color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=final_observable["id"], label_id=tag_vt["id"])
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
return "File found on VirusTotal, knowledge attached."
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
return self._process_file(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ImportFilePdfObservables:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.create_indicator = get_config_variable(
"PDF_OBSERVABLES_CREATE_INDICATOR",
["pdf_observables", "create_indicator"],
config,
)
def _process_message(self, data):
file_path = data["file_path"]
file_name = os.path.basename(file_path)
work_context = data["work_context"]
file_uri = self.helper.opencti_url + file_path
self.helper.log_info("Importing the file " + file_uri)
# Get the file
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
# Write the file
path = "/tmp/" + file_name
f = open(path, "wb")
f.write(file_content)
f.close()
# Parse
bundle = {
"type": "bundle",
"id": "bundle--" + str(uuid.uuid4()),
"spec_version": "2.0",
"objects": [],
}
observed_data = {
"id": "observed-data--" + str(uuid.uuid4()),
"type": "observed-data",
"x_opencti_indicator_create": self.create_indicator,
"objects": {},
}
i = 0
parser = iocp.IOC_Parser(None, "pdf", True, "pdfminer", "json")
parsed = parser.parse(path)
os.remove(path)
if parsed != []:
for file in parsed:
if file != None:
for page in file:
if page != []:
for match in page:
resolved_match = self.resolve_match(match)
if resolved_match:
observable = {
"type":
resolved_match["type"],
"x_opencti_observable_type":
resolved_match["type"],
"x_opencti_observable_value":
resolved_match["value"],
"x_opencti_indicator_create":
self.create_indicator,
}
observed_data["objects"][i] = observable
i += 1
else:
self.helper.log_error("Could not parse the report!")
# Get context
if len(observed_data["objects"]) > 0:
bundle["objects"].append(observed_data)
if work_context is not None and len(work_context) > 0:
report = self.helper.api.report.read(id=work_context)
if report is not None:
report_stix = {
"type":
"report",
"id":
report["stix_id_key"],
"name":
report["name"],
"description":
report["description"],
"published":
self.helper.api.stix2.format_date(report["published"]),
"object_refs": [],
}
report_stix["object_refs"].append(observed_data["id"])
bundle["objects"].append(report_stix)
bundles_sent = self.helper.send_stix2_bundle(
json.dumps(bundle), None, False, False)
return [
"Sent " + str(len(bundles_sent)) +
" stix bundle(s) for worker import"
]
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
def resolve_match(self, match):
types = {
"MD5": ["File-MD5"],
"SHA1": ["File-SHA1"],
"SHA256": ["File-SHA256"],
"Filename": ["File-Name"],
"IP": ["IPv4-Addr"],
"Host": ["Domain"],
"Filepath": ["File-Name"],
"URL": ["URL"],
"Email": ["Email-Address"],
}
type = match["type"]
value = match["match"]
if type in types:
resolved_types = types[type]
if resolved_types[0] == "IPv4-Addr":
type_0 = self.detect_ip_version(value)
else:
type_0 = resolved_types[0]
return {"type": type_0, "value": value}
else:
return False
def detect_ip_version(self, value):
if len(value) > 16:
return "IPv6-Addr"
else:
return "IPv4-Addr"
class HygieneConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.warninglists = WarningLists()
# Create Hygiene Tag
self.label_hygiene = self.helper.api.label.create(
value="Hygiene", color="#fc0341"
)
def _process_observable(self, observable) -> str:
# Extract IPv4, IPv6 and Domain from entity data
observable_value = observable["observable_value"]
# Search in warninglist
result = self.warninglists.search(observable_value)
# Iterate over the hits
if result:
self.helper.log_info(
"Hit found for %s in warninglists" % (observable_value)
)
for hit in result:
self.helper.log_info(
"Type: %s | Name: %s | Version: %s | Descr: %s"
% (hit.type, hit.name, hit.version, hit.description)
)
# We set the score based on the number of warning list entries
if len(result) >= 5:
score = "5"
elif len(result) >= 3:
score = "10"
elif len(result) == 1:
score = "15"
else:
score = "20"
self.helper.log_info(
f"number of hits ({len(result)}) setting score to {score}"
)
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"], label_id=self.label_hygiene["id"]
)
self.helper.api.stix_cyber_observable.update_field(
id=observable["id"], key="x_opencti_score", value=score
)
for indicator_id in observable["indicatorsIds"]:
self.helper.api.stix_domain_object.add_label(
id=indicator_id, label_id=self.label_hygiene["id"]
)
self.helper.api.stix_domain_object.update_field(
id=indicator_id, key="x_opencti_score", value=score
)
# Create external references
external_reference_id = self.helper.api.external_reference.create(
source_name="misp-warninglist",
url="https://github.com/MISP/misp-warninglists/tree/main/"
+ LIST_MAPPING[hit.name],
external_id=hit.name,
description=hit.description,
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable["id"],
external_reference_id=external_reference_id["id"],
)
return "Observable value found on warninglist and tagged accordingly"
def _process_message(self, data) -> str:
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class UnpacMeConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.identity = self.helper.api.identity.create(
type="Organization",
name="UnpacMe",
description="UnpacMe",
)["standard_id"]
self.octi_api_url = get_config_variable("OPENCTI_URL",
["opencti", "url"], config)
# Get URL and private from config, use to instantiate the client
user_agent = get_config_variable(
"UNPAC_ME_USER_AGENT",
["unpac_me", "user_agent"],
config,
)
api_key = get_config_variable(
"UNPAC_ME_API_KEY",
["unpac_me", "api_key"],
config,
)
self.private = get_config_variable(
"UNPAC_ME_PRIVATE",
["unpac_me", "private"],
config,
)
self.unpacme_client = UnpacMeApi(api_key=api_key,
user_agent=user_agent)
# Other config settings
self.family_color = get_config_variable(
"UNPAC_ME_FAMILY_COLOR",
["unpac_me", "family_color"],
config,
)
self.default_tag_color = get_config_variable(
"UNPAC_ME_FAMILY_COLOR",
["unpac_me", "tag_color"],
config,
)
self.less_noise = get_config_variable(
"UNPAC_ME_LESS_NOISE",
["unpac_me", "less_noise"],
config,
)
self.max_tlp = get_config_variable(
"UNPAC_ME_MAX_TLP",
["unpac_me", "max_tlp"],
config,
)
def _process_results(self, observable, results):
bundle_objects = []
unpack_id = results["id"]
# Create external reference
analysis_url = f"https://www.unpac.me/results/{unpack_id}"
external_reference = self.helper.api.external_reference.create(
source_name="UnpacMe Results",
url=analysis_url,
description="UnpacMe Results",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable["id"],
external_reference_id=external_reference["id"],
)
# Create default labels
extracted_label = self.helper.api.label.create(
value="extracted", color=self.default_tag_color)
# Parse the results
label_ids = []
for result_dict in results["results"]:
sha256 = result_dict["hashes"]["sha256"]
# If less noise, check to ensure the files were identified as malware
if self.less_noise:
self.helper.log_info("Less noise is enabled.")
if not result_dict["malware_id"]:
self.helper.log_info(
f"Skipping upload of {sha256} as it had no matching family."
)
continue
# Download the file
file_contents = self.unpacme_client.download(sha256=sha256)
# Upload as Artifact to OpenCTI
mime_type = magic.from_buffer(file_contents, mime=True)
kwargs = {
"file_name": sha256,
"data": file_contents,
"mime_type": mime_type,
"x_opencti_description": "UnpacMe extracted file.",
}
response = self.helper.api.stix_cyber_observable.upload_artifact(
**kwargs)
# Create Relationship between original and newly uploaded Artifact
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=response["standard_id"],
target_ref=observable["standard_id"],
)
bundle_objects.append(relationship)
# Attach default "extracted" label
if response["id"] != observable["id"]:
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=extracted_label["id"])
# If found malware ids, attach as labels
for malware_id_dict in result_dict["malware_id"]:
family_label = self.helper.api.label.create(
value=malware_id_dict["name"], color=self.family_color)
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=family_label["id"])
label_ids.append(family_label["id"])
# Attach all identified tags to the Artifact
for label_id in label_ids:
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"], label_id=family_label["id"])
# Serialize and send all bundles
if bundle_objects:
bundle = Bundle(objects=bundle_objects,
allow_custom=True).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return f"Sent {len(bundles_sent)} stix bundle(s) for worker import"
else:
return "Nothing to attach"
def _process_file(self, observable):
if not observable["importFiles"]:
raise ValueError(
f"No files found for {observable['observable_value']}")
# Build the URI to download the file
file_id = observable["importFiles"][0]["id"]
file_uri = f"{self.octi_api_url}/storage/get/{file_id}"
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
# Submit sample for analysis
upload = self.unpacme_client.upload(data=file_content,
private=self.private)
# Wait for the analysis to finish
while True:
response = self.unpacme_client.status(upload=upload)
if response == UnpacMeStatus.COMPLETE:
break
elif response == UnpacMeStatus.FAIL:
raise ValueError(f"UnpacMe failed to analyze {file_id}")
time.sleep(20)
# Analysis is complete, get the results
results = self.unpacme_client.results(upload=upload)
results = results.raw_json
self.helper.log_info(
f"Analysis complete, processing results: {results}...")
return self._process_results(observable, results)
def _process_observable(self, observable):
self.helper.log_info("Processing the observable " +
observable["observable_value"])
# If File, Artifact
if observable["entity_type"] == "Artifact":
return self._process_file(observable)
else:
raise ValueError(
f"Failed to process observable, {observable['entity_type']} is not a supported entity type."
)
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
if observable is None:
raise ValueError(
"Observable not found "
"(may be linked to data seggregation, check your group and permissions)"
)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ExportReportPdf:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# ExportReportPdf specific config settings
self.primary_color = get_config_variable(
"EXPORT_REPORT_PDF_PRIMARY_COLOR",
["export_report_pdf", "primary_color"],
config,
)
self.secondary_color = get_config_variable(
"EXPORT_REPORT_PDF_SECONDARY_COLOR",
["export_report_pdf", "secondary_color"],
config,
)
self.set_colors()
self.company_address_line_1 = get_config_variable(
"EXPORT_REPORT_PDF_COMPANY_ADDRESS_LINE_1",
["export_report_pdf", "company_address_line_1"],
config,
)
self.company_address_line_2 = get_config_variable(
"EXPORT_REPORT_PDF_COMPANY_ADDRESS_LINE_2",
["export_report_pdf", "company_address_line_2"],
config,
)
self.company_address_line_3 = get_config_variable(
"EXPORT_REPORT_PDF_COMPANY_ADDRESS_LINE_3",
["export_report_pdf", "company_address_line_3"],
config,
)
self.company_phone_number = get_config_variable(
"EXPORT_REPORT_PDF_COMPANY_PHONE_NUMBER",
["export_report_pdf", "company_phone_number"],
config,
)
self.company_email = get_config_variable(
"EXPORT_REPORT_PDF_COMPANY_EMAIL",
["export_report_pdf", "company_email"],
config,
)
self.company_website = get_config_variable(
"EXPORT_REPORT_PDF_COMPANY_WEBSITE",
["export_report_pdf", "company_website"],
config,
)
self.indicators_only = get_config_variable(
"EXPORT_REPORT_PDF_INDICATORS_ONLY",
["export_report_pdf", "indicators_only"],
config,
)
self.defang_urls = get_config_variable(
"EXPORT_REPORT_PDF_DEFANG_URLS",
["export_report_pdf", "defang_urls"],
config,
)
def _process_message(self, data):
file_name = data["file_name"]
# TODO this can be implemented to filter every entity and observable
# max_marking = data["max_marking"]
entity_type = data["entity_type"]
if entity_type != "Report":
raise ValueError(
f'This Connector can only process entities of type "Report" and not of type "{entity_type}".'
)
# Get the Report
report_dict = self.helper.api.report.read(id=data["entity_id"])
# Extract values for inclusion in output pdf
report_marking = report_dict.get("objectMarking", None)
if report_marking:
report_marking = report_marking[-1]["definition"]
report_name = report_dict["name"]
report_description = report_dict.get("description",
"No description available.")
report_confidence = report_dict["confidence"]
report_id = report_dict["id"]
report_external_refs = [
external_ref_dict["url"]
for external_ref_dict in report_dict["externalReferences"]
]
report_objs = report_dict["objects"]
report_date = datetime.datetime.now().strftime("%b %d %Y")
context = {
"report_name": report_name,
"report_description": report_description,
"report_marking": report_marking,
"report_confidence": report_confidence,
"report_external_refs": report_external_refs,
"report_date": report_date,
"company_address_line_1": self.company_address_line_1,
"company_address_line_2": self.company_address_line_2,
"company_address_line_3": self.company_address_line_3,
"company_phone_number": self.company_phone_number,
"company_email": self.company_email,
"company_website": self.company_website,
"entities": {},
"observables": {},
}
# Process each STIX Object
for report_obj in report_objs:
obj_entity_type = report_obj["entity_type"]
obj_id = report_obj["standard_id"]
# Handle StixCyberObservables entities
if obj_entity_type == "StixFile" or StixCyberObservableTypes.has_value(
obj_entity_type):
observable_dict = self.helper.api.stix_cyber_observable.read(
id=obj_id)
# If only include indicators and
# the observable doesn't have an indicator, skip it
if self.indicators_only and not observable_dict["indicators"]:
self.helper.log_info(
f"Skipping {obj_entity_type} observable with value {observable_dict['observable_value']} as it was not an Indicator."
)
continue
if obj_entity_type not in context["observables"]:
context["observables"][obj_entity_type] = []
# Defang urls
if self.defang_urls and obj_entity_type == "Url":
observable_dict["observable_value"] = observable_dict[
"observable_value"].replace("http", "hxxp", 1)
context["observables"][obj_entity_type].append(observable_dict)
# Handle all other entities
else:
reader_func = self.get_reader(obj_entity_type)
if reader_func is None:
self.helper.log_error(
f'Could not find a function to read entity with type "{obj_entity_type}"'
)
continue
entity_dict = reader_func(id=obj_id)
if obj_entity_type not in context["entities"]:
context["entities"][obj_entity_type] = []
context["entities"][obj_entity_type].append(entity_dict)
# Render html with input variables
env = Environment(
loader=FileSystemLoader(os.path.abspath(os.getcwd())))
template = env.get_template("src/resources/report.html")
html_string = template.render(context)
# Generate pdf from html string
pdf_contents = HTML(string=html_string,
base_url="src/resources").write_pdf()
# Upload the output pdf
self.helper.log_info(f"Uploading: {file_name}")
self.helper.api.stix_domain_object.add_file(
id=report_id,
file_name=file_name,
data=pdf_contents,
mime_type="application/pdf",
)
return "Export done"
def set_colors(self):
with open("src/resources/report.css.template", "r") as f:
new_css = f.read()
new_css = new_css.replace("<primary_color>", self.primary_color)
new_css = new_css.replace("<secondary_color>",
self.secondary_color)
with open("src/resources/report.css", "w") as f:
f.write(new_css)
def get_reader(self, entity_type):
"""
Returns the function to use for calling the OpenCTI to
read data for a particular entity type.
entity_type: a str representing the entity type, i.e. Indicator
returns: a function or None if entity type is not supported
"""
reader = {
"Stix-Domain-Object": self.helper.api.stix_domain_object.read,
"Attack-Pattern": self.helper.api.attack_pattern.read,
"Campaign": self.helper.api.campaign.read,
"Note": self.helper.api.note.read,
"Observed-Data": self.helper.api.observed_data.read,
"Organization": self.helper.api.identity.read,
"Opinion": self.helper.api.opinion.read,
"Report": self.helper.api.report.read,
"Sector": self.helper.api.identity.read,
"System": self.helper.api.identity.read,
"Course-Of-Action": self.helper.api.course_of_action.read,
"Identity": self.helper.api.identity.read,
"Indicator": self.helper.api.indicator.read,
"Individual": self.helper.api.identity.read,
"Infrastructure": self.helper.api.infrastructure.read,
"Intrusion-Set": self.helper.api.intrusion_set.read,
"Malware": self.helper.api.malware.read,
"Threat-Actor": self.helper.api.threat_actor.read,
"Tool": self.helper.api.tool.read,
"Vulnerability": self.helper.api.vulnerability.read,
"Incident": self.helper.api.incident.read,
"City": self.helper.api.location.read,
"Country": self.helper.api.location.read,
"Region": self.helper.api.location.read,
"Position": self.helper.api.location.read,
"Location": self.helper.api.location.read,
}
return reader.get(entity_type, None)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class ExportFileCsv:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.helper = OpenCTIConnectorHelper(config)
def export_dict_list_to_csv(self, data):
output = io.StringIO()
headers = sorted(set().union(*(d.keys() for d in data)))
csv_data = [headers]
for d in data:
row = []
for h in headers:
if h not in d:
row.append('')
elif isinstance(d[h], str):
row.append(d[h])
elif isinstance(d[h], list):
if len(d[h]) > 0 and isinstance(d[h][0], str):
row.append(','.join(d[h]))
elif len(d[h]) > 0 and isinstance(d[h][0], dict):
rrow = []
for r in d[h]:
if 'name' in r:
rrow.append(r['name'])
elif 'definition' in r:
rrow.append(r['definition'])
row.append(','.join(rrow))
else:
row.append('')
elif isinstance(d[h], dict):
if 'name' in d[h]:
row.append(d[h]['name'])
else:
row.append('')
else:
row.append('')
csv_data.append(row)
writer = csv.writer(output,
delimiter=';',
quotechar='"',
quoting=csv.QUOTE_ALL)
writer.writerows(csv_data)
return output.getvalue()
def _process_message(self, data):
entity_id = data['entity_id']
entity_type = data['entity_type']
file_name = data['file_name']
file_context = data['file_context']
export_type = data['export_type']
list_args = data['list_args']
max_marking_definition = data['max_marking_definition']
if entity_id is not None:
self.helper.log_info('Exporting: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
entity_data = self.helper.api.stix_domain_entity.read(id=entity_id)
entities_list = [entity_data]
if 'objectRefsIds' in entity_data:
for id in entity_data['objectRefsIds']:
entity = self.helper.api.stix_domain_entity.read(id=id)
entities_list.append(entity)
csv_data = self.export_dict_list_to_csv(entities_list)
self.helper.log_info('Uploading: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
self.helper.api.stix_domain_entity.push_entity_export(
entity_id, file_name, csv_data)
self.helper.log_info('Export done: ' + entity_type + '/' +
export_type + '(' + entity_id + ') to ' +
file_name)
else:
self.helper.log_info('Exporting list: ' + entity_type + '/' +
export_type + ' to ' + file_name)
max_marking_definition_entity = self.helper.api.marking_definition.read(
id=max_marking_definition
) if max_marking_definition is not None else None
if IdentityTypes.has_value(entity_type):
if list_args['filters'] is not None:
list_args['filters'].append({
'key': 'entity_type',
'values': [entity_type]
})
else:
list_args['filters'] = [{
'key': 'entity_type',
'values': [entity_type]
}]
entity_type = 'identity'
# List
lister = {
'identity': self.helper.api.identity.list,
'threat-actor': self.helper.api.threat_actor.list,
'intrusion-set': self.helper.api.intrusion_set.list,
'campaign': self.helper.api.campaign.list,
'incident': self.helper.api.incident.list,
'malware': self.helper.api.malware.list,
'tool': self.helper.api.tool.list,
'vulnerability': self.helper.api.vulnerability.list,
'attack-pattern': self.helper.api.attack_pattern.list,
'course-of-action': self.helper.api.course_of_action.list,
'report': self.helper.api.report.list,
'indicator': self.helper.api.indicator.list
}
do_list = lister.get(
entity_type.lower(), lambda **kwargs: self.helper.
log_error('Unknown object type "' + entity_type +
'", doing nothing...'))
entities_list = do_list(search=list_args['search'],
filters=list_args['filters'],
orderBy=list_args['orderBy'],
orderMode=list_args['orderMode'],
getAll=True)
csv_data = self.export_dict_list_to_csv(entities_list)
self.helper.log_info('Uploading: ' + entity_type + '/' +
export_type + ' to ' + file_name)
self.helper.api.stix_domain_entity.push_list_export(
entity_type, file_name, csv_data, file_context,
json.dumps(list_args))
self.helper.log_info('Export done: ' + entity_type + '/' +
export_type + ' to ' + file_name)
return ['Export done']
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class HygieneConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.warninglists = WarningLists()
# Create Hygiene Tag
self.tag_hygiene = self.helper.api.tag.create(
tag_type="Hygiene",
value="Hygiene",
color="#fc0341",
)
def _process_observable(self, observable):
# Extract IPv4, IPv6 and Domain from entity data
observable_value = observable["observable_value"]
# Search in warninglist
result = self.warninglists.search(observable_value)
# Iterate over the hits
if result:
self.helper.log_info("Hit found for %s in warninglists" %
(observable_value))
for hit in result:
self.helper.log_info(
"Type: %s | Name: %s | Version: %s | Descr: %s" %
(hit.type, hit.name, hit.version, hit.description))
self.helper.api.stix_entity.add_tag(
id=observable["id"], tag_id=self.tag_hygiene["id"])
# Create external references
external_reference_id = self.helper.api.external_reference.create(
source_name="misp-warninglist",
url="https://github.com/MISP/misp-warninglists/tree/master"
+ LIST_MAPPING[hit.name],
external_id=hit.name,
description=hit.description,
)
self.helper.api.stix_entity.add_external_reference(
id=observable["id"],
external_reference_id=external_reference_id["id"],
)
return [
"observable value found on warninglist and tagged accordingly"
]
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_observable.read(id=entity_id)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class MalBeaconConnector:
"""Malbeacon connector class"""
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL", ["connector", "confidence_level"],
config)
self.api_key = get_config_variable("MALBEACON_API_KEY",
["malbeacon", "api_key"], config)
self.author = self.helper.api.identity.create(
name="Malbeacon",
type="Organization",
description="""The first system of its kind, MalBeacon implants \
beacons via malware bot check-in traffic. Adversaries conducting \
campaigns in the wild who are logging in to these malware C2 \
panels can now be tracked. MalBeacon is a tool for the good guys \
that provides additional intelligence on attack attribution.""",
update=True,
)
def _process_observable(self, observable) -> str:
logger.info(f"processing observable: {observable}")
# Extract IPv4, IPv6, Hostname and Domain from entity data
obs_val = observable["observable_value"]
obs_typ = observable["entity_type"]
obs_id = observable["id"]
if obs_typ == "Domain-Name":
self._process_c2(obs_val, obs_id)
elif obs_typ in ["IPv4-Addr", "IPv6-Addr"]:
self._process_c2(obs_val, obs_id)
elif obs_typ in "Email-Address":
# TODO: not implemented yet
pass
else:
return "no information found on malbeacon"
return "observable value found on malbeacon API and knowledge added"
def _process_message(self, data) -> list:
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
################################
# Helper Functions
################################
def _api_call(self, url_path):
api_base_url = "https://api.malbeacon.com/v1/"
url = urljoin(api_base_url, url_path)
try:
r = requests.get(url, headers={"X-Api-Key": self.api_key})
except requests.exceptions.RequestException as e:
logger.error(f"error in malbeacon api request: {e}")
return None
return r.json()
def _process_c2(self, obs_value, obs_id):
already_processed = []
reference = self.helper.api.external_reference.create(
source_name="Malbeacon C2 Domains",
url="https://malbeacon.com/illuminate",
description="Found in Malbeacon C2 Domains",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=obs_id, external_reference_id=reference["id"])
data = self._api_call("c2/c2/" + obs_value)
# If the API returns a JSON document with a message
# there probably has been an error or no information
# could be retreived from the Malbeacon database
try:
api_error = data["message"]
logger.error(f"Error in API request: {api_error}")
return None
except (ValueError, TypeError):
pass
try:
for entry in data:
c2_beacon = C2Beacon.parse_obj(entry)
logger.info(
f"Processing: {c2_beacon.cti_date} {c2_beacon.actorip} {c2_beacon.actorhostname}"
)
######################################################
# Process what we know about the actors infrastructure
######################################################
if (c2_beacon.actorip != "NA"
and c2_beacon.actorip not in already_processed):
self.helper.api.stix_cyber_observable.create(
simple_observable_key="IPv4-Addr.value",
simple_observable_value=c2_beacon.actorip,
simple_observable_description=
f"Malbeacon Actor IP Address for C2 {obs_value}",
createdBy=self.author["id"],
x_opencti_score=int(self.confidence_level),
createIndicator=True,
)
# TODO: find and implement meaningful relationships
# self.helper.api.stix_core_relationship.create(
# fromId=obs_id,
# toId=actor_ip_obs["id"],
# relationship_type="based-on",
# createdBy=self.author["id"],
# )
if c2_beacon.actorhostname != "NA":
self.helper.api.stix_cyber_observable.create(
simple_observable_key="Domain-Name.value",
simple_observable_value=c2_beacon.actorhostname,
simple_observable_description=
f"Malbeacon Actor DomainName for C2 {obs_value}",
createdBy=self.author["id"],
x_opencti_score=int(self.confidence_level),
createIndicator=True,
)
# TODO: find and implement meaningful relationships
# self.helper.api.stix_core_relationship.create(
# fromId=actor_domain_obs["id"],
# toId=actor_ip_obs["id"],
# relationship_type="resolves-to",
# createdBy=self.author["id"],
# )
# Make sure we only process this specific IP once
already_processed.append(c2_beacon.actorip)
except Exception as err:
logger.error(f"error processing c2 beacons: {err}")
return None
class ExportFileCsv:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def export_dict_list_to_csv(self, data):
output = io.StringIO()
headers = sorted(set().union(*(d.keys() for d in data)))
csv_data = [headers]
for d in data:
row = []
for h in headers:
if h not in d:
row.append("")
elif isinstance(d[h], str):
row.append(d[h])
elif isinstance(d[h], list):
if len(d[h]) > 0 and isinstance(d[h][0], str):
row.append(",".join(d[h]))
elif len(d[h]) > 0 and isinstance(d[h][0], dict):
rrow = []
for r in d[h]:
if "name" in r:
rrow.append(r["name"])
elif "definition" in r:
rrow.append(r["definition"])
row.append(",".join(rrow))
else:
row.append("")
elif isinstance(d[h], dict):
if "name" in d[h]:
row.append(d[h]["name"])
else:
row.append("")
else:
row.append("")
csv_data.append(row)
writer = csv.writer(output,
delimiter=";",
quotechar='"',
quoting=csv.QUOTE_ALL)
writer.writerows(csv_data)
return output.getvalue()
def _process_message(self, data):
entity_id = data["entity_id"]
entity_type = data["entity_type"]
file_name = data["file_name"]
file_context = data["file_context"]
export_type = data["export_type"]
list_args = data["list_args"]
if entity_id is not None:
self.helper.log_info("Exporting: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
entity_data = self.helper.api.stix_domain_entity.read(id=entity_id)
entities_list = [entity_data]
if "objectRefsIds" in entity_data:
for id in entity_data["objectRefsIds"]:
entity = self.helper.api.stix_domain_entity.read(id=id)
entities_list.append(entity)
csv_data = self.export_dict_list_to_csv(entities_list)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
self.helper.api.stix_domain_entity.push_entity_export(
entity_id, file_name, csv_data)
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
else:
self.helper.log_info("Exporting list: " + entity_type + "/" +
export_type + " to " + file_name)
if IdentityTypes.has_value(entity_type):
if list_args["filters"] is not None:
list_args["filters"].append({
"key": "entity_type",
"values": [entity_type]
})
else:
list_args["filters"] = [{
"key": "entity_type",
"values": [entity_type]
}]
entity_type = "identity"
# List
lister = {
"identity": self.helper.api.identity.list,
"threat-actor": self.helper.api.threat_actor.list,
"intrusion-set": self.helper.api.intrusion_set.list,
"campaign": self.helper.api.campaign.list,
"incident": self.helper.api.incident.list,
"malware": self.helper.api.malware.list,
"tool": self.helper.api.tool.list,
"vulnerability": self.helper.api.vulnerability.list,
"attack-pattern": self.helper.api.attack_pattern.list,
"course-of-action": self.helper.api.course_of_action.list,
"report": self.helper.api.report.list,
"indicator": self.helper.api.indicator.list,
}
do_list = lister.get(
entity_type.lower(),
lambda **kwargs: self.helper.
log_error('Unknown object type "' + entity_type +
'", doing nothing...'),
)
entities_list = do_list(
search=list_args["search"],
filters=list_args["filters"],
orderBy=list_args["orderBy"],
orderMode=list_args["orderMode"],
getAll=True,
)
csv_data = self.export_dict_list_to_csv(entities_list)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + " to " + file_name)
self.helper.api.stix_domain_entity.push_list_export(
entity_type, file_name, csv_data, file_context,
json.dumps(list_args))
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + " to " + file_name)
return ["Export done"]
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class SightingConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.organization = get_config_variable(
"ORGANIZATION", ["sighting", "organization"], config
)
self.labels = get_config_variable(
"SIGHTING_LABELS", ["sighting", "labels"], config
).split(",")
self.identity = self.helper.api.identity.create(
type="Organization",
name=self.organization,
description=self.organization + " created by the Sighting connector",
)
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
indicator = self.helper.api.indicator.read(
customAttributes="""
id
observables {
edges {
node {
id
}
}
}
""",
filters=[
{
"key": "pattern",
"values": [observable["observable_value"]],
"operator": "wildcard",
}
],
)
# If no indicator, return
if indicator is None:
return
# Check if indicator and observable are already linked, if not, create the "based-on" relationship
current_observable_is_present = False
for indicator_observable in indicator["observables"]:
if indicator_observable["id"] == observable["id"]:
current_observable_is_present = True
if not current_observable_is_present:
# Create a relationship "based-on"
self.helper.api.stix_core_relationship.create(
fromId=indicator["id"],
toId=observable["id"],
relationship_type="based-on",
)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class HygieneConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
warninglists_slow_search = bool(
get_config_variable(
"HYGIENE_WARNINGLISTS_SLOW_SEARCH",
["hygiene", "warninglists_slow_search"],
config,
default=False,
)
)
self.enrich_subdomains = bool(
get_config_variable(
"HYGIENE_ENRICH_SUBDOMAINS",
["hygiene", "enrich_subdomains"],
config,
default=False,
)
)
self.helper.log_info(f"Warning lists slow search: {warninglists_slow_search}")
self.warninglists = WarningLists(slow_search=warninglists_slow_search)
# Create Hygiene Tag
self.label_hygiene = self.helper.api.label.create(
value="Hygiene", color="#fc0341"
)
if self.enrich_subdomains:
self.label_hygiene_parent = self.helper.api.label.create(
value="Hygiene_parent", color="#fc0341"
)
def _process_observable(self, observable) -> str:
# Extract IPv4, IPv6 and Domain from entity data
observable_value = observable["observable_value"]
observable_type = observable["entity_type"]
# Search in warninglist
result = self.warninglists.search(observable_value)
# If not found and the domain is a subdomain, search with the parent.
use_parent = False
if not result and self.enrich_subdomains == True:
if observable_type == "Domain-Name":
ext = tldextract.extract(observable_value)
if observable_value != ext.domain + "." + ext.suffix:
result = self.warninglists.search(ext.domain + "." + ext.suffix)
use_parent = True
# Iterate over the hits
if result:
self.helper.log_info(
"Hit found for %s in warninglists" % (observable_value)
)
for hit in result:
self.helper.log_info(
"Type: %s | Name: %s | Version: %s | Descr: %s"
% (hit.type, hit.name, hit.version, hit.description)
)
# We set the score based on the number of warning list entries
if len(result) >= 5:
score = "5"
elif len(result) >= 3:
score = "10"
elif len(result) == 1:
score = "15"
else:
score = "20"
self.helper.log_info(
f"number of hits ({len(result)}) setting score to {score}"
)
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"],
label_id=self.label_hygiene["id"]
if use_parent == False
else self.label_hygiene_parent["id"],
)
self.helper.api.stix_cyber_observable.update_field(
id=observable["id"],
input={"key": "x_opencti_score", "value": score},
)
for indicator_id in observable["indicatorsIds"]:
self.helper.api.stix_domain_object.add_label(
id=indicator_id,
label_id=self.label_hygiene["id"]
if use_parent == False
else self.label_hygiene_parent["id"],
)
self.helper.api.stix_domain_object.update_field(
id=indicator_id,
input={"key": "x_opencti_score", "value": score},
)
# Create external references
external_reference_id = self.helper.api.external_reference.create(
source_name="misp-warninglist",
url="https://github.com/MISP/misp-warninglists/tree/main/"
+ LIST_MAPPING[hit.name],
external_id=hit.name,
description=hit.description,
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable["id"],
external_reference_id=external_reference_id["id"],
)
return "Observable value found on warninglist and tagged accordingly"
def _process_message(self, data) -> str:
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
if observable is None:
raise ValueError(
"Observable not found (or the connector does not has access to this observable, check the group of the connector user)"
)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class IpInfoConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.token = get_config_variable("IPINFO_TOKEN", ["ipinfo", "token"],
config)
self.max_tlp = get_config_variable("IPINFO_MAX_TLP",
["ipinfo", "max_tlp"], config)
def _generate_stix_bundle(self, country, city, loc, observable_id):
# Generate stix bundle
country_location = Location(
id=OpenCTIStix2Utils.generate_random_stix_id("location"),
name=country.name,
country=country.official_name
if hasattr(country, "official_name") else country.name,
custom_properties={
"x_opencti_location_type":
"Country",
"x_opencti_aliases": [
country.official_name
if hasattr(country, "official_name") else country.name
],
},
)
loc_split = loc.split(",")
city_location = Location(
id=OpenCTIStix2Utils.generate_random_stix_id("location"),
name=city,
country=country.official_name
if hasattr(country, "official_name") else country.name,
latitude=loc_split[0],
longitude=loc_split[1],
custom_properties={"x_opencti_location_type": "City"},
)
city_to_country = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="located-at",
source_ref=city_location.id,
target_ref=country_location.id,
)
observable_to_city = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="located-at",
source_ref=observable_id,
target_ref=city_location.id,
confidence=self.helper.connect_confidence_level,
)
return Bundle(
objects=[
country_location,
city_location,
city_to_country,
observable_to_city,
],
allow_custom=True,
).serialize()
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
# Extract IP from entity data
observable_id = observable["standard_id"]
observable_value = observable["value"]
# Get the geo loc from the API
api_url = "https://ipinfo.io/" + observable_value + "/json/?token=" + self.token
response = requests.request(
"GET",
api_url,
headers={
"accept": "application/json",
"content-type": "application/json"
},
)
json_data = response.json()
country = pycountry.countries.get(alpha_2=json_data["country"])
if country is None:
raise ValueError(
"IpInfo was not able to find a country for this IP address")
bundle = self._generate_stix_bundle(country, json_data["city"],
json_data["loc"], observable_id)
bundles_sent = self.helper.send_stix2_bundle(bundle)
return "Sent " + str(
len(bundles_sent)) + " stix bundle(s) for worker import"
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class VirusTotalConnector:
"""VirusTotal connector."""
_CONNECTOR_RUN_INTERVAL_SEC = 60 * 60
_API_URL = "https://www.virustotal.com/api/v3"
def __init__(self):
# Instantiate the connector helper from config
config_file_path = Path(
__file__).parent.parent.resolve() / "config.yml"
config = (yaml.load(open(config_file_path, encoding="utf-8"),
Loader=yaml.FullLoader)
if config_file_path.is_file() else {})
self.helper = OpenCTIConnectorHelper(config)
token = get_config_variable("VIRUSTOTAL_TOKEN",
["virustotal", "token"], config)
self.max_tlp = get_config_variable("VIRUSTOTAL_MAX_TLP",
["virustotal", "max_tlp"], config)
self.client = VirusTotalClient(self._API_URL, token)
# Cache to store YARA rulesets.
self.yara_cache = {}
def _create_yara_indicator(
self,
yara: dict,
valid_from: Optional[int] = None) -> Optional[Indicator]:
"""Create an indicator containing the YARA rule from VirusTotal."""
valid_from_date = (datetime.datetime.min if valid_from is None else
datetime.datetime.utcfromtimestamp(valid_from))
ruleset_id = yara.get("ruleset_id", "No ruleset id provided")
self.helper.log_info(f"[VirusTotal] Retrieving ruleset {ruleset_id}")
# Lookup in the cache for the ruleset id, otherwise, request VirusTotal API.
if ruleset_id in self.yara_cache:
self.helper.log_debug(
f"Retrieving YARA ruleset {ruleset_id} from cache.")
ruleset = self.yara_cache[ruleset_id]
else:
self.helper.log_debug(
f"Retrieving YARA ruleset {ruleset_id} from API.")
ruleset = self.client.get_yara_ruleset(ruleset_id)
self.yara_cache[ruleset_id] = ruleset
# Parse the rules to find the correct one.
parser = plyara.Plyara()
rules = parser.parse_string(ruleset["data"]["attributes"]["rules"])
rule_name = yara.get("rule_name", "No ruleset name provided")
rule = [r for r in rules if r["rule_name"] == rule_name]
if len(rule) == 0:
self.helper.log_warning(f"No YARA rule for rule name {rule_name}")
return None
return self.helper.api.indicator.create(
name=yara.get("rule_name", "No rulename provided"),
description=json.dumps({
"description":
yara.get("description", "No description provided"),
"author":
yara.get("author", "No author provided"),
"source":
yara.get("source", "No source provided"),
"ruleset_id":
ruleset_id,
"ruleset_name":
yara.get("ruleset_name", "No ruleset name provided"),
}),
pattern=plyara.utils.rebuild_yara_rule(rule[0]),
pattern_type="yara",
valid_from=self.helper.api.stix2.format_date(valid_from_date),
x_opencti_main_observable_type="StixFile",
)
def _process_file(self, observable):
json_data = self.client.get_file_info(observable["observable_value"])
if "error" in json_data:
if json_data["error"]["message"] == "Quota exceeded":
self.helper.log_info("Quota reached, waiting 1 hour.")
sleep(self._CONNECTOR_RUN_INTERVAL_SEC)
elif "not found" in json_data["error"]["message"]:
self.helper.log_info("File not found on VirusTotal.")
return "File not found on VirusTotal."
else:
raise ValueError(json_data["error"]["message"])
if "data" in json_data:
data = json_data["data"]
attributes = data["attributes"]
# Update the current observable
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=observable["id"],
input={
"key": "hashes.MD5",
"value": attributes["md5"]
},
)
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "hashes.SHA-1",
"value": attributes["sha1"]
},
)
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "hashes.SHA-256",
"value": attributes["sha256"]
},
)
if observable["entity_type"] == "StixFile":
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "size",
"value": str(attributes["size"])
},
)
if observable["name"] is None and len(attributes["names"]) > 0:
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "name",
"value": attributes["names"][0]
},
)
del attributes["names"][0]
if len(attributes["names"]) > 0:
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "x_opencti_additional_names",
"value": attributes["names"],
},
)
# Create external reference
external_reference = self.helper.api.external_reference.create(
source_name="VirusTotal",
url="https://www.virustotal.com/gui/file/" +
attributes["sha256"],
description=attributes["magic"],
)
# Create tags
for tag in attributes["tags"]:
tag_vt = self.helper.api.label.create(value=tag,
color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=final_observable["id"], label_id=tag_vt["id"])
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
if "crowdsourced_yara_results" in attributes:
self.helper.log_info(
"[VirusTotal] adding yara results to file.")
# Add YARA rules (only if a rule is given).
yaras = list(
filter(
None,
[
self._create_yara_indicator(
yara, attributes.get("creation_date", None))
for yara in attributes["crowdsourced_yara_results"]
],
))
self.helper.log_debug(
f"[VirusTotal] Indicators created: {yaras}")
# Create the relationships (`related-to`) between the yaras and the file.
for yara in yaras:
self.helper.api.stix_core_relationship.create(
fromId=final_observable["id"],
toId=yara["id"],
relationship_type="related-to",
)
return "File found on VirusTotal, knowledge attached."
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
return self._process_file(observable)
def start(self):
"""Start the main loop."""
self.helper.listen(self._process_message)
class ShodanConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.token = get_config_variable("SHODAN_TOKEN", ["shodan", "token"],
config)
self.max_tlp = get_config_variable("SHODAN_MAX_TLP",
["shodan", "max_tlp"], config)
self.shodanAPI = shodan.Shodan(self.token)
def _generate_host_description(self, shodanHostResponse):
# Generate Hostname Desc Block
Hostnames = "Hostnames:"
for host in shodanHostResponse["hostnames"]:
Hostnames = Hostnames + f"\n - {host}"
# Generate Domain Desc Block
Domains = "Domains:"
for domain in shodanHostResponse["domains"]:
Domains = Domains + f"\n - {domain}"
# Generate Services Desc Block
Services = "Services:\n"
for service in shodanHostResponse["data"]:
serviceData = service["data"].strip()
Services = (
Services +
f'\n**{str(service["port"])}:**\n```\n{serviceData}\n```')
if "opts" in service:
print(service["opts"])
if "heartbleed" in service["opts"]:
Services = (
Services +
f'\nHEARTBLEED: {service["opts"]["heartbleed"]}')
Services = Services + "\n------------------"
# Create the description for the Observable
Observable_Description = f"""
**ORG:** {shodanHostResponse["org"]}
**ISP:** {shodanHostResponse["isp"]}
**OS:** {str(shodanHostResponse["os"])}
--------------------------
{Hostnames}
--------------------------
{Domains}
--------------------------
{Services}
"""
return Observable_Description
def _generate_x509(self, shodanHostResponse):
x509s = []
for service in shodanHostResponse["data"]:
if "ssl" in service:
sslObject = service["ssl"]
issued: datetime = datetime.strptime(
sslObject["cert"]["issued"], "%Y%m%d%H%M%SZ")
expires: datetime = datetime.strptime(
sslObject["cert"]["expires"], "%Y%m%d%H%M%SZ")
x509 = self.helper.api.stix_cyber_observable.create(
observableData={
"type":
"x509-certificate",
"issuer":
", ".join((
f"{k}={v}"
for k, v in sslObject["cert"]["subject"].items())),
"validity_not_before":
issued.isoformat().split(".")[0] + "Z",
"validity_not_after":
expires.isoformat().split(".")[0] + "Z",
"subject":
", ".join(
(f"{k}={v}"
for k, v in sslObject["cert"]["issuer"].items())),
"serial_number":
":".join([
str(sslObject["cert"]["serial"])[i:i + 2]
for i in range(
0, len(str(sslObject["cert"]["serial"])), 2)
]),
# "version": str(sslObject["cert"]["version"]),
"hashes": {
"sha256":
sslObject["cert"]["fingerprint"]["sha256"],
"sha1": sslObject["cert"]["fingerprint"]["sha1"],
},
"signature_algorithm":
sslObject["cert"]["sig_alg"],
"subject_public_key_algorithm":
sslObject["cert"]["pubkey"]["type"],
},
update=True,
)
x509s.append(x509)
return x509s
def _generate_domains(self, shodanHostResponse):
domains = []
for domain in shodanHostResponse["domains"]:
domainX = self.helper.api.stix_cyber_observable.create(
observableData={
"type": "domain-name",
"value": domain,
},
update=True,
)
domains.append(domainX)
return domains
def _convert_shodan_to_stix(self, shodanHostResponse, observable):
# --------------------------------------------------------------------
# Helpers
# --------------------------------------------------------------------
# Now
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
# Pull Tags via API
tags = shodanHostResponse["tags"]
# Create tags
for tag in shodanHostResponse["tags"]:
tag_shodan = self.helper.api.label.create(value=tag)
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"], label_id=tag_shodan["id"])
# Create description
Description = self._generate_host_description(shodanHostResponse)
x509s = self._generate_x509(shodanHostResponse)
domains = self._generate_domains(shodanHostResponse)
# Create ASN Helper Object
ASNumber = int(shodanHostResponse["asn"].replace("AS", ""))
asn = self.helper.api.stix_cyber_observable.create(
observableData={
"type": "autonomous-system",
"number": ASNumber,
"name": shodanHostResponse["asn"],
},
update=True,
objectLabel=tags,
)
# --------------------------------------------------------------------
# STIX Objects
# --------------------------------------------------------------------
# Create Indicator
final_indicator = self.helper.api.indicator.create(
name=shodanHostResponse["ip_str"],
description=Description,
pattern_type="stix",
pattern=f"[ipv4-addr:value = '{shodanHostResponse['ip_str']}']",
x_opencti_main_observable_type="IPv4-Addr",
valid_from=now,
update=True,
objectLabel=tags,
confidence=self.helper.connect_confidence_level,
x_opencti_detection=True,
)
# Update the current observable
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=observable["id"],
key="x_opencti_description",
value=Description)
for tag in tags:
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"], label_name=tag)
# --------------------------------------------------------------------
# Relationships
# --------------------------------------------------------------------
# Link Indicator to Observable
self.helper.api.stix_core_relationship.create(
fromId=final_indicator["id"],
toId=observable["id"],
relationship_type="based-on",
update=True,
confidence=self.helper.connect_confidence_level,
)
# Link ASN to Observable
self.helper.api.stix_cyber_observable_relationship.create(
fromId=final_observable["id"],
toId=asn["id"],
relationship_type="obs_belongs-to",
update=True,
confidence=self.helper.connect_confidence_level,
)
# Link x509 to Observable
for x509 in x509s:
self.helper.api.stix_core_relationship.create(
fromId=observable["id"],
toId=x509["id"],
relationship_type="related-to",
update=True,
confidence=self.helper.connect_confidence_level,
)
# Link Domains to Observable
for domain in domains:
self.helper.api.stix_cyber_observable_relationship.create(
fromId=domain["id"],
toId=observable["id"],
relationship_type="resolves-to",
update=True,
confidence=self.helper.connect_confidence_level,
)
# --------------------------------------------------------------------
# References
# --------------------------------------------------------------------
# Create external reference to shodan
external_reference = self.helper.api.external_reference.create(
source_name="Shodan",
url="https://shodan.io/host/" + shodanHostResponse["ip_str"],
description=
f'[{shodanHostResponse["country_code"]}] [{shodanHostResponse["region_code"]} - {shodanHostResponse["city"]}] - {" ".join(shodanHostResponse["hostnames"])}',
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
# Extract IP from entity data
observable_value = observable["value"]
# Get Shodan API Response
try:
response = self.shodanAPI.host(observable_value)
except Exception as e:
return str(e)
# Process and send Shodan Data to OpenCTI
self._convert_shodan_to_stix(response, observable)
return "[SUCCESS] Shodan IP Found, data sent in"
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class AbuseIPDBConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.SafeLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.api_key = get_config_variable("ABUSEIPDB_API_KEY",
["abuseipdb", "api_key"], config)
self.max_tlp = get_config_variable("ABUSEIPDB_MAX_TLP",
["abuseipdb", "max_tlp"], config)
self.whitelist_label = self.helper.api.label.create(value="whitelist",
color="#4caf50")
@staticmethod
def extract_abuse_ipdb_category(category_number):
# Reference: https://www.abuseipdb.com/categories
mapping = {
"3": "Fraud Orders",
"4": "DDOS Attack",
"5": "FTP Brute-Force",
"6": "Ping of Death",
"7": "Phishing",
"8": "Fraud VOIP",
"9": "Open Proxy",
"10": "Web Spam",
"11": "Email Spam",
"12": "Blog Spam",
"13": "VPN IP",
"14": "Port Scan",
"15": "Hacking",
"16": "SQL Injection",
"17": "Spoofing",
"18": "Brute Force",
"19": "Bad Web Bot",
"20": "Exploited Host",
"21": "Web App Attack",
"22": "SSH",
"23": "IoT Targeted",
}
return mapping.get(str(category_number), "unknown category")
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
# Extract IP from entity data
observable_id = observable["standard_id"]
observable_value = observable["value"]
url = "https://api.abuseipdb.com/api/v2/check"
headers = {
"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded",
"Key": "%s" % self.api_key,
}
params = {
"maxAgeInDays": 365,
"verbose": "True",
"ipAddress": observable_value
}
response = requests.get(url, headers=headers, params=params)
data = response.json()
data = data["data"]
self.helper.api.stix_cyber_observable.update_field(
id=observable_id,
key="x_opencti_score",
value=str(data["abuseConfidenceScore"]),
)
if data["isWhitelisted"]:
external_reference = self.helper.api.external_reference.create(
source_name="AbuseIPDB (whitelist)",
url="https://www.abuseipdb.com/check/" + observable_value,
description="This IP address is from within our whitelist.",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable_id,
external_reference_id=external_reference["id"])
self.helper.api.stix_cyber_observable.add_label(
id=observable_id, label_id=self.whitelist_label["id"])
return "IP found in AbuseIPDB WHITELIST."
if len(data["reports"]) > 0:
for report in data["reports"]:
country = self.helper.api.stix_domain_object.get_by_stix_id_or_name(
name=report["reporterCountryName"])
self.helper.api.stix_sighting_relationship.create(
fromId=observable_id,
toId=country["id"],
count=1,
first_seen=report["reportedAt"],
last_seen=report["reportedAt"],
)
for category in report["categories"]:
category_text = self.extract_abuse_ipdb_category(category)
label = self.helper.api.label.create(value=category_text)
self.helper.api.stix_cyber_observable.add_label(
id=observable_id, label_id=label["id"])
return "IP found in AbuseIPDB with reports, knowledge attached."
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class IpInfoConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}
self.helper = OpenCTIConnectorHelper(config)
self.token = os.getenv('IPINFO_TOKEN') or config['ipinfo']['token']
def _generate_stix_bundle(self, country, city, observable_id):
# Generate stix bundle
country_identity = Identity(
name=country.name,
identity_class='group',
custom_properties={
'x_opencti_identity_type': 'country',
'x_opencti_alias': [country.official_name],
}
)
city_identity = Identity(
name=city,
identity_class='group',
custom_properties={
'x_opencti_identity_type': 'city'
}
)
city_to_country = Relationship(
relationship_type='localization',
source_ref=city_identity.id,
target_ref=country_identity.id,
)
observable_to_city = Relationship(
relationship_type='localization',
source_ref=observable_id,
target_ref=city_identity.id,
custom_properties={
'x_opencti_weight': self.helper.connect_confidence_level
}
)
return Bundle(objects=[country_identity, city_identity,
city_to_country, observable_to_city]).serialize()
def _process_message(self, data):
entity_id = data['entity_id']
observable = self.helper.api.get_stix_observable(entity_id)
# Extract IP from entity data
observable_id = observable['stix_id_key']
observable_value = observable['observable_value']
# Get the geo loc from the API
api_url = 'https://ipinfo.io/' + observable_value + '?token=' + self.token
response = requests.request("GET", api_url, headers={
'accept': "application/json",
'content-type': "application/json",
})
json_data = json.loads(response.text)
country = pycountry.countries.get(alpha_2=json_data['country'])
bundle = self._generate_stix_bundle(country, json_data['city'], observable_id)
bundles_sent = self.helper.send_stix2_bundle(bundle)
return ['Sent ' + str(len(bundles_sent)) + ' stix bundle(s) for worker import']
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class HybridAnalysis:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.api_key = get_config_variable("HYBRID_ANALYSIS_TOKEN",
["hybrid_analysis", "api_key"],
config)
self.environment_id = get_config_variable(
"HYBRID_ANALYSIS_ENVIRONMENT_ID",
["hybrid_analysis", "environment_id"],
config,
True,
110,
)
self.max_tlp = get_config_variable("HYBRID_ANALYSIS_MAX_TLP",
["hybrid_analysis", "max_tlp"],
config)
self.api_url = "https://www.hybrid-analysis.com/api/v2"
self.headers = {
"api-key": self.api_key,
"user-agent": "OpenCTI Hybrid Analysis Connector - Version 4.5.5",
"accept": "application/json",
}
self.identity = self.helper.api.identity.create(
type="Organization",
name="Hybrid Analysis",
description="Hybrid Analysis Sandbox.",
)["standard_id"]
self._CONNECTOR_RUN_INTERVAL_SEC = 60 * 60
def _send_knowledge(self, observable, report):
bundle_objects = []
final_observable = observable
if observable["entity_type"] in ["StixFile", "Artifact"]:
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.MD5",
value=report["md5"])
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.SHA-1",
value=report["sha1"])
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.SHA-256",
value=report["sha256"],
)
if "name" not in final_observable or final_observable[
"name"] is None:
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="x_opencti_additional_names",
value=report["submit_name"],
operation="add",
)
if final_observable["entity_type"] == "StixFile":
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="size",
value=str(report["size"]),
)
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="x_opencti_score",
value=str(report["threat_score"]),
)
# Create external reference
external_reference = self.helper.api.external_reference.create(
source_name="Hybrid Analysis",
url="https://www.hybrid-analysis.com/sample/" + report["sha256"],
description="Hybrid Analysis Report",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
# Create tags
for tag in report["type_short"]:
tag_ha = self.helper.api.label.create(value=tag, color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=final_observable["id"], label_id=tag_ha["id"])
# Attach the TTPs
for tactic in report["mitre_attcks"]:
if (tactic["malicious_identifiers_count"] > 0
or tactic["suspicious_identifiers_count"] > 0):
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id(
"attack-pattern"),
created_by_ref=self.identity,
name=tactic["technique"],
custom_properties={
"x_mitre_id": tactic["attck_id"],
},
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=attack_pattern.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
# Attach the domains
for domain in report["domains"]:
domain_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key="Domain-Name.value",
value=domain,
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=domain_stix.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(domain_stix)
bundle_objects.append(relationship)
# Attach the IP addresses
for host in report["hosts"]:
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=self.detect_ip_version(host) + ".value",
value=host,
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=host_stix.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
# Attach other files
for file in report["extracted_files"]:
if file["threat_level"] > 0:
file_stix = File(
id=OpenCTIStix2Utils.generate_random_stix_id("file"),
hashes={
"MD5": file["md5"],
"SHA-1": file["sha1"],
"SHA-256": file["sha256"],
},
size=file["size"],
name=file["name"],
custom_properties={"x_opencti_labels": file["type_tags"]},
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="drops",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=file_stix.id,
)
bundle_objects.append(file_stix)
bundle_objects.append(relationship)
for tactic in report["mitre_attcks"]:
if (tactic["malicious_identifiers_count"] > 0
or tactic["suspicious_identifiers_count"] > 0):
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id(
"attack-pattern"),
created_by_ref=self.identity,
name=tactic["technique"],
custom_properties={
"x_mitre_id": tactic["attck_id"],
},
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=attack_pattern.id,
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
if len(bundle_objects) > 0:
bundle = Bundle(objects=bundle_objects).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return ("Sent " + str(len(bundles_sent)) +
" stix bundle(s) for worker import")
else:
return "Nothing to attach"
def _submit_url(self, observable):
self.helper.log_info("Observable is a URL, triggering the sandbox...")
values = {
"url": observable["observable_value"],
"environment_id": self.environment_id,
}
r = requests.post(
self.api_url + "/submit/url",
headers=self.headers,
data=values,
)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
job_id = result["job_id"]
state = "IN_QUEUE"
self.helper.log_info("Analysis in progress...")
while state == "IN_QUEUE" or state == "IN_PROGRESS":
r = requests.get(
self.api_url + "/report/" + job_id + "/state",
headers=self.headers,
)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
state = result["state"]
time.sleep(30)
if state == "ERROR":
raise ValueError(result["error"])
r = requests.get(
self.api_url + "/report/" + job_id + "/summary",
headers=self.headers,
)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
self.helper.log_info("Analysis done, attaching knowledge...")
return self._send_knowledge(observable, result)
def _trigger_sandbox(self, observable):
self.helper.log_info("File not found in HA, triggering the sandbox...")
file_name = observable["importFiles"][0]["name"]
file_uri = observable["importFiles"][0]["id"]
file_content = self.helper.api.fetch_opencti_file(
self.api_url + file_uri, True)
# Write the file
f = open(file_name, "wb")
f.write(file_content)
f.close()
files = {"file": open(file_name, "rb")}
values = {"environment_id": self.environment_id}
r = requests.post(
self.api_url + "/submit/file",
headers=self.headers,
files=files,
data=values,
)
os.remove(file_name)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
job_id = result["job_id"]
state = "IN_QUEUE"
self.helper.log_info("Analysis in progress...")
while state == "IN_QUEUE" or state == "IN_PROGRESS":
r = requests.get(
self.api_url + "/report/" + job_id + "/state",
headers=self.headers,
)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
state = result["state"]
time.sleep(30)
if state == "ERROR":
raise ValueError(result["error"])
r = requests.get(
self.api_url + "/report/" + job_id + "/summary",
headers=self.headers,
)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
self.helper.log_info("Analysis done, attaching knowledge...")
return self._send_knowledge(observable, result)
def _process_observable(self, observable):
self.helper.log_info("Processing the observable " +
observable["observable_value"])
# If File or Artifact
result = []
if observable["entity_type"] in ["StixFile", "Artifact"]:
# First, check if the file is present is HA
values = {"hash": observable["observable_value"]}
r = requests.post(
self.api_url + "/search/hash",
headers=self.headers,
data=values,
)
if r.status_code > 299:
raise ValueError(r.text)
result = r.json()
if len(result) > 0:
# One report is found
self.helper.log_info("Already found in HA, attaching knowledge...")
return self._send_knowledge(observable, result[0])
# If URL
if observable["entity_type"] in [
"Url", "Domain-Name", "X-OpenCTI-Hostname"
]:
return self._submit_url(observable)
# If no file
if "importFiles" not in observable or len(
observable["importFiles"]) == 0:
return "Observable not found and no file to upload in the sandbox"
return self._trigger_sandbox(observable)
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
if observable is None:
raise ValueError(
"Observable not found "
"(may be linked to data seggregation, check your group and permissions)"
)
# Extract TLP
tlp = "TLP:WHITE"
for marking_definition in observable["objectMarking"]:
if marking_definition["definition_type"] == "TLP":
tlp = marking_definition["definition"]
if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp):
raise ValueError(
"Do not send any data, TLP of the observable is greater than MAX TLP"
)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
def detect_ip_version(self, value):
if len(value) > 16:
return "IPv6-Addr"
else:
return "IPv4-Addr"
class ExportFileCsv:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
def export_dict_list_to_csv(self, data):
output = io.StringIO()
headers = sorted(set().union(*(d.keys() for d in data)))
csv_data = [headers]
for d in data:
row = []
for h in headers:
if h not in d:
row.append("")
elif isinstance(d[h], str):
row.append(d[h])
elif isinstance(d[h], list):
if len(d[h]) > 0 and isinstance(d[h][0], str):
row.append(",".join(d[h]))
elif len(d[h]) > 0 and isinstance(d[h][0], dict):
rrow = []
for r in d[h]:
if "name" in r:
rrow.append(r["name"])
elif "definition" in r:
rrow.append(r["definition"])
row.append(",".join(rrow))
else:
row.append("")
elif isinstance(d[h], dict):
if "name" in d[h]:
row.append(d[h]["name"])
else:
row.append("")
else:
row.append("")
csv_data.append(row)
writer = csv.writer(output,
delimiter=";",
quotechar='"',
quoting=csv.QUOTE_ALL)
writer.writerows(csv_data)
return output.getvalue()
def _process_message(self, data):
file_name = data["file_name"]
export_scope = data["export_scope"] # single or list
export_type = data["export_type"] # Simple or Full
# max_marking = data["max_marking"] # TODO Implement marking restriction
entity_type = data["entity_type"]
if export_scope == "single":
entity_id = data["entity_id"]
self.helper.log_info("Exporting: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
entity_data = self.helper.api.stix_domain_object.read(id=entity_id)
entities_list = []
if "objectsIds" in entity_data:
for id in entity_data["objectsIds"]:
entity = self.helper.api.stix_domain_object.read(id=id)
if entity is None:
entity = self.helper.api.stix_cyber_observable.read(
id=id)
if entity is not None:
del entity["objectLabelIds"]
entities_list.append(entity)
del entity_data["objectLabelIds"]
del entity_data["objectsIds"]
entities_list.append(entity_data)
csv_data = self.export_dict_list_to_csv(entities_list)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
self.helper.api.stix_domain_object.push_entity_export(
entity_id, file_name, csv_data)
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + "(" + entity_id + ") to " +
file_name)
else:
list_params = data["list_params"]
self.helper.log_info("Exporting list: " + entity_type + "/" +
export_type + " to " + file_name)
final_entity_type = entity_type
if IdentityTypes.has_value(entity_type):
if list_params["filters"] is not None:
list_params["filters"].append({
"key": "entity_type",
"values": [entity_type]
})
else:
list_params["filters"] = [{
"key": "entity_type",
"values": [entity_type]
}]
final_entity_type = "Identity"
if LocationTypes.has_value(entity_type):
if list_params["filters"] is not None:
list_params["filters"].append({
"key": "entity_type",
"values": [entity_type]
})
else:
list_params["filters"] = [{
"key": "entity_type",
"values": [entity_type]
}]
final_entity_type = "Location"
if StixCyberObservableTypes.has_value(entity_type):
if list_params["filters"] is not None:
list_params["filters"].append({
"key": "entity_type",
"values": [entity_type]
})
else:
list_params["filters"] = [{
"key": "entity_type",
"values": [entity_type]
}]
final_entity_type = "Stix-Cyber-Observable"
# List
lister = {
"Attack-Pattern":
self.helper.api.attack_pattern.list,
"Campaign":
self.helper.api.campaign.list,
"Note":
self.helper.api.note.list,
"Observed-Data":
self.helper.api.observed_data.list,
"Opinion":
self.helper.api.opinion.list,
"Report":
self.helper.api.report.list,
"Course-Of-Action":
self.helper.api.course_of_action.list,
"Identity":
self.helper.api.identity.list,
"Indicator":
self.helper.api.indicator.list,
"Infrastructure":
self.helper.api.infrastructure.list,
"Intrusion-Set":
self.helper.api.intrusion_set.list,
"Location":
self.helper.api.location.list,
"Malware":
self.helper.api.malware.list,
"Threat-Actor":
self.helper.api.threat_actor.list,
"Tool":
self.helper.api.tool.list,
"Vulnerability":
self.helper.api.vulnerability.list,
"X-OpenCTI-Incident":
self.helper.api.x_opencti_incident.list,
"Stix-Cyber-Observable":
self.helper.api.stix_cyber_observable.list,
}
do_list = lister.get(
final_entity_type,
lambda **kwargs: self.helper.
log_error('Unknown object type "' + final_entity_type +
'", doing nothing...'),
)
entities_list = do_list(
search=list_params["search"],
filters=list_params["filters"],
orderBy=list_params["orderBy"],
orderMode=list_params["orderMode"],
types=list_params["types"] if "types" in list_params else None,
getAll=True,
)
csv_data = self.export_dict_list_to_csv(entities_list)
self.helper.log_info("Uploading: " + entity_type + "/" +
export_type + " to " + file_name)
if entity_type != "Stix-Cyber-Observable":
self.helper.api.stix_domain_object.push_list_export(
entity_type, file_name, csv_data, json.dumps(list_params))
else:
self.helper.api.stix_cyber_observable.push_list_export(
file_name, csv_data, json.dumps(list_params))
self.helper.log_info("Export done: " + entity_type + "/" +
export_type + " to " + file_name)
return "Export done"
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
