def sent(jd_id):
details = srv.get_jd_details(jd_id)
html = details.get('jd_html')
if request.method == 'GET':
form = TagEntryForm()
present_tags = tag_srv.get_tagged_sentences_for_jd(jd_id).get(
'taggings')
if not present_tags:
present_tags = [{
'sentence': sent,
'tag': 'Tags'
} for sent in details.get('jd_sentences')]
for ptag in present_tags:
st = SentenceTag()
st.sentence = ptag.get('sentence')
st.tag = ptag.get('tag')
form.tag_list.append_entry(st)
return render_template('sent.html', form=form, jd_html=Markup(html))
else:
out = Cut({}, sep='-')
for key, value in request.form.to_dict().items():
out.setdefault(key, value)
out = dict(out)
tags = list(out['tag_list'].values())
tag_srv.perform_sentence_tagging(jd_id, tags)
return render_template('sent.html', tagged=True, jd_html=Markup(html))
def import_cti_event(self,
timestamp: datetime,
data: dict,
is_update: bool = False) -> dict:
logger.debug(f"Querying indicator: { data['x_opencti_id']}")
entity = self.helper.api.indicator.read(id=data["x_opencti_id"])
logger.debug(entity)
_result: dict = {}
_document: Cut = {}
if data["type"] != "indicator":
logger.error(
f"Data type unsupported: {data['type']}. Only 'indicators are currently supported."
)
return None
if is_update is True:
update_time: str = (datetime.now(
tz=timezone.utc).isoformat().replace("+00:00", "Z"))
try:
# Attempt to retreive existing document
logger.debug(f"Retrieving document id: {data['x_opencti_id']}")
_result = self.es_client.get(index=self.idx_pattern,
id=data["x_opencti_id"],
doc_type="_doc")
except NotFoundError:
logger.warn(
f"Document not found to update at /{self.idx}/_doc/{data['x_opencti_id']}"
)
logger.warn("Skipping")
return {}
except RequestError as err:
logger.error(
f"Unexpected error retreiving document at /{self.idx}/_doc/{data['x_opencti_id']}:",
err.__dict__,
)
if _result["found"] is True:
_document = Cut(_result["_source"])
_write_idx = _result["_index"]
if data.get("x_data_update", None):
if data["x_data_update"].get("replace", None):
if entity["pattern_type"] == "stix":
# Pull in any indicator updates
_indicator: dict = self._create_ecs_indicator_stix(
entity)
if _indicator == {}:
return {}
_document["threatintel.indicator"] = _indicator
if entity.get("killChainPhases", None):
phases = []
for phase in sorted(
entity["killChainPhases"],
key=lambda i: (
i["kill_chain_name"],
i["x_opencti_order"],
),
):
phases.append({
"killchain_name":
phase["kill_chain_name"],
"phase_name":
phase["phase_name"],
"opencti_phase_order":
phase["x_opencti_order"],
})
_document.setdefault(
"threatintel.opencti.killchain_phases", phases)
else:
logger.warning(
f"Unsupported indicator pattern type: {entity['pattern_type']}. Skipping."
)
return _document
for k, v in data["x_data_update"].get("replace",
{}).items():
logger.debug(
f"Updating field {k} -> {entity_field_mapping.get(k)} to {v}"
)
try:
_field = entity_field_mapping.get(k)
_document.setdefault(_field, v)
_document[_field] = v
except KeyError as err:
logger.error(
f"Unable to find field mapping for {k}", err)
_document["threatintel.opencti.updated_at"] = update_time
# This scrubs the Cut object and returns a dict
_document = remove_nones(_document)
try:
# Don't render timestamped index since this is an update
# Submit to Elastic index
logger.debug(
f"Updating doc to {_write_idx}:\n {_document}")
self.es_client.index(
index=_write_idx,
id=data["x_opencti_id"],
body=_document,
)
except RequestError as err:
logger.error("Unexpected error:", err, _document)
except Exception as err:
logger.error("Something else happened", err, _document)
return _document
creation_time: str = (datetime.now(
tz=timezone.utc).isoformat().replace("+00:00", "Z"))
_document: dict = {
"@timestamp": timestamp,
"event": {
"created": creation_time,
"kind": "enrichment",
"category": "threat",
"type": "indicator",
"dataset": "threatintel.opencti",
},
"threatintel": {},
}
if len(entity.get("externalReferences", [])) > 0:
_document["event"]["reference"] = [
item.get("url", None) for item in entity["externalReferences"]
]
if self.config.get("opencti.platform_url") is not None:
_document["event"]["url"] = urllib.parse.urljoin(
f"{self.config.get('opencti.platform_url')}",
f"/dashboard/observations/indicators/{entity['id']}",
)
_document["event"]["risk_score"] = entity.get("x_opencti_score", None)
_document["event"]["risk_score_norm"] = entity.get(
"x_opencti_score", None)
_document["threatintel"]["confidence"] = entity.get("confidence", None)
_document["threatintel"]["confidence_norm"] = entity.get(
"confidence", None)
_document["threatintel"]["opencti"] = {
"internal_id": entity.get("id", None),
"valid_from": entity.get("valid_from", None),
"valid_until": entity.get("valid_until", None),
"enable_detection": entity.get("x_opencti_detection", None),
"original_pattern": entity.get("pattern", None),
"pattern_type": entity.get("pattern_type", None),
"created_at": entity.get("created_at", None),
"updated_at": entity.get("created_at", None),
"revoked": entity.get("revoked", None),
}
if entity.get("killChainPhases", None):
phases = []
for phase in sorted(
entity["killChainPhases"],
key=lambda i: (i["kill_chain_name"], i["x_opencti_order"]),
):
phases.append({
"killchain_name": phase["kill_chain_name"],
"phase_name": phase["phase_name"],
"opencti_phase_order": phase["x_opencti_order"],
})
_document["threatintel"]["opencti"]["killchain_phases"] = phases
if entity.get("x_mitre_platforms", None):
_document["threatintel"]["opencti"]["mitre"] = {
"platforms": entity.get("x_mitre_platforms", None)
}
if entity["pattern_type"] == "stix":
_indicator: dict = self._create_ecs_indicator_stix(entity)
if _indicator == {}:
return {}
_document["threatintel"]["stix"] = {
"id": entity.get("standard_id")
}
_document["threatintel"]["indicator"] = _indicator
else:
logger.warning(
f"Unsupported indicator pattern type: {entity['pattern_type']}. Skipping."
)
return {}
_document = remove_nones(_document)
try:
# Render date-specific index, if we're doing logstash style indices
_write_idx = self.write_idx
m = self.pattern.search(_write_idx)
if m is not None:
m = m.groupdict()
if m.get("modulo", None) is not None:
_fmt = m.get("format") or DM_DEFAULT_FMT
logger.debug(f"{m['modulo']} -> {_fmt}")
_val = dm(m.get("modulo"), now=timestamp).format(_fmt)
_write_idx = self.pattern.sub(_val, _write_idx)
# Submit to Elastic index
logger.debug(f"Indexing doc to {_write_idx}:\n {_document}")
self.es_client.index(
index=_write_idx,
id=data["x_opencti_id"],
body=_document,
)
except RequestError as err:
logger.error("Unexpected error:", err, _document)
except Exception as err:
logger.error("Something else happened", err, _document)
return _document