def analyze_outdated(self):
# do outdated logic
fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
if self.EXPIRATION:
fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.now() - self.EXPIRATION})
fltr &= Q(**self.CUSTOM_FILTER) & Q(_cls__contains=self.ACTS_ON)
self.bulk(Observable.objects(fltr))
def execute(self):
self.export_file_handle = codecs.open(self.output_file, 'w+', "utf-8")
q = Q(tags__name__in=[t.name for t in self.include_tags]) & Q(tags__name__nin=[t.name for t in self.exclude_tags])
q &= Q(_cls__contains=self.acts_on)
output = self.template.render(Observable.objects(q))
self.write(output)
self.export_file_handle.close()
def analyze_outdated(self):
class_filter = Q()
for acts_on in iterify(self.ACTS_ON):
class_filter |= Q(_cls="Observable.{}".format(acts_on))
# do outdated logic
fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
if self.EXPIRATION:
fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.utcnow() - self.EXPIRATION})
fltr &= self.CUSTOM_FILTER & class_filter
self.bulk(Observable.objects(fltr).no_cache())
def execute(self):
q_include = Q()
for t in self.include_tags:
q_include |= Q(tags__match={'name': t.name, 'fresh': True})
q_exclude = Q(tags__name__nin=[t.name for t in self.exclude_tags])
q = Q(tags__not__size=0,
tags__match={'fresh': True}) & q_include & q_exclude & Q(
_cls="Observable.{}".format(self.acts_on))
return self.template.render(
self.filter_ignore_tags(Observable.objects(q).no_cache()),
self.output_file)
def nodesearch(self, query):
result = []
query = re.compile("^{}".format(query), re.IGNORECASE)
observables = Observable.objects(value=query).limit(5)
entities = Entity.objects(name=query).limit(5)
for results in [observables, entities]:
for node in results:
result.append(node.to_mongo())
return render(result)
def nodesearch(self, query):
result = []
query = re.compile("^{}".format(query), re.IGNORECASE)
observables = Observable.objects(value=query).limit(5)
entities = Entity.objects(name=query).limit(5)
for results in [observables, entities]:
for node in results:
result.append(node.to_mongo())
return render(result)
def execute(self):
q_include = Q()
for t in self.include_tags:
q_include |= Q(tags__match={"name": t.name, "fresh": True})
q_exclude = Q(tags__name__nin=[t.name for t in self.exclude_tags])
q = (
Q(tags__not__size=0, tags__match={"fresh": True})
& q_include
& q_exclude
& Q(_cls="Observable.{}".format(self.acts_on))
)
return self.template.render(self.filter_ignore_tags(Observable.objects(q).no_cache()), self.output_file)
def match_observables(observables):
# Remove empty observables
observables = [observable for observable in observables if observable]
extended_query = set(observables) | set(derive(observables))
added_entities = set()
data = {"matches": [], "unknown": set(observables), "entities": [], "known": [], "neighbors": []}
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for o, i in Indicator.search(extended_query):
o = Observable.add_text(o)
match = i.info()
match.update({"observable": o.info(), "related": [], "suggested_tags": set()})
for nodes in i.neighbors().values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description or l.tag}
match["related"].append(node_data)
# uniquely add node information to related entitites
if node.name not in added_entities:
nodeinfo = node.info()
nodeinfo['type'] = node.type
data["entities"].append(nodeinfo)
added_entities.add(node.name)
o_tags = o.get_tags()
[match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]
data["matches"].append(match)
del_from_set(data["unknown"], o.value)
return data
def _get_selected_observables(self, data):
if isinstance(data, MultiDict):
ids = data.getlist("ids")
query = data.get("query")
else:
ids = data.get("ids", None)
query = data.get("query", None)
if ids:
return Observable.objects(id__in=ids)
elif query:
query = json.loads(query)
fltr = query.get("filter", {})
params = query.get("params", {})
regex = params.pop("regex", False)
ignorecase = params.pop("ignorecase", False)
return get_queryset(Observable, fltr, regex, ignorecase)
else:
return []
def _get_selected_observables(self, data):
if isinstance(data, MultiDict):
ids = data.getlist('ids')
query = data.get('query')
else:
ids = data.get('ids', None)
query = data.get('query', None)
if ids:
return Observable.objects(id__in=ids)
elif query:
query = json.loads(query)
fltr = query.get('filter', {})
params = query.get('params', {})
regex = params.pop('regex', False)
ignorecase = params.pop('ignorecase', False)
return get_queryset(Observable, fltr, regex, ignorecase)
else:
return []
def _get_selected_observables(self, data):
if isinstance(data, MultiDict):
ids = data.getlist('ids')
query = data.get('query')
else:
ids = data.get('ids', None)
query = data.get('query', None)
if ids:
return Observable.objects(id__in=ids)
elif query:
query = json.loads(query)
fltr = query.get('filter', {})
params = query.get('params', {})
regex = params.pop('regex', False)
ignorecase = params.pop('ignorecase', False)
return get_queryset(Observable, fltr, regex, ignorecase)
else:
return []
def delete(self, id):
tag = self.objectmanager.objects.get(id=id)
tag.delete()
Observable.objects(tags__name=tag.name).update(pull__tags__name=tag.name)
return render({"status": "ok"})
def match_observables(observables, save_matches=False, fetch_neighbors=True):
# Remove empty observables
observables, extended_query = derive(observables)
observables = list(observables)
data = {
"matches": [],
"unknown": set(observables),
"entities": {},
"known": [],
"neighbors": [],
}
# add to "known"
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
if fetch_neighbors:
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value
not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for nodes in o.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {
"entity": node.type,
"name": node.name,
"link_description": l.description
}
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"observables": []}
if 'observables' not in ent['matches']:
ent['matches']['observables'] = []
info = node.info()
o_info = o.info()
info['matched_observable'] = {
"value": o_info['value'],
"tags": [t['name'] for t in o_info['tags']],
"human_url": o_info['human_url'],
"url": o_info['url'],
"context": o_info['context']
}
if info not in ent['matches']['observables']:
ent['matches']['observables'].append(info)
data['entities'][node.name] = ent
# add to "matches"
for o, i in Indicator.search(extended_query):
if save_matches:
o = Observable.add_text(o)
else:
o = Observable.guess_type(o)(value=o)
try:
o.validate()
except ObservableValidationError:
pass
try:
o = Observable.objects.get(value=o.value)
except Exception:
pass
match = i.info()
match.update({
"observable": o.info() if o.id else o.value,
"related": [],
"suggested_tags": set()
})
for nodes in i.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {
"entity": node.type,
"name": node.name,
"link_description": l.description
}
match["related"].append(node_data)
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"indicators": []}
if 'indicators' not in ent['matches']:
ent['matches']['indicators'] = []
info = i.info()
info['matched_observable'] = o.value
if info not in ent['matches']['indicators']:
ent['matches']['indicators'].append(info)
data['entities'][node.name] = ent
o_tags = o.get_tags()
for tag in node.generate_tags():
if tag not in o_tags:
match["suggested_tags"].add(tag)
data["matches"].append(match)
data['entities'] = data['entities'].values()
return data
def match_observables(observables, save_matches=False, fetch_neighbors=True):
# Remove empty observables
observables = [refang(observable) for observable in observables if observable]
extended_query = set(observables) | set(derive(observables))
data = {
"matches": [],
"unknown": set(observables),
"entities": {},
"known": [],
"neighbors": [],
}
# add to "known"
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
if fetch_neighbors:
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for nodes in o.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"observables": []}
if 'observables' not in ent['matches']:
ent['matches']['observables'] = []
info = node.info()
o_info = o.info()
info['matched_observable'] = {
"value": o_info['value'],
"tags": [t['name'] for t in o_info['tags']],
"human_url": o_info['human_url'],
"url": o_info['url']
}
if info not in ent['matches']['observables']:
ent['matches']['observables'].append(info)
data['entities'][node.name] = ent
# add to "matches"
for o, i in Indicator.search(extended_query):
if save_matches:
o = Observable.add_text(o)
else:
o = Observable.guess_type(o)(value=o)
try:
o.validate()
except ObservableValidationError:
pass
try:
o = Observable.objects.get(value=o.value)
except Exception:
pass
match = i.info()
match.update({"observable": o.info(), "related": [], "suggested_tags": set()})
for nodes in i.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
match["related"].append(node_data)
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"indicators": []}
if 'indicators' not in ent['matches']:
ent['matches']['indicators'] = []
info = i.info()
info['matched_observable'] = o.value
if info not in ent['matches']['indicators']:
ent['matches']['indicators'].append(info)
data['entities'][node.name] = ent
o_tags = o.get_tags()
[match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]
data["matches"].append(match)
data['entities'] = data['entities'].values()
return data
def query(self):
q = Q(tags__name__in=[t.name for t in self.include_tags]) & Q(tags__name__nin=[t.name for t in self.exclude_tags])
for o in Observable.objects(q):
self.format(o)