def derive(strings):
values = set()
observables = set()
for string in iterify(strings):
if string:
try:
t = Observable.guess_type(string)
observable = t(value=string)
observable.normalize()
observables.add(observable)
values.add(observable.value)
except ObservableValidationError:
values.add(string)
new = []
for observable in observables:
for a in analyzers.get(observable.__class__, []):
new.extend([
n for n in a.analyze_string(observable.value)
if n and n not in values
])
if len(new) == 0:
return values, values
else:
_, extended = derive(new + list(values))
return values, extended
def tag(self, new_tags, strict=False):
new_tags = iterify(new_tags)
if strict:
remove = set([t.name for t in self.tags]) - set(new_tags)
for tag in remove:
self.modify(pull__tags__name=tag)
for new_tag in new_tags:
if new_tag.strip() != '':
new_tag = Tag(name=new_tag)
new_tag.clean()
try: # check if tag is a replacement
tag = Tag.objects.get(replaces=new_tag.name)
except DoesNotExist:
tag = Tag.get_or_create(name=new_tag.name)
# search for related entities and link them
for e in Entity.objects(tags__in=[tag.name]):
self.link_to(e, 'Tagged', 'tags')
if not self.modify({"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now()):
self.modify(push__tags=ObservableTag(name=tag.name))
tag.modify(inc__count=1)
return self.reload()
def tag(self, new_tags, strict=False, expiration=None):
"""Tags an observable.
An observable can be tagged to add more information as to what it represents.
Args:
new_tags:
An array of strings to tag the observable with.
strict:
Set to ``True`` to replace all existing tags with the ``new_tags``.
expiration:
Timedelta field after which the Tag will not be considered fresh anymore.
Returns:
A fresh Observable instance as reloaded from the database.
"""
new_tags = iterify(new_tags)
if strict:
remove = set([t.name for t in self.tags]) - set(new_tags)
for tag in remove:
self.modify(pull__tags__name=tag)
tagged = False
for new_tag in new_tags:
if new_tag.strip() != '':
tagged = True
new_tag = Tag(name=new_tag)
new_tag.clean()
try: # check if tag is a replacement
tag = Tag.objects.get(replaces=new_tag.name)
except DoesNotExist:
tag = Tag.get_or_create(name=new_tag.name)
if not expiration:
expiration = tag.default_expiration
extra_tags = tag.produces + [tag]
# search for related entities and link them
for e in Entity.objects(tags__in=[tag.name]):
self.active_link_to(e, 'Tagged', 'tags', clean_old=False)
for tag in extra_tags:
if not self.modify(
{"tags__name": tag.name}, set__tags__S__fresh=True,
set__tags__S__last_seen=datetime.utcnow()):
self.modify(
push__tags=ObservableTag(
name=tag.name, expiration=expiration))
tag.modify(inc__count=1)
if tagged:
self.update(set__last_tagged=datetime.utcnow())
return self.reload()
def inner(*args, **kwargs):
# a user needs at least one of the roles to be granted access
for r in iterify(roles[0]):
if current_user.has_role(r):
return f(*args, **kwargs)
else:
abort(401)
def derive(observables):
"""Indicate that the module needs a specific attribute to work properly.
This function is only useful in abstract modules, in order to make sure
that modules that inherit from this class correctly defines needed class
attributes.
Args:
variables: a string or an array of strings containing the name of
needed class attributes.
Raises:
ModuleInitializationError: One of the needed attributes is not
correctly defined.
"""
new = []
observables = list(iterify(observables))
for i, observable in enumerate(observables):
try:
t = Observable.guess_type(observable)
temp = t(value=observable)
temp.clean()
observable = temp.value
observables[i] = observable
for a in analyzers.get(t, []):
new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
except ObservableValidationError:
pass
if len(new) == 0:
return observables
else:
return observables + derive(new)
def derive(observables):
"""Indicate that the module needs a specific attribute to work properly.
This function is only useful in abstract modules, in order to make sure
that modules that inherit from this class correctly defines needed class
attributes.
Args:
variables: a string or an array of strings containing the name of
needed class attributes.
Raises:
ModuleInitializationError: One of the needed attributes is not
correctly defined.
"""
new = []
for observable in iterify(observables):
try:
t = Observable.guess_type(observable)
for a in analyzers.get(t, []):
new.extend([
n for n in a.analyze_string(observable)
if n and n not in observables
])
except ObservableValidationError:
pass
if len(new) == 0:
return observables
else:
return derive(new + observables)
def info(self):
i = {k: v for k, v in self._data.items() if k in ["name", "description", "last_run", "enabled", "status"]}
i['frequency'] = str(self.frequency)
i['expiration'] = str(self.EXPIRATION)
i['acts_on'] = iterify(self.ACTS_ON)
i['id'] = str(self.id)
return i
def multidelete(self):
data = loads(request.data)
ids = iterify(data['ids'])
for i, inv in enumerate(Investigation.objects(links__id__in=ids)):
inv.modify({"links__id": id}, set__links__S__id="local-{}-{}".format(time.time(), i))
self.objectmanager.objects(id__in=ids).delete()
return render({"deleted": ids})
def post_save(cls, sender, document, created):
if issubclass(sender, Observable):
if created:
for analytics in cls.analytics.itervalues():
if analytics.enabled and sender.__name__ in iterify(
analytics.ACTS_ON):
analytics.each(document)
def tag(self, new_tags, strict=False, expiration=None):
"""Tags an observable.
An observable can be tagged to add more information as to what it represents.
Args:
new_tags:
An array of strings to tag the observable with.
strict:
Set to ``True`` to replace all existing tags with the ``new_tags``.
expiration:
Timedelta field after which the Tag will not be considered fresh anymore.
Returns:
A fresh Observable instance as reloaded from the database.
"""
new_tags = iterify(new_tags)
if strict:
remove = set([t.name for t in self.tags]) - set(new_tags)
for tag in remove:
self.modify(pull__tags__name=tag)
tagged = False
for new_tag in new_tags:
if new_tag.strip() != '':
tagged = True
new_tag = Tag(name=new_tag)
new_tag.clean()
try: # check if tag is a replacement
tag = Tag.objects.get(replaces=new_tag.name)
except DoesNotExist:
tag = Tag.get_or_create(name=new_tag.name)
if not expiration:
expiration = tag.default_expiration
extra_tags = tag.produces + [tag]
# search for related entities and link them
for e in Entity.objects(tags__in=[tag.name]):
self.active_link_to(e, 'Tagged', 'tags', clean_old=False)
for tag in extra_tags:
if not self.modify(
{"tags__name": tag.name},
set__tags__S__fresh=True,
set__tags__S__last_seen=datetime.utcnow()):
self.modify(push__tags=ObservableTag(
name=tag.name, expiration=expiration))
tag.modify(inc__count=1)
if tagged:
self.update(set__last_tagged=datetime.utcnow())
return self.reload()
def derive(strings):
values = set()
observables = set()
for string in iterify(strings):
if string:
try:
t = Observable.guess_type(string)
observable = t(value=string)
observable.normalize()
observables.add(observable)
values.add(observable.value)
except ObservableValidationError:
values.add(string)
new = []
for observable in observables:
for a in analyzers.get(observable.__class__, []):
new.extend([
n for n in a.analyze_string(observable.value)
if n and n not in values
])
if len(new) == 0:
return values, values
else:
_, extended = derive(new + list(values))
return values, extended
def info(self):
i = {k: v for k, v in self._data.items() if k in ["name", "description", "last_run", "enabled", "status"]}
i['frequency'] = str(self.frequency)
i['expiration'] = str(self.EXPIRATION)
i['acts_on'] = iterify(self.ACTS_ON)
i['id'] = str(self.id)
return i
def inner(*args, **kwargs):
# a user needs at least one of the roles to be granted access
for r in iterify(roles[0]):
if current_user.has_role(r):
return f(*args, **kwargs)
else:
abort(401)
def post_save(cls, sender, document, created):
if issubclass(sender, Observable):
if getattr(document, 'new', False):
for analytics in cls.analytics.values():
if analytics.enabled and sender.__name__ in iterify(
analytics.ACTS_ON):
document.new = False
analytics.each(document)
def info(self):
i = {
k: v
for k, v in self._data.items()
if k in ["name", "description", "enabled"]
}
i['acts_on'] = iterify(self.ACTS_ON)
i['id'] = str(self.id)
return i
def multidelete(self):
"""Deletes multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:>json [ObjectID] deleted: Array of Element IDs that were successfully deleted
"""
data = loads(request.data)
ids = iterify(data['ids'])
self.objectmanager.objects(id__in=ids).delete()
return render({"deleted": ids})
def link_to(self, nodes, description, source, first_seen=None, last_seen=None):
links = set()
nodes = iterify(nodes)
for node in nodes:
link = Link.connect(self, node)
link.add_history(source, description, first_seen, last_seen)
links.add(link)
return list(links)
def multidelete(self):
"""Deletes multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:>json [ObjectID] deleted: Array of Element IDs that were successfully deleted
"""
data = loads(request.data)
ids = iterify(data['ids'])
self.objectmanager.objects(id__in=ids).delete()
return render({"deleted": ids})
def link_to(self, nodes, description, source, first_seen=None, last_seen=None):
links = set()
nodes = iterify(nodes)
for node in nodes:
link = Link.connect(self, node)
link.add_history(source, description, first_seen, last_seen)
links.add(link)
return list(links)
def inner(*args, **kwargs):
oname = object_name
if not oname:
oname = getattr(args[0], 'klass', getattr(args[0], 'objectmanager', args[0].__class__)).__name__.lower()
# a user must have all permissions in order to be granted access
for p in iterify(permissions):
if not current_user.has_permission(oname, p):
# improve this and make it redirect to login
abort(401)
else:
return f(*args, **kwargs)
def analyze_outdated(self):
class_filter = Q()
for acts_on in iterify(self.ACTS_ON):
class_filter |= Q(_cls__contains=acts_on)
# do outdated logic
fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
if self.EXPIRATION:
fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.now() - self.EXPIRATION})
fltr &= Q(**self.CUSTOM_FILTER) & class_filter
self.bulk(Observable.objects(fltr))
def analyze_outdated(self):
class_filter = Q()
for acts_on in iterify(self.ACTS_ON):
class_filter |= Q(_cls="Observable.{}".format(acts_on))
# do outdated logic
fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
if self.EXPIRATION:
fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.utcnow() - self.EXPIRATION})
fltr &= self.CUSTOM_FILTER & class_filter
self.bulk(Observable.objects(fltr).no_cache())
def multiupdate(self):
"""Updates multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:query [Object] new: JSON object representing fields to update
:>json [ObjectID] updated: Array of Element IDs that were successfully updated
"""
data = loads(request.data)
ids = iterify(data['ids'])
new_data = data['new']
self.objectmanager.objects(id__in=ids).update(new_data)
return render({"updated": list(self.objectmanager.objects(ids__in=ids))})
def change_all_tags(old_tags, new_tag):
"""Changes tags on all observables
Args:
old_tags: A string or array of strings representing tag names to change
new_tag: The new tag name by which all ``old_tags`` should be replaced
"""
old_tags = iterify(old_tags)
for o in Observable.objects(tags__name__in=old_tags):
for old_tag in old_tags:
o.change_tag(old_tag, new_tag)
def multiupdate(self):
"""Updates multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:query [Object] new: JSON object representing fields to update
:>json [ObjectID] updated: Array of Element IDs that were successfully updated
"""
data = loads(request.data)
ids = iterify(data['ids'])
new_data = data['new']
self.objectmanager.objects(id__in=ids).update(new_data)
return render({"updated": list(self.objectmanager.objects(ids__in=ids))})
def change_all_tags(old_tags, new_tag):
"""Changes tags on all observables
Args:
old_tags: A string or array of strings representing tag names to change
new_tag: The new tag name by which all ``old_tags`` should be replaced
"""
old_tags = iterify(old_tags)
for o in Observable.objects(tags__name__in=old_tags):
for old_tag in old_tags:
o.change_tag(old_tag, new_tag)
def derive(observables):
new = []
for observable in iterify(observables):
try:
t = Observable.guess_type(observable)
for a in analyzers.get(t, []):
new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
except ObservableValidationError:
pass
if len(new) == 0:
return observables
else:
return derive(new + observables)
def link_from_data(observable, data, path, klass, description):
data = get_value_at(data, path)
if data is None:
return []
links = set()
for value in iterify(data):
try:
node = klass.get_or_create(value=value)
except FieldDoesNotExist:
node = klass.get_or_create(name=value)
links.update(observable.active_link_to(node, description, 'DomainTools'))
return list(links)
def active_link_to(self, nodes, description, source, clean_old=True):
links = set()
nodes = iterify(nodes)
for node in nodes:
link = Link.connect(self, node)
link.add_history(source, description, active=True)
links.add(link)
if clean_old:
for link, node in self.outgoing():
if node not in nodes:
active_link = link.get_active(description)
if active_link:
active_link.active = False
link.save(validate=False)
return list(links)
def active_link_to(self, nodes, description, source, clean_old=True):
links = set()
nodes = iterify(nodes)
for node in nodes:
link = Link.connect(self, node)
link.add_history(source, description, active=True)
links.add(link)
if clean_old:
for link, node in self.outgoing():
if node not in nodes:
active_link = link.get_active(description)
if active_link:
active_link.active = False
link.save(validate=False)
return list(links)
def post_save(cls, sender, document, created):
if issubclass(sender, Observable):
if created:
for analytics in cls.analytics.itervalues():
if analytics.enabled and sender.__name__ in iterify(analytics.ACTS_ON):
analytics.each(document)
def add(self, investigation):
investigation.add(iterify(request.json["links"]), iterify(request.json["nodes"]))
return render(investigation.info())
def add_replaces(self, tags):
self.replaces += list(set(iterify(tags) + self.replaces))
return self.save()
def remove(self, id):
i = get_object_or_404(self.objectmanager, id=id)
data = loads(request.data)
i.remove(iterify(data['links']), iterify(data['nodes']))
return render(i.info())
def untag(self, tags):
for tag in iterify(tags):
self.modify(pull__tags__name=tag)
def add_replaces(self, tags):
self.replaces += list(set(iterify(tags) + self.replaces))
return self.save()
def info(self):
i = {k: v for k, v in self._data.items() if k in ["name", "description", "enabled"]}
i['acts_on'] = iterify(self.ACTS_ON)
i['id'] = str(self.id)
return i
def add_produces(self, tags):
tags = [Tag.get_or_create(name=t) for t in iterify(tags)]
self.produces = tags
return self.save()
def add(self, id):
i = get_object_or_404(self.objectmanager, id=id)
data = loads(request.data)
i.add(iterify(data["links"]), iterify(data["nodes"]))
return render(i.info())
def change_all_tags(old_tags, new_tag):
for o in Observable.objects(tags__name__in=iterify(old_tags)):
for old_tag in old_tags:
o.change_tag(old_tag, new_tag)
def add_produces(self, tags):
self.produces = [Tag.get_or_create(name=t) for t in iterify(tags)]
return self.save()
def untag(self, tags):
for tag in iterify(tags):
self.modify(pull__tags__name=tag)
def remove(self, id):
i = get_object_or_404(self.objectmanager, id=id)
data = loads(request.data)
i.remove(iterify(data['links']), iterify(data['nodes']))
return render(i.info())
