def post(self, id=None, action=None):
# special actions
if action:
# special actions
if action == "refresh":
execute_export.delay(id)
return render({"id": id})
elif action == "toggle":
e = Export.objects.get(id=id)
e.enabled = not e.enabled
e.save()
return render({"id": id, "status": e.enabled})
else:
restful_abort(400, error="action must be either refresh or toggle")
else: # normal crud - se if we can make this DRY
params = request.json
params['frequency'] = string_to_timedelta(params.get('frequency', '1:00:00'))
params['include_tags'] = [Tag.objects.get(name=name.strip()) for name in params['include_tags'].split(',') if name.strip()]
params['exclude_tags'] = [Tag.objects.get(name=name.strip()) for name in params['exclude_tags'].split(',') if name.strip()]
if not id:
return render(self.objectmanager(**params).save().info())
else:
self.objectmanager.objects(id=id).update(**params)
def post(self, id=None):
if not id:
return render(self.objectmanager(**request.json).save().info())
else:
obj = self.objectmanager.objects.get(id=id)
obj.clean_update(**request.json)
return render({"status": "ok"})
def last(self, id, observable_id):
try:
results = analytics.AnalyticsResults.objects(
analytics=id, observable=observable_id,
status="finished").order_by('-datetime').limit(1)
return render(self._analytics_results(results[0]))
except:
return render(None)
def post(self, id, action):
if action not in ["refresh", "toggle"]:
restful_abort(400, error="action must be either refresh or toggle")
if action == "refresh":
update_feed.delay(id)
return render({"id": id})
elif action == "toggle":
f = Feed.objects.get(id=id)
f.enabled = not f.enabled
f.save()
return render({"id": id, "status": f.enabled})
def post(self, id):
"""Create a new Tag
Edit an existing Tag according to the JSON object passed in the ``POST`` data.
If the name of a tag is changed, it will repeat the change in all Observables associated
with this tag.
:query ObjectID id: Element ID
:<json object params: JSON object containing fields to set
"""
try:
data = self._parse_request(request.json)
t = self.objectmanager.objects.get(id=id)
oldname = t.name
data['default_expiration'] = int(data['default_expiration'])
t.clean_update(**data)
# we override this so change_all_tags can be called
if data['name'] != oldname:
observables.Observable.change_all_tags(oldname, data['name'])
return render({"status": "ok"})
except TagValidationError as e:
abort(400)
except Exception as e:
import traceback
traceback.print_exc()
abort(400)
def multidelete(self):
data = loads(request.data)
ids = iterify(data['ids'])
for i, inv in enumerate(Investigation.objects(links__id__in=ids)):
inv.modify({"links__id": id}, set__links__S__id="local-{}-{}".format(time.time(), i))
self.objectmanager.objects(id__in=ids).delete()
return render({"deleted": ids})
def get(self, id):
"""Get details on a specific element
:query ObjectID id: Element ID
"""
obj = self.objectmanager.objects.get(id=id)
return render(obj, self.template_single)
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
bulk = params.pop('observables', [])
_refang = params.pop('refang', False)
for item in bulk:
value = item['value']
tags = item.get('tags', [])
if _refang:
obs = self.objectmanager.add_text(refang(value), tags)
else:
obs = self.objectmanager.add_text(value, tags)
self._modify_observable(
obs, {
'source': item.get('source'),
'context': item.get('context'),
})
added.append(obs)
return render(added)
def post(self):
q = request.get_json(silent=True)
params = q.pop("params", {})
observables = []
for o in q["observables"]:
try:
obs = Observable.guess_type(o['value'])(value=o['value'])
obs.clean()
observables.append(obs.value)
# Save observables & eventual tags to database
if params.get('save_query', False):
obs = obs.save()
obs.tag(o.get("tags", []))
obs.add_source("query")
except ObservableValidationError:
continue
# match observables with known indicators
data = match_observables([o for o in observables])
# find related observables (eg. URLs for domain, etc.)
# related_observables = [obs.get_related() for obs in observables]
# data = self.match_observables(related_observable)
#
# we need to find a way to degrade the "confidence" in
# hits obtained from related observables
return render(data, "analysis.html")
def post(self):
query = request.get_json(silent=True) or {}
fltr = query.get('filter', {})
params = query.get('params', {})
regex = params.pop('regex', False)
if regex:
fltr = {key: re.compile(value) for key, value in fltr.items()}
page = params.pop('page', 1) - 1
rng = params.pop('range', 50)
print "Filter:", fltr
for key, value in fltr.copy().items():
if key == 'tags':
if not regex:
fltr['tags__name__in'] = fltr.pop('tags').split(',')
else:
fltr['tags__name'] = fltr.pop('tags')
try:
data = []
for o in self.objectmanager.objects(**fltr)[page * rng:(page + 1) * rng]:
info = o.info()
info['uri'] = url_for("api.{}".format(self.__class__.__name__.lower()), id=str(o.id))
data.append(info)
except InvalidQueryError as e:
restful_abort(400, invalid_query=str(e))
return render(data, self.template)
def match(self):
"""Match observables against Yeti's intelligence repository.
Takes an array of observables, expands them and tries to match them against specific indicators or known observables.
To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
the "expanded" observable array will also include the hostname ``google.com``.
:<json [string] observables: An array of observables to be analyzed
:>json [Entity] entities: Related ``Entity`` objects
:>json [Observable] known: ``Observable`` objects that are already present in database
:>json [Indicator] matches: ``Indicators`` that matched observables
:>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
:>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
"""
params = request.json
observables = params.pop('observables', [])
fetch_neighbors = params.pop('fetch_neighbors', True)
add_unknown = bool(params.pop('add_unknown', False))
if add_unknown and current_user.has_permission('observable', 'write'):
for o in observables:
Observable.add_text(o)
data = match_observables(observables, save_matches=add_unknown and current_user.has_permission('observable', 'write'), fetch_neighbors=fetch_neighbors)
return render(data)
def merge(self):
"""Merge one or more tags
Merge one or more tags into a single tag. This is useful for
replacing one or several tags with other tags.
:<json [String] merge: Array of Strings (tag names) representing tags to be merged.
:<json String merge_into: The tag to merge into
:<json boolean make_dict: Create a Tag dictionary out of this merge.
In the future, tags included in the ``merge``
object will be automatically
replaced by the tag specified in ``merge_into``.
"""
tags = request.json['merge']
merge_into = self.objectmanager.objects.get(name=request.json['merge_into'])
make_dict = request.json['make_dict']
merged = 0
observables.Observable.change_all_tags(tags, merge_into.name)
for tag in tags:
oldtag = self.objectmanager.objects.get(name=tag)
merge_into.count += oldtag.count
merge_into.produces += [i for i in oldtag.produces if i not in merge_into.produces and i != merge_into]
merge_into.save()
oldtag.delete()
merged += 1
if make_dict:
merge_into.add_replaces(tags)
return render({"merged": merged, "into": merge_into.name})
def content(self, id):
"""Return export content
Returns a given export's content.
:query ObjectID id: Export ID
:resheader X-Yeti-Export-MD5: The MD5 hash of the exported content. Use it to check the export's integrity
"""
try:
e = self.objectmanager.objects.get(id=id)
except DoesNotExist:
return render({
"error": "No Export found for id {}".format(id)
}), 404
if e.output_dir.startswith("/"):
d = e.output_dir
else:
d = os.path.join(
os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.abspath(__file__))))),
e.output_dir)
response = make_response(
send_from_directory(
d, e.name, as_attachment=True, attachment_filename=e.name))
response.headers['X-Yeti-Export-MD5'] = e.hash_md5
return response
def refresh(self, id):
"""Runs a Scheduled Analytics
:query ObjectID id: Scheduled Analytics ObjectID
:>json ObjectID id: ID of refreshed Scheduled Analytics
"""
schedule.delay(id)
return render({"id": id})
def delete(self, id):
"""Deletes the corresponding entry from the database
:query ObjectID id: Element ID
:>json string deleted: The deleted element's ObjectID
"""
obj = self.objectmanager.objects.get(id=id)
obj.delete()
return render({"deleted": id})
def refresh(self, id):
"""Runs a Feed
:query ObjectID id: Feed ID
:>json ObjectId id: Feed ID
"""
feed.update_feed.delay(id)
return render({"id": id})
def multidelete(self):
"""Deletes multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:>json [ObjectID] deleted: Array of Element IDs that were successfully deleted
"""
data = loads(request.data)
ids = iterify(data['ids'])
self.objectmanager.objects(id__in=ids).delete()
return render({"deleted": ids})
def remove_context(self, id):
"""Removes context from an observable
:<json object context: Context JSON to be added. Must include a ``source`` key.
:>json object: The context object that was actually delete
"""
observable = get_object_or_404(self.objectmanager, id=id)
context = request.json.pop('context', {})
observable.remove_context(context)
return render(context)
def new(self):
"""Create a new element
Create a new element from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing fields to set
"""
params = self._parse_request(request.json)
obj = self.objectmanager(**params).save()
return render(obj)
def refresh(self, id):
"""Refresh an export
Manually executes an export if it is not already exporting.
:query ObjectID id: Export ID
:>json ObjectID id: The export's ObjectID
"""
exports.execute_export.delay(id)
return render({"id": id})
def delete(self, id):
"""Deletes the corresponding entry from the database
:query ObjectID id: Element ID
:>json string deleted: The deleted element's ObjectID
"""
obj = self.objectmanager.objects.get(id=id)
for i, inv in enumerate(Investigation.objects(links__id=id)):
inv.modify({"links__id": id}, set__links__S__id="local-{}-{}".format(time.time(), i))
obj.delete()
return render({"deleted": id})
def post(self):
if 'file' in request.files:
f = AttachedFile.from_upload(request.files['file'])
else:
data = loads(request.data)
if 'file' in data:
f = AttachedFile.from_upload(data['file'])
else:
abort(400)
return render({'filename': url_for('api.AttachedFiles:get', id=f.id)})
def post(self, id=None):
if not id:
data = request.json
data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()]
data['replaces'] = request.json['replaces'].split(',')
return render(Tag(**data).save().info())
else:
try:
data = request.json
data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()]
data['replaces'] = request.json['replaces'].split(',')
t = Tag.objects.get(id=id)
t.update(**data)
Observable.change_all_tags(t.name, data['name'])
return render({"status": "ok"})
except TagValidationError as e:
restful_abort(400, error=str(e))
except Exception as e:
import traceback
traceback.print_exc()
restful_abort(400, error='Must specify name and produces parameters')
def multiupdate(self):
"""Updates multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:query [Object] new: JSON object representing fields to update
:>json [ObjectID] updated: Array of Element IDs that were successfully updated
"""
data = loads(request.data)
ids = iterify(data['ids'])
new_data = data['new']
self.objectmanager.objects(id__in=ids).update(new_data)
return render({"updated": list(self.objectmanager.objects(ids__in=ids))})
def delete(self, id):
"""Deletes a Tag
Also remove the tag from any tagged elements.
:query ObjectID id: Element ID
:>json string deleted: The deleted element's ObjectID
"""
tag = self.objectmanager.objects.get(id=id)
tag.delete()
observables.Observable.objects(tags__name=tag.name).update(pull__tags__name=tag.name)
return render({"deleted": id})
def post(self, id):
"""Modify an element
Edit an existing element according to the JSON object passed in the ``POST`` data.
:query ObjectID id: Element ID
:<json object params: JSON object containing fields to set
"""
obj = self.objectmanager.objects.get(id=id)
params = self._parse_request(request.json)
obj = obj.clean_update(**params)
return render(obj)
def match(self):
params = request.json
observables = params.pop('observables', [])
add_unknown = bool(params.pop('add_unknown', False))
if add_unknown:
for o in observables:
Observable.add_text(o)
data = match_observables(observables, save_matches=add_unknown)
return render(data)
def context(self, id):
"""Add context to an observable
:<json object context: Context JSON to be added. Must include a ``source`` key.
:<json string old_source: String defining the source to be replaced.
:>json object: The context object that was actually added
"""
observable = get_object_or_404(self.objectmanager, id=id)
context = request.json.pop('context', {})
old_source = request.json.pop('old_source', None)
observable.add_context(context, replace_source=old_source)
return render(context)
def nodesearch(self, query):
result = []
query = re.compile("^{}".format(query), re.IGNORECASE)
observables = Observable.objects(value=query).limit(5)
entities = Entity.objects(name=query).limit(5)
for results in [observables, entities]:
for node in results:
result.append(node.to_mongo())
return render(result)
def list_files(self, id):
"""List files attached to an element
:query ObjectID id: Element ID
:<json object files: JSON object containing a list of serialized AttachedFile objects
"""
l = []
entity = get_object_or_404(self.objectmanager, id=id)
for f in entity.attached_files:
i = f.info()
i['content_uri'] = url_for("api.Entity:file_content", sha256=f.sha256)
l.append(i)
return render(l)
def nodesearch(self, query):
result = []
query = re.compile("^{}".format(query), re.IGNORECASE)
observables = Observable.objects(value=query).limit(5)
entities = Entity.objects(name=query).limit(5)
for results in [observables, entities]:
for node in results:
result.append(node.to_mongo())
return render(result)
def multiupdate(self):
"""Updates multiple entries from the database
:query [ObjectID] ids: Array of Element IDs
:query [Object] new: JSON object representing fields to update
:>json [ObjectID] updated: Array of Element IDs that were successfully updated
"""
data = loads(request.data)
ids = iterify(data["ids"])
new_data = data["new"]
self.objectmanager.objects(id__in=ids).update(new_data)
return render(
{"updated": list(self.objectmanager.objects(ids__in=ids))})
def toggle(self, id):
"""Toggle an export
Toggles an export. A deactivated export will not execute when called (manually or scheduled)
:query ObjectID id: Export ID
:>json ObjectID id: The export's ObjectID
:>json boolean status: The result of the toggle operation (``true`` means the export has been enabled, ``false`` means it has been disabled)
"""
e = self.objectmanager.objects.get(id=id)
e.enabled = not e.enabled
e.save()
return render({"id": id, "status": e.enabled})
def toggle(self, id):
"""Toggles a Feed
Feeds can be individually disabled using this endpoint.
:query ObjectID id: Analytics ID
:>json ObjectID id: The Analytics's ObjectID
:>json boolean status: The result of the toggle operation (``true`` means the export has been enabled, ``false`` means it has been disabled)
"""
f = self.objectmanager.objects.get(id=id)
f.enabled = not f.enabled
f.save()
return render({"id": id, "status": f.enabled})
def delete(self, id):
"""Deletes a Tag
Also remove the tag from any tagged elements.
:query ObjectID id: Element ID
:>json string deleted: The deleted element's ObjectID
"""
tag = self.objectmanager.objects.get(id=id)
tag.delete()
observables.Observable.objects(tags__name=tag.name).update(
pull__tags__name=tag.name)
return render({"deleted": id})
def toggle(self, id):
"""Toggle an export
Toggles an export. A deactivated export will not execute when called (manually or scheduled)
:query ObjectID id: Export ID
:>json ObjectID id: The export's ObjectID
:>json boolean status: The result of the toggle operation (``true`` means the export has been enabled, ``false`` means it has been disabled)
"""
e = self.objectmanager.objects.get(id=id)
e.enabled = not e.enabled
e.save()
return render({"id": id, "status": e.enabled})
def tuples(self, klass, node_id, type_filter):
query = request.get_json(silent=True) or {}
fltr = query.get("filter", {})
params = query.get("params", {})
klass = NODES_CLASSES[klass.lower().split('.')[0]]
filter_class = NODES_CLASSES[type_filter.lower().split('.')[0]]
node = klass.objects.get(id=node_id)
regex = bool(params.pop('regex', False))
ignorecase = bool(params.pop('ignorecase', False))
page = int(params.pop("page", 1)) - 1
rng = int(params.pop("range", 50))
print "[{}] Filter: {}".format(self.__class__.__name__, fltr)
print filter_class, fltr, regex, ignorecase, page, rng
neighbors = node.neighbors_advanced(filter_class, fltr, regex,
ignorecase, page, rng)
_all = []
links = []
objs = []
for link, obj in neighbors:
links.append(link)
objs.append(obj)
_all.append((link, obj))
data = {"data": objs, "links": links}
# First argument of render is the "data" variable in the template.
# We override this behavior for these templates to include links
# using the ctx argument
if issubclass(filter_class, Entity):
return render(data, template='entity_api.html', ctx=data)
if issubclass(filter_class, Indicator):
return render(data, template='indicator_api.html', ctx=data)
if issubclass(filter_class, Observable):
return render(data, template='observable_api.html', ctx=data)
def toggle(self, id):
"""Toggles a One-shot Analytics
One-Shot Analytics can be individually disabled using this endpoint.
:query ObjectID id: Analytics ID
:>json ObjectID id: The Analytics's ObjectID
:>json boolean status: The result of the toggle operation (``true`` means the export has been enabled, ``false`` means it has been disabled)
"""
analytics = get_object_or_404(self.objectmanager, id=id)
analytics.enabled = not analytics.enabled
analytics.save()
return render({"id": analytics.id, "status": analytics.enabled})
def new(self):
"""Create a new Observable
Create a new Observable from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing fields to set
:<json boolean refang: If set, the observable will be refanged before being added to the database
"""
params = request.json
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(params.pop('value')))
else:
obs = self.objectmanager.add_text(params.pop('value'))
return render(self._modify_observable(obs, params))
def delete(self, id):
"""Deletes the corresponding entry from the database
:query ObjectID id: Element ID
:>json string deleted: The deleted element's ObjectID
"""
obj = self.objectmanager.objects.get(id=id)
for i, inv in enumerate(Investigation.objects(links__id=id)):
inv.modify(
{"links__id": id},
set__links__S__id="local-{}-{}".format(time.time(), i),
)
obj.delete()
return render({"deleted": id})
def index(self):
data = []
for obj in self.objectmanager.objects.all():
info = obj.info()
info['available'] = True
if hasattr(obj, 'settings') and not current_user.has_settings(
obj.settings):
info['available'] = False
data.append(info)
return render(data, template=self.template)
def add_file(self):
"""Adds a new File
Create a new File from the form passed in the ``POST`` data. Each file
should be passed in the ``files`` parameter. Multiple files can be
added in one request.
The file body will be stored as an AttachedFile object.
:<file form parameter: Field containing file(s) to store
:<unzip form parameter ([true|false]): Uncompress archive and add files
separately
"""
files = save_uploaded_files()
return render(files)
def tuples(self, klass, node_id, type_filter):
query = request.get_json(silent=True) or {}
fltr = query.get("filter", {})
params = query.get("params", {})
klass = NODES_CLASSES[klass.lower().split(".")[0]]
filter_class = NODES_CLASSES[type_filter.lower().split(".")[0]]
node = klass.objects.get(id=node_id)
regex = bool(params.pop("regex", False))
ignorecase = bool(params.pop("ignorecase", False))
page = int(params.pop("page", 1)) - 1
rng = int(params.pop("range", 50))
print("[{}] Filter: {}".format(self.__class__.__name__, fltr))
print(filter_class, fltr, regex, ignorecase, page, rng)
neighbors = node.neighbors_advanced(
filter_class, fltr, regex, ignorecase, page, rng
)
_all = []
links = []
objs = []
for link, obj in neighbors:
links.append(link)
objs.append(obj)
_all.append((link, obj))
data = {"objs": objs, "links": links}
if issubclass(filter_class, Entity):
return render(data, template="entity_api.html")
if issubclass(filter_class, Indicator):
return render(data, template="indicator_api.html")
if issubclass(filter_class, Observable):
return render(data, template="observable_api.html")
def multiupdate(self):
data = loads(request.data)
ids = data['ids']
new_description = data['new']['description']
updated = []
for link in self.objectmanager.objects(id__in=ids):
# link.select_related() #does not work
# must call src and dst to dereference dbrefs and not raise an exception
link.src
link.dst
link.description = new_description
link.save()
updated.append(link.id)
return render({"updated": updated})
def toggle(self, id):
"""Toggles a Inline Analytics
Inline Analytics can be individually disabled using this endpoint.
:query ObjectID id: Analytics ID
:>json ObjectID id: The Analytics's ObjectID
:>json boolean status: The result of the toggle operation (``true`` means the export has been enabled, ``false`` means it has been disabled)
"""
a = self.objectmanager.objects.get(id=id)
a.enabled = not a.enabled
a.save()
analytics.InlineAnalytics.analytics[a.name] = a
return render({"id": id, "status": a.enabled})
def run(self, id):
"""Runs a One-Shot Analytics
Asynchronously runs a One-Shot Analytics against a given observable.
Returns an ``AnalyticsResults`` instance, which can then be used to fetch
the analytics results
:query ObjectID id: Analytics ID
:form ObjectID id: Observable ID
:>json object: JSON object representing the ``AnalyticsResults`` instance
"""
analytics = get_object_or_404(self.objectmanager, id=id)
observable = get_object_or_404(Observable, id=request.form.get('id'))
return render(
analytics.run(observable, current_user.settings).to_mongo())
def new(self):
"""Create a new element
Create a new element from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing fields to set
"""
params = self._parse_request(request.json)
objectmanager = self.objectmanager
if 'type' in params and hasattr(self, 'subobjects'):
objectmanager = self.subobjects.get(params['type'])
if objectmanager is None:
abort(400)
params.pop('type', None)
obj = objectmanager(**params).save()
return render(obj)
def new(self):
"""Create a new link
Create a new link from the JSON object passed in the ``POST`` data.
:<json object params: JSON object containing object ids to link
"""
type_map = {
"observable": observables.Observable,
"entity": entities.Entity,
"indicator": indicators.Indicator,
}
mandatory_params = ["type_src", "type_dst", "link_src", "link_dst"]
params = request.json
if not all(key in params for key in mandatory_params):
abort(400)
type_src = params["type_src"]
type_dst = params["type_dst"]
src_object_class = type_map.get(type_src)
dst_object_class = type_map.get(type_dst)
if not src_object_class or not dst_object_class:
abort(404)
src = get_object_or_404(src_object_class, id=params["link_src"])
dst = get_object_or_404(dst_object_class, id=params["link_dst"])
try:
if params.get("first_seen") and params.get("last_seen"):
link = src.link_to(
dst,
params.get("description"),
params.get("source"),
params["first_seen"],
params["last_seen"],
)
else:
link = src.active_link_to(dst, params.get("description"),
params.get("source"))
except Exception as e:
logging.error(e)
abort(400)
return render({"link": link})
def search_existence(self):
"""Query investigation based on given observable, incident or entity
Query[ref]: class of the given node, which should be observable or entity
Query[id]: the id of the given node.
"""
REF_CLASS = ('observable', 'entity')
data = loads(request.data)
if 'id' not in data or 'ref' not in data:
response = {
'status': 'error',
'message': 'missing argument.'
}
elif not ObjectId.is_valid(data['id']):
response = {
'status': 'error',
'message': 'given id is not valid.'
}
elif data['ref'] not in REF_CLASS:
response = {
'status': 'error',
'message': 'reference class is not valid.'
}
else:
query = {
'nodes': {
'$elemMatch': {
'$id': ObjectId(data['id']),
'$ref': data['ref']
}
},
}
response = self.objectmanager.objects(__raw__=query).order_by('-updated')
for inv in response:
if not inv.name:
inv['name'] = 'Unnamed Investigation'
return render(response)
def get(self, klass, node_id):
klass = NODES_CLASSES[klass.lower().split('.')[0]]
node = klass.objects.get(id=node_id)
result = {'links': list(), 'nodes': list()}
result['nodes'].append(node.to_mongo())
node_ids = set()
links = list(set(list(node.incoming()) + list(node.outgoing())))
for link, node in links:
if node.id not in node_ids:
node_ids.add(node)
result['nodes'].append(node.to_mongo())
result['links'].append(link.to_dict())
return render(result)
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [String] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
observables = params.pop('observables', [])
for item in observables:
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(item))
else:
obs = self.objectmanager.add_text(item)
added.append(self._modify_observable(obs, params.copy()))
return render(added)
def content(self, id):
"""Return export content
Returns a given export's content.
:query ObjectID id: Export ID
:resheader X-Yeti-Export-MD5: The MD5 hash of the exported content. Use it to check the export's integrity
"""
try:
e = self.objectmanager.objects.get(id=id)
except DoesNotExist:
return render({"error": "No Export found for id {}".format(id)}), 404
if e.output_dir.startswith("/"):
d = e.output_dir
else:
d = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), e.output_dir)
response = make_response(send_from_directory(d, e.name, as_attachment=True, attachment_filename=e.name))
response.headers['X-Yeti-Export-MD5'] = e.hash_md5
return response
def bulk(self):
"""Bulk-add observables
Bulk-add Observables from an array of strings.
:<json [{string: observable, tags: [string]}] observables: Array of Strings representing observables (URLs, IPs, hostnames, etc.)
:<json boolean refang: If set, the observables will be refanged before being added to the database
"""
added = []
params = request.json
observables = params.pop('observables', [])
for item in observables:
obs = item['value']
tags = item['tags']
if params.pop('refang', None):
obs = self.objectmanager.add_text(refang(obs), tags)
else:
obs = self.objectmanager.add_text(obs, tags)
added.append(obs)
return render(added)
def search_existence(self):
"""Query investigation based on given observable, incident or entity
Query[ref]: class of the given node, which should be observable or entity
Query[id]: the id of the given node.
"""
# ToDo sharing permissions
REF_CLASS = ("observable", "entity")
data = loads(request.data)
if "id" not in data or "ref" not in data:
response = {"status": "error", "message": "missing argument."}
elif not ObjectId.is_valid(data["id"]):
response = {"status": "error", "message": "given id is not valid."}
elif data["ref"] not in REF_CLASS:
response = {
"status": "error",
"message": "reference class is not valid."
}
else:
query = {
"nodes": {
"$elemMatch": {
"$id": ObjectId(data["id"]),
"$ref": data["ref"]
}
},
}
response = self.objectmanager.objects(
__raw__=query).order_by("-updated")
for inv in response:
if not inv.name:
inv["name"] = "Unnamed Investigation"
return render(response)
def post(self):
"""Launches a simple search against the database
This endpoint is mostly used by paginators in Yeti.
:<json object params: JSON object specifying the ``page``, ``range`` and ``regex`` variables.
:<json integer params.page: Page or results to return (default: 1)
:<json integer params.range: How many results to return (default: 50)
:<json boolean params.regex: Set to true if the arrays in ``filter`` are to be treated as regular expressions (default: false)
:<json object filter: JSON object specifying keys to be matched in the database. Each key must contain an array of OR-matched values.
:reqheader Accept: must be set to ``application/json``
:reqheader Content-Type: must be set to ``application/json``
"""
query = request.get_json(silent=True) or {}
try:
data = self.search(query)
except InvalidQueryError as e:
logging.error(e)
abort(400)
return render(data, self.template)
def merge(self):
"""Merge one or more tags
Merge one or more tags into a single tag. This is useful for
replacing one or several tags with other tags.
:<json [String] merge: Array of Strings (tag names) representing tags to be merged.
:<json String merge_into: The tag to merge into
:<json boolean make_dict: Create a Tag dictionary out of this merge.
In the future, tags included in the ``merge``
object will be automatically
replaced by the tag specified in ``merge_into``.
"""
tags = request.json['merge']
merge_into = self.objectmanager.objects.get(
name=request.json['merge_into'])
make_dict = request.json['make_dict']
merged = 0
observables.Observable.change_all_tags(tags, merge_into.name)
for tag in tags:
oldtag = self.objectmanager.objects.get(name=tag)
merge_into.count += oldtag.count
merge_into.produces += [
i for i in oldtag.produces
if i not in merge_into.produces and i != merge_into
]
merge_into.save()
oldtag.delete()
merged += 1
if make_dict:
merge_into.add_replaces(tags)
return render({"merged": merged, "into": merge_into.name})
def post(self, id):
obs = self.objectmanager.objects.get(id=id)
j = request.json
if not current_user.has_permission("observable", "tag") and "tags" in j:
abort(401)
return render(self._modify_observable(obs, request.json))
def index(self):
"""List all corresponding entries in the database. **Do not use on large datasets!**
"""
objects = [o.info() for o in self.objectmanager.objects.all()]
return render(objects, template=self.template)
def rename(self, id):
i = get_object_or_404(self.objectmanager, id=id)
i.modify(name=request.json['name'], updated=datetime.utcnow())
return render("ok")
def remove(self, id):
i = get_object_or_404(self.objectmanager, id=id)
data = loads(request.data)
i.remove(iterify(data['links']), iterify(data['nodes']))
return render(i.info())