def tasks_delete(task_id):
response = {}
task = db.view_task(task_id)
if task:
if task.status == TASK_RUNNING:
return HTTPError(500, "The task is currently being "
"processed, cannot delete")
if db.delete_task(task_id):
delete_folder(os.path.join(CUCKOO_ROOT, "storage",
"analyses", "%d" % task_id))
if FULL_DB:
task = results_db.analysis.find_one({"info.id": task_id})
for processes in task.get("behavior", {}).get("processes", []):
[results_db.calls.remove(call) for call in processes.get("calls", [])]
results_db.analysis.remove({"info.id": task_id})
response["status"] = "OK"
else:
return HTTPError(500, "An error occurred while trying to "
"delete the task")
else:
return HTTPError(404, "Task not found")
return jsonize(response)
def tasks_delete(request, task_id):
if request.method != "GET":
resp = {"error": True, "error_value": "Method not allowed"}
return jsonize(resp, response=True)
if not apiconf.taskdelete.get("enabled"):
resp = {"error": True,
"error_value": "Task Deletion API is Disabled"}
return jsonize(resp, response=True)
check = validate_task(task_id)
if check["error"]:
return jsonize(check, response=True)
resp = {}
if db.delete_task(task_id):
resp["error"] = False
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % task_id))
resp["data"] = "Task ID {0} has been deleted".format(task_id)
else:
resp = {"error": True,
"error_value": ("An error occured when trying to delete "
"task {0}".format(task_id))}
return jsonize(resp, response=True)
def cuckoo_clean_failed_url_tasks():
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for failed tasks.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
try:
results_db = MongoClient(host, port)[mdb]
except:
log.warning("Unable to connect MongoDB database: %s", mdb)
return
done = False
while not done:
rtmp = results_db.analysis.find({"info.category": "url", "network.http.0": {"$exists": False}},{"info.id": 1},sort=[("_id", -1)]).limit( 100 )
if rtmp and rtmp.count() > 0:
for e in rtmp:
if e["info"]["id"]:
print e["info"]["id"]
try:
results_db.suricata.remove({"info.id": int(e["info"]["id"])})
except:
print "failed to remove %s" % (e["info"]["id"])
if db.delete_task(e["info"]["id"]):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % e["info"]["id"]))
else:
print "failed to remove %s" % (e["info"]["id"])
try:
results_db.analysis.remove({"info.id": int(e["info"]["id"])})
except:
print "failed to remove %s" % (e["info"]["id"])
else:
done = True
else:
done = True
def cuckoo_clean_before_day(days=None):
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for tasks completed before now - days.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
if not days:
print "No days argument provided bailing"
return
create_structure()
init_console_logging()
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
try:
results_db = MongoClient(host, port)[mdb]
except:
log.warning("Unable to connect to MongoDB database: %s", mdb)
return
added_before = datetime.datetime.now() - datetime.timedelta(days=int(days))
old_tasks = db.list_tasks(added_before=added_before)
for e in old_tasks:
new = e.to_dict()
print int(new["id"])
try:
results_db.suricata.remove({"info.id": int(new["id"])})
except:
print "failed to remove suricata info (may not exist) %s" % (int(new["id"]))
try:
results_db.analysis.remove({"info.id": int(new["id"])})
except:
print "failed to remove analysis info (may not exist) %s" % (int(new["id"]))
if db.delete_task(new["id"]):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % int(new["id"])))
else:
print "failed to remove faile task %s from DB" % (int(new["id"]))
def cuckoo_clean_failed_tasks():
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for failed tasks.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
try:
results_db = MongoClient(host, port)[mdb]
except:
log.warning("Unable to connect to MongoDB database: %s", mdb)
return
failed_tasks_a = db.list_tasks(status=TASK_FAILED_ANALYSIS)
failed_tasks_p = db.list_tasks(status=TASK_FAILED_PROCESSING)
failed_tasks_r = db.list_tasks(status=TASK_FAILED_REPORTING)
for e in failed_tasks_a,failed_tasks_p,failed_tasks_r:
for el2 in e:
new = el2.to_dict()
print int(new["id"])
try:
results_db.suricata.remove({"info.id": int(new["id"])})
except:
print "failed to remove suricata info (may not exist) %s" % (int(new["id"]))
try:
results_db.analysis.remove({"info.id": int(new["id"])})
except:
print "failed to remove analysis info (may not exist) %s" % (int(new["id"]))
if db.delete_task(new["id"]):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % int(new["id"])))
else:
print "failed to remove faile task %s from DB" % (int(new["id"]))
def tasks_delete(task_id):
response = {}
task = db.view_task(task_id)
if task:
if task.status == TASK_RUNNING:
return HTTPError(500, "The task is currently being " "processed, cannot delete")
if db.delete_task(task_id):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id))
response["status"] = "OK"
else:
return HTTPError(500, "An error occurred while trying to " "delete the task")
else:
return HTTPError(404, "Task not found")
return jsonize(response)
def tasks_delete(task_id):
response = {}
task = db.view_task(task_id)
if task:
if task.status == TASK_RUNNING:
return json_error(
500, "The task is currently being "
"processed, cannot delete")
if db.delete_task(task_id):
delete_folder(
os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%d" % task_id))
response["status"] = "OK"
else:
return json_error(
500, "An error occurred while trying to "
"delete the task")
else:
return json_error(404, "Task not found")
return jsonify(response)
def tasks_delete(task_id):
response = {}
task = db.view_task(task_id)
if task:
if task.status == TASK_RUNNING:
return HTTPError(500, "The task is currently being " "processed, cannot delete")
if db.delete_task(task_id):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id))
if FULL_DB:
task = results_db.analysis.find_one({"info.id": task_id})
if task is not None:
for processes in task.get("behavior", {}).get("processes", []):
[results_db.calls.remove(call) for call in processes.get("calls", [])]
results_db.analysis.remove({"info.id": task_id})
response["status"] = "OK"
else:
return HTTPError(500, "An error occurred while trying to " "delete the task")
else:
return HTTPError(404, "Task not found")
return jsonize(response)
def tasks_delete(task_id):
response = {}
task = db.view_task(task_id)
if task:
if task.status == TASK_RUNNING:
return HTTPError(500, "The task is currently being "
"processed, cannot delete")
if db.delete_task(task_id):
delete_folder(os.path.join(CUCKOO_ROOT, "storage",
"analyses", "%d" % task_id))
if mongo_report:
collection.delete_one({'info.id': task_id})
# TODO: Delete the elements of a report not in cuckoo.analysis.
response["status"] = "OK"
db.unlock_machine(task.machine)
else:
return HTTPError(500, "An error occurred while trying to "
"delete the task")
else:
return HTTPError(404, "Task not found")
return jsonize(response)
def delete_data(tid):
if isinstance(tid, dict):
if "info.id" in tid:
tid = tid["info.id"]
elif tid.get("info", {}).get("id", 0):
tid = tid["info"]["id"]
elif "id" in tid:
tid = tid["id"]
try:
log.info("removing %s from analysis db" % (tid))
if repconf.mongodb.enabled:
mongo_delete_data(tid)
elif repconf.elasticsearchdb.enabled:
delete_analysis_and_related_calls(tid)
except Exception as e:
log.error(
"failed to remove analysis info (may not exist) %s due to %s" %
(tid, e),
exc_info=True)
if db.delete_task(tid):
delete_folder(
os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % tid))
else:
log.info("failed to remove faile task %s from DB" % (tid))
def cuckoo_clean_before_day(args):
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for tasks completed before now - days.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
if not args.delete_older_than_days:
print "No days argument provided bailing"
return
else:
days = args.delete_older_than_days
create_structure()
init_console_logging()
id_arr = []
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
user = cfg.mongodb.get("username", None)
password = cfg.mongodb.get("password", None)
try:
results_db = MongoClient(cfg.mongodb.host,
port=port,
username=user,
password=password,
authSource=mdb
)[mdb]
except:
log.warning("Unable to connect to MongoDB database: %s", mdb)
return
added_before = datetime.now() - timedelta(days=int(days))
if args.files_only_filter:
print("file filter applied")
old_tasks = db.list_tasks(
added_before=added_before, category="file")
elif args.urls_only_filter:
print("url filter applied")
old_tasks = db.list_tasks(
added_before=added_before, category="url")
else:
old_tasks = db.list_tasks(added_before=added_before)
for e in old_tasks:
new = e.to_dict()
print int(new["id"])
id_arr.append({"info.id": (int(new["id"]))})
print "number of matching records %s before suri/custom filter " % len(id_arr)
if id_arr and args.suricata_zero_alert_filter:
result = list(results_db.analysis.find({"suricata.alerts.alert": {
"$exists": False}, "$or": id_arr}, {"info.id": 1}))
tmp_arr = []
for entry in result:
tmp_arr.append(entry["info"]["id"])
id_arr = tmp_arr
if id_arr and args.custom_include_filter:
result = list(results_db.analysis.find({"info.custom": {
"$regex": args.custom_include_filter}, "$or": id_arr}, {"info.id": 1}))
tmp_arr = []
for entry in result:
tmp_arr.append(entry["info"]["id"])
id_arr = tmp_arr
print "number of matching records %s" % len(id_arr)
for e in id_arr:
try:
print "removing %s from analysis db" % (e)
results_db.analysis.remove({"info.id": e})
except:
print "failed to remove analysis info (may not exist) %s" % (e)
if db.delete_task(e):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % e))
else:
print "failed to remove faile task %s from DB" % (e)
def test_delete_folder(self, rnd_tmp_folder):
folder = "/tmp/" + rnd_tmp_folder
os.mkdir(folder)
utils.delete_folder(folder)
def cuckoo_clean_before_day(args):
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for tasks completed before now - days.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
if not args.delete_older_than_days:
print "No days argument provided bailing"
return
else:
days = args.delete_older_than_days
create_structure()
init_console_logging()
id_arr = []
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
try:
results_db = MongoClient(host, port)[mdb]
except:
log.warning("Unable to connect to MongoDB database: %s", mdb)
return
added_before = datetime.now() - timedelta(days=int(days))
if args.files_only_filter:
print("file filter applied")
old_tasks = db.list_tasks(added_before=added_before,category="file")
elif args.urls_only_filter:
print("url filter applied")
old_tasks = db.list_tasks(added_before=added_before,category="url")
else:
old_tasks = db.list_tasks(added_before=added_before)
for e in old_tasks:
new = e.to_dict()
print int(new["id"])
id_arr.append({"info.id":(int(new["id"]))})
print "number of matching records %s before suri/custom filter " % len(id_arr)
if id_arr and args.suricata_zero_alert_filter:
result = list(results_db.analysis.find({"suricata.alerts.alert": {"$exists": False}, "$or": id_arr},{"info.id":1}))
tmp_arr =[]
for entry in result:
tmp_arr.append(entry["info"]["id"])
id_arr = tmp_arr
if id_arr and args.custom_include_filter:
result = list(results_db.analysis.find({"info.custom": {"$regex": args.custom_include_filter},"$or": id_arr},{"info.id":1}))
tmp_arr = []
for entry in result:
tmp_arr.append(entry["info"]["id"])
id_arr = tmp_arr
print "number of matching records %s" % len(id_arr)
for e in id_arr:
try:
print "removing %s from analysis db" % (e)
results_db.analysis.remove({"info.id": e})
except:
print "failed to remove analysis info (may not exist) %s" % (e)
if db.delete_task(e):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % e))
else:
print "failed to remove faile task %s from DB" % (e)
def cuckoo_clean_failed_url_tasks():
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for failed tasks.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
user = cfg.mongodb.get("username", None)
password = cfg.mongodb.get("password", None)
try:
results_db = MongoClient(cfg.mongodb.host,
port=port,
username=user,
password=password,
authSource=mdb)[mdb]
except:
log.warning("Unable to connect MongoDB database: %s", mdb)
return
done = False
while not done:
rtmp = results_db.analysis.find(
{
"info.category": "url",
"network.http.0": {
"$exists": False
}
}, {
"info.id": 1
},
sort=[("_id", -1)]).limit(100)
if rtmp and rtmp.count() > 0:
for e in rtmp:
if e["info"]["id"]:
print e["info"]["id"]
if db.delete_task(e["info"]["id"]):
delete_folder(
os.path.join(CUCKOO_ROOT, "storage",
"analyses",
"%s" % e["info"]["id"]))
else:
print "failed to remove %s" % (e["info"]["id"])
try:
results_db.analysis.remove(
{"info.id": int(e["info"]["id"])})
except:
print "failed to remove %s" % (e["info"]["id"])
else:
done = True
else:
done = True
def test_delete_folder_err(self, rnd_tmp_folder, mocker):
folder = "/tmp/" + rnd_tmp_folder
os.mkdir(folder)
mocker.patch("shutil.rmtree", side_effect=OSError)
with pytest.raises(CuckooOperationalError):
utils.delete_folder(folder)
def cuckoo_clean_before_day(args):
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for tasks completed before now - days.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
if not args.delete_older_than_days:
print("No days argument provided bailing")
return
else:
days = args.delete_older_than_days
create_structure()
init_console_logging()
id_arr = []
# Initialize the database connection.
db = Database()
results_db, mdb = connect_to_mongo()
if not results_db:
log.info("Can't connect to mongo")
return
added_before = datetime.now() - timedelta(days=int(days))
if args.files_only_filter:
print("file filter applied")
old_tasks = db.list_tasks(added_before=added_before,category="file")
elif args.urls_only_filter:
print("url filter applied")
old_tasks = db.list_tasks(added_before=added_before,category="url")
else:
old_tasks = db.list_tasks(added_before=added_before)
for e in old_tasks:
new = e.to_dict()
print(int(new["id"]))
id_arr.append({"info.id": (int(new["id"]))})
print("number of matching records %s before suri/custom filter " % len(id_arr))
if id_arr and args.suricata_zero_alert_filter:
result = list(results_db.analysis.find({"suricata.alerts.alert": {"$exists": False}, "$or": id_arr},{"info.id":1}))
tmp_arr = []
for entry in result:
tmp_arr.append(entry["info"]["id"])
id_arr = tmp_arr
if id_arr and args.custom_include_filter:
result = list(results_db.analysis.find({"info.custom": {"$regex": args.custom_include_filter},"$or": id_arr},{"info.id":1}))
tmp_arr = []
for entry in result:
tmp_arr.append(entry["info"]["id"])
id_arr = tmp_arr
print("number of matching records %s" % len(id_arr))
for e in id_arr:
try:
print("removing %s from analysis db" % (e))
results_db.analysis.remove({"info.id": e})
except:
print("failed to remove analysis info (may not exist) %s" % (e))
if db.delete_task(e):
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % e))
else:
print("failed to remove faile task %s from DB" % (e))
def remove_task(task_id):
#for this not to take eternity you need to create the following indexes
#mongo
#use cuckoo
#db.analysis.createIndex({"shots":1},{"background":1})
#db.analysis.createIndex({"network.pcap_id":1},{"background":1})
#db.analysis.createIndex({"network.sorted_pcap_id":1},{"background":1})
#db.analysis.createIndex({"dropped.object_id":1},{"background":1})
#db.suricata.createIndex({"files.object_id":1},{"background":1})
#db.analysis.createIndex({"info.custom":1},{"background":1})
# Initialize the database connection.
db = Database()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
from gridfs import GridFS
try:
results_db = MongoClient(host, port)[mdb]
except:
log.warning("Unable to connect to MongoDB database: %s", mdb)
return
analyses = results_db.analysis.find({"info.id": int(task_id)})
suri = results_db.suricata.find({"info.id": int(task_id)})
fs = GridFS(results_db)
print "going to delete task id %s" % (task_id)
# Checks if more analysis found with the same ID, like if process.py was run manually.
if analyses.count() > 1:
message = "Multiple tasks with this ID deleted."
elif analyses.count() == 1:
message = "Task deleted."
if analyses.count() > 0:
# Delete dups too.
for analysis in analyses:
print "deleting target"
# Delete sample if not used.
if "file_id" in analysis["target"]:
if results_db.analysis.find({"target.file_id": ObjectId(analysis["target"]["file_id"])}).count() == 1:
fs.delete(ObjectId(analysis["target"]["file_id"]))
print "deleting screenshots"
# Delete screenshots.
for shot in analysis["shots"]:
if results_db.analysis.find({"shots": ObjectId(shot)}).count() == 1:
fs.delete(ObjectId(shot))
print "deleting pcap"
# Delete network pcap.
if "pcap_id" in analysis["network"] and results_db.analysis.find({"network.pcap_id": ObjectId(analysis["network"]["pcap_id"])}).count() == 1:
fs.delete(ObjectId(analysis["network"]["pcap_id"]))
print "deleting sorted_pcap"
# Delete sorted pcap
if "sorted_pcap_id" in analysis["network"] and results_db.analysis.find({"network.sorted_pcap_id": ObjectId(analysis["network"]["sorted_pcap_id"])}).count() == 1:
fs.delete(ObjectId(analysis["network"]["sorted_pcap_id"]))
print "deleting dropped"
# Delete dropped.
for drop in analysis["dropped"]:
if "object_id" in drop and results_db.analysis.find({"dropped.object_id": ObjectId(drop["object_id"])}).count() == 1:
fs.delete(ObjectId(drop["object_id"]))
print "deleting calls"
# Delete calls.
for process in analysis.get("behavior", {}).get("processes", []):
for call in process["calls"]:
results_db.calls.remove({"_id": ObjectId(call)})
print "remove analysis data"
# Delete analysis data.
results_db.analysis.remove({"_id": ObjectId(analysis["_id"])})
# we may not have any suri entries
if suri.count() > 0:
for suricata in suri:
if "files" in suricata.keys():
print "removing suri files"
for entry in suricata["files"]:
if "object_id" in entry and results_db.suricata.find({"files.object_id": ObjectId(entry["object_id"])}).count() == 1:
fs.delete(ObjectId(entry["object_id"]))
results_db.suricata.remove({"_id": ObjectId(suricata["_id"])})
print "remove task from db"
result = db.delete_task(task_id)
if not result:
print "failed to remove from db"
print "removing file structure"
delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
"%s" % int(task_id)))