def tasks_delete(task_id): response = {} task = db.view_task(task_id) if task: if task.status == TASK_RUNNING: return HTTPError(500, "The task is currently being " "processed, cannot delete") if db.delete_task(task_id): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id)) if FULL_DB: task = results_db.analysis.find_one({"info.id": task_id}) for processes in task.get("behavior", {}).get("processes", []): [results_db.calls.remove(call) for call in processes.get("calls", [])] results_db.analysis.remove({"info.id": task_id}) response["status"] = "OK" else: return HTTPError(500, "An error occurred while trying to " "delete the task") else: return HTTPError(404, "Task not found") return jsonize(response)
def tasks_delete(request, task_id): if request.method != "GET": resp = {"error": True, "error_value": "Method not allowed"} return jsonize(resp, response=True) if not apiconf.taskdelete.get("enabled"): resp = {"error": True, "error_value": "Task Deletion API is Disabled"} return jsonize(resp, response=True) check = validate_task(task_id) if check["error"]: return jsonize(check, response=True) resp = {} if db.delete_task(task_id): resp["error"] = False delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id)) resp["data"] = "Task ID {0} has been deleted".format(task_id) else: resp = {"error": True, "error_value": ("An error occured when trying to delete " "task {0}".format(task_id))} return jsonize(resp, response=True)
def cuckoo_clean_failed_url_tasks(): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for failed tasks. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. create_structure() init_console_logging() # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") try: results_db = MongoClient(host, port)[mdb] except: log.warning("Unable to connect MongoDB database: %s", mdb) return done = False while not done: rtmp = results_db.analysis.find({"info.category": "url", "network.http.0": {"$exists": False}},{"info.id": 1},sort=[("_id", -1)]).limit( 100 ) if rtmp and rtmp.count() > 0: for e in rtmp: if e["info"]["id"]: print e["info"]["id"] try: results_db.suricata.remove({"info.id": int(e["info"]["id"])}) except: print "failed to remove %s" % (e["info"]["id"]) if db.delete_task(e["info"]["id"]): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % e["info"]["id"])) else: print "failed to remove %s" % (e["info"]["id"]) try: results_db.analysis.remove({"info.id": int(e["info"]["id"])}) except: print "failed to remove %s" % (e["info"]["id"]) else: done = True else: done = True
def cuckoo_clean_before_day(days=None): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for tasks completed before now - days. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. if not days: print "No days argument provided bailing" return create_structure() init_console_logging() # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") try: results_db = MongoClient(host, port)[mdb] except: log.warning("Unable to connect to MongoDB database: %s", mdb) return added_before = datetime.datetime.now() - datetime.timedelta(days=int(days)) old_tasks = db.list_tasks(added_before=added_before) for e in old_tasks: new = e.to_dict() print int(new["id"]) try: results_db.suricata.remove({"info.id": int(new["id"])}) except: print "failed to remove suricata info (may not exist) %s" % (int(new["id"])) try: results_db.analysis.remove({"info.id": int(new["id"])}) except: print "failed to remove analysis info (may not exist) %s" % (int(new["id"])) if db.delete_task(new["id"]): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % int(new["id"]))) else: print "failed to remove faile task %s from DB" % (int(new["id"]))
def cuckoo_clean_failed_tasks(): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for failed tasks. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. create_structure() init_console_logging() # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") try: results_db = MongoClient(host, port)[mdb] except: log.warning("Unable to connect to MongoDB database: %s", mdb) return failed_tasks_a = db.list_tasks(status=TASK_FAILED_ANALYSIS) failed_tasks_p = db.list_tasks(status=TASK_FAILED_PROCESSING) failed_tasks_r = db.list_tasks(status=TASK_FAILED_REPORTING) for e in failed_tasks_a,failed_tasks_p,failed_tasks_r: for el2 in e: new = el2.to_dict() print int(new["id"]) try: results_db.suricata.remove({"info.id": int(new["id"])}) except: print "failed to remove suricata info (may not exist) %s" % (int(new["id"])) try: results_db.analysis.remove({"info.id": int(new["id"])}) except: print "failed to remove analysis info (may not exist) %s" % (int(new["id"])) if db.delete_task(new["id"]): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % int(new["id"]))) else: print "failed to remove faile task %s from DB" % (int(new["id"]))
def tasks_delete(task_id): response = {} task = db.view_task(task_id) if task: if task.status == TASK_RUNNING: return HTTPError(500, "The task is currently being " "processed, cannot delete") if db.delete_task(task_id): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id)) response["status"] = "OK" else: return HTTPError(500, "An error occurred while trying to " "delete the task") else: return HTTPError(404, "Task not found") return jsonize(response)
def tasks_delete(task_id): response = {} task = db.view_task(task_id) if task: if task.status == TASK_RUNNING: return json_error( 500, "The task is currently being " "processed, cannot delete") if db.delete_task(task_id): delete_folder( os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id)) response["status"] = "OK" else: return json_error( 500, "An error occurred while trying to " "delete the task") else: return json_error(404, "Task not found") return jsonify(response)
def tasks_delete(task_id): response = {} task = db.view_task(task_id) if task: if task.status == TASK_RUNNING: return HTTPError(500, "The task is currently being " "processed, cannot delete") if db.delete_task(task_id): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id)) if FULL_DB: task = results_db.analysis.find_one({"info.id": task_id}) if task is not None: for processes in task.get("behavior", {}).get("processes", []): [results_db.calls.remove(call) for call in processes.get("calls", [])] results_db.analysis.remove({"info.id": task_id}) response["status"] = "OK" else: return HTTPError(500, "An error occurred while trying to " "delete the task") else: return HTTPError(404, "Task not found") return jsonize(response)
def tasks_delete(task_id): response = {} task = db.view_task(task_id) if task: if task.status == TASK_RUNNING: return HTTPError(500, "The task is currently being " "processed, cannot delete") if db.delete_task(task_id): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%d" % task_id)) if mongo_report: collection.delete_one({'info.id': task_id}) # TODO: Delete the elements of a report not in cuckoo.analysis. response["status"] = "OK" db.unlock_machine(task.machine) else: return HTTPError(500, "An error occurred while trying to " "delete the task") else: return HTTPError(404, "Task not found") return jsonize(response)
def delete_data(tid): if isinstance(tid, dict): if "info.id" in tid: tid = tid["info.id"] elif tid.get("info", {}).get("id", 0): tid = tid["info"]["id"] elif "id" in tid: tid = tid["id"] try: log.info("removing %s from analysis db" % (tid)) if repconf.mongodb.enabled: mongo_delete_data(tid) elif repconf.elasticsearchdb.enabled: delete_analysis_and_related_calls(tid) except Exception as e: log.error( "failed to remove analysis info (may not exist) %s due to %s" % (tid, e), exc_info=True) if db.delete_task(tid): delete_folder( os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % tid)) else: log.info("failed to remove faile task %s from DB" % (tid))
def cuckoo_clean_before_day(args): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for tasks completed before now - days. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. if not args.delete_older_than_days: print "No days argument provided bailing" return else: days = args.delete_older_than_days create_structure() init_console_logging() id_arr = [] # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") user = cfg.mongodb.get("username", None) password = cfg.mongodb.get("password", None) try: results_db = MongoClient(cfg.mongodb.host, port=port, username=user, password=password, authSource=mdb )[mdb] except: log.warning("Unable to connect to MongoDB database: %s", mdb) return added_before = datetime.now() - timedelta(days=int(days)) if args.files_only_filter: print("file filter applied") old_tasks = db.list_tasks( added_before=added_before, category="file") elif args.urls_only_filter: print("url filter applied") old_tasks = db.list_tasks( added_before=added_before, category="url") else: old_tasks = db.list_tasks(added_before=added_before) for e in old_tasks: new = e.to_dict() print int(new["id"]) id_arr.append({"info.id": (int(new["id"]))}) print "number of matching records %s before suri/custom filter " % len(id_arr) if id_arr and args.suricata_zero_alert_filter: result = list(results_db.analysis.find({"suricata.alerts.alert": { "$exists": False}, "$or": id_arr}, {"info.id": 1})) tmp_arr = [] for entry in result: tmp_arr.append(entry["info"]["id"]) id_arr = tmp_arr if id_arr and args.custom_include_filter: result = list(results_db.analysis.find({"info.custom": { "$regex": args.custom_include_filter}, "$or": id_arr}, {"info.id": 1})) tmp_arr = [] for entry in result: tmp_arr.append(entry["info"]["id"]) id_arr = tmp_arr print "number of matching records %s" % len(id_arr) for e in id_arr: try: print "removing %s from analysis db" % (e) results_db.analysis.remove({"info.id": e}) except: print "failed to remove analysis info (may not exist) %s" % (e) if db.delete_task(e): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % e)) else: print "failed to remove faile task %s from DB" % (e)
def test_delete_folder(self, rnd_tmp_folder): folder = "/tmp/" + rnd_tmp_folder os.mkdir(folder) utils.delete_folder(folder)
def cuckoo_clean_before_day(args): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for tasks completed before now - days. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. if not args.delete_older_than_days: print "No days argument provided bailing" return else: days = args.delete_older_than_days create_structure() init_console_logging() id_arr = [] # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") try: results_db = MongoClient(host, port)[mdb] except: log.warning("Unable to connect to MongoDB database: %s", mdb) return added_before = datetime.now() - timedelta(days=int(days)) if args.files_only_filter: print("file filter applied") old_tasks = db.list_tasks(added_before=added_before,category="file") elif args.urls_only_filter: print("url filter applied") old_tasks = db.list_tasks(added_before=added_before,category="url") else: old_tasks = db.list_tasks(added_before=added_before) for e in old_tasks: new = e.to_dict() print int(new["id"]) id_arr.append({"info.id":(int(new["id"]))}) print "number of matching records %s before suri/custom filter " % len(id_arr) if id_arr and args.suricata_zero_alert_filter: result = list(results_db.analysis.find({"suricata.alerts.alert": {"$exists": False}, "$or": id_arr},{"info.id":1})) tmp_arr =[] for entry in result: tmp_arr.append(entry["info"]["id"]) id_arr = tmp_arr if id_arr and args.custom_include_filter: result = list(results_db.analysis.find({"info.custom": {"$regex": args.custom_include_filter},"$or": id_arr},{"info.id":1})) tmp_arr = [] for entry in result: tmp_arr.append(entry["info"]["id"]) id_arr = tmp_arr print "number of matching records %s" % len(id_arr) for e in id_arr: try: print "removing %s from analysis db" % (e) results_db.analysis.remove({"info.id": e}) except: print "failed to remove analysis info (may not exist) %s" % (e) if db.delete_task(e): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % e)) else: print "failed to remove faile task %s from DB" % (e)
def cuckoo_clean_failed_url_tasks(): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for failed tasks. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. create_structure() init_console_logging() # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") user = cfg.mongodb.get("username", None) password = cfg.mongodb.get("password", None) try: results_db = MongoClient(cfg.mongodb.host, port=port, username=user, password=password, authSource=mdb)[mdb] except: log.warning("Unable to connect MongoDB database: %s", mdb) return done = False while not done: rtmp = results_db.analysis.find( { "info.category": "url", "network.http.0": { "$exists": False } }, { "info.id": 1 }, sort=[("_id", -1)]).limit(100) if rtmp and rtmp.count() > 0: for e in rtmp: if e["info"]["id"]: print e["info"]["id"] if db.delete_task(e["info"]["id"]): delete_folder( os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % e["info"]["id"])) else: print "failed to remove %s" % (e["info"]["id"]) try: results_db.analysis.remove( {"info.id": int(e["info"]["id"])}) except: print "failed to remove %s" % (e["info"]["id"]) else: done = True else: done = True
def test_delete_folder_err(self, rnd_tmp_folder, mocker): folder = "/tmp/" + rnd_tmp_folder os.mkdir(folder) mocker.patch("shutil.rmtree", side_effect=OSError) with pytest.raises(CuckooOperationalError): utils.delete_folder(folder)
def cuckoo_clean_before_day(args): """Clean up failed tasks It deletes all stored data from file system and configured databases (SQL and MongoDB for tasks completed before now - days. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. if not args.delete_older_than_days: print("No days argument provided bailing") return else: days = args.delete_older_than_days create_structure() init_console_logging() id_arr = [] # Initialize the database connection. db = Database() results_db, mdb = connect_to_mongo() if not results_db: log.info("Can't connect to mongo") return added_before = datetime.now() - timedelta(days=int(days)) if args.files_only_filter: print("file filter applied") old_tasks = db.list_tasks(added_before=added_before,category="file") elif args.urls_only_filter: print("url filter applied") old_tasks = db.list_tasks(added_before=added_before,category="url") else: old_tasks = db.list_tasks(added_before=added_before) for e in old_tasks: new = e.to_dict() print(int(new["id"])) id_arr.append({"info.id": (int(new["id"]))}) print("number of matching records %s before suri/custom filter " % len(id_arr)) if id_arr and args.suricata_zero_alert_filter: result = list(results_db.analysis.find({"suricata.alerts.alert": {"$exists": False}, "$or": id_arr},{"info.id":1})) tmp_arr = [] for entry in result: tmp_arr.append(entry["info"]["id"]) id_arr = tmp_arr if id_arr and args.custom_include_filter: result = list(results_db.analysis.find({"info.custom": {"$regex": args.custom_include_filter},"$or": id_arr},{"info.id":1})) tmp_arr = [] for entry in result: tmp_arr.append(entry["info"]["id"]) id_arr = tmp_arr print("number of matching records %s" % len(id_arr)) for e in id_arr: try: print("removing %s from analysis db" % (e)) results_db.analysis.remove({"info.id": e}) except: print("failed to remove analysis info (may not exist) %s" % (e)) if db.delete_task(e): delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % e)) else: print("failed to remove faile task %s from DB" % (e))
def remove_task(task_id): #for this not to take eternity you need to create the following indexes #mongo #use cuckoo #db.analysis.createIndex({"shots":1},{"background":1}) #db.analysis.createIndex({"network.pcap_id":1},{"background":1}) #db.analysis.createIndex({"network.sorted_pcap_id":1},{"background":1}) #db.analysis.createIndex({"dropped.object_id":1},{"background":1}) #db.suricata.createIndex({"files.object_id":1},{"background":1}) #db.analysis.createIndex({"info.custom":1},{"background":1}) # Initialize the database connection. db = Database() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") from gridfs import GridFS try: results_db = MongoClient(host, port)[mdb] except: log.warning("Unable to connect to MongoDB database: %s", mdb) return analyses = results_db.analysis.find({"info.id": int(task_id)}) suri = results_db.suricata.find({"info.id": int(task_id)}) fs = GridFS(results_db) print "going to delete task id %s" % (task_id) # Checks if more analysis found with the same ID, like if process.py was run manually. if analyses.count() > 1: message = "Multiple tasks with this ID deleted." elif analyses.count() == 1: message = "Task deleted." if analyses.count() > 0: # Delete dups too. for analysis in analyses: print "deleting target" # Delete sample if not used. if "file_id" in analysis["target"]: if results_db.analysis.find({"target.file_id": ObjectId(analysis["target"]["file_id"])}).count() == 1: fs.delete(ObjectId(analysis["target"]["file_id"])) print "deleting screenshots" # Delete screenshots. for shot in analysis["shots"]: if results_db.analysis.find({"shots": ObjectId(shot)}).count() == 1: fs.delete(ObjectId(shot)) print "deleting pcap" # Delete network pcap. if "pcap_id" in analysis["network"] and results_db.analysis.find({"network.pcap_id": ObjectId(analysis["network"]["pcap_id"])}).count() == 1: fs.delete(ObjectId(analysis["network"]["pcap_id"])) print "deleting sorted_pcap" # Delete sorted pcap if "sorted_pcap_id" in analysis["network"] and results_db.analysis.find({"network.sorted_pcap_id": ObjectId(analysis["network"]["sorted_pcap_id"])}).count() == 1: fs.delete(ObjectId(analysis["network"]["sorted_pcap_id"])) print "deleting dropped" # Delete dropped. for drop in analysis["dropped"]: if "object_id" in drop and results_db.analysis.find({"dropped.object_id": ObjectId(drop["object_id"])}).count() == 1: fs.delete(ObjectId(drop["object_id"])) print "deleting calls" # Delete calls. for process in analysis.get("behavior", {}).get("processes", []): for call in process["calls"]: results_db.calls.remove({"_id": ObjectId(call)}) print "remove analysis data" # Delete analysis data. results_db.analysis.remove({"_id": ObjectId(analysis["_id"])}) # we may not have any suri entries if suri.count() > 0: for suricata in suri: if "files" in suricata.keys(): print "removing suri files" for entry in suricata["files"]: if "object_id" in entry and results_db.suricata.find({"files.object_id": ObjectId(entry["object_id"])}).count() == 1: fs.delete(ObjectId(entry["object_id"])) results_db.suricata.remove({"_id": ObjectId(suricata["_id"])}) print "remove task from db" result = db.delete_task(task_id) if not result: print "failed to remove from db" print "removing file structure" delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % int(task_id)))