def cuckoo_clean_lower_score(args):
"""Clean up tasks with score <= X
It deletes all stored data from file system and configured databases (SQL
and MongoDB for tasks.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
if not args.malscore:
log.info("No malscore argument provided bailing")
return
create_structure()
init_console_logging()
id_arr = []
if not is_reporting_db_connected():
return
if repconf.mongodb and repconf.mongodb.enabled:
results_db = connect_to_mongo()[mdb]
result = list(results_db.analysis.find({"malscore": {"$lte": args.malscore}}))
id_arr = [entry["info"]["id"] for entry in result]
elif repconf.elasticsearchdb.enabled:
id_arr = [d["_source"]["info"]["id"] for d in all_docs(
index=get_analysis_index(), query={
"query": {
"range": {
"malscore": {
"lte": args.malscore
}
}
}
}, _source=["info.id"])]
log.info(("number of matching records %s" % len(id_arr)))
resolver_pool.map(lambda tid: delete_data(tid), id_arr)
def cuckoo_clean():
"""Clean up cuckoo setup.
It deletes logs, all stored data from file system and configured databases (SQL
and MongoDB.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
# Drop all tables.
db.drop()
if repconf.mongodb.enabled:
mongo_drop_database(mdb)
elif repconf.elasticsearchdb.enabled and not repconf.elasticsearchdb.searchonly:
analyses = all_docs(index=get_analysis_index(),
query={"query": {
"match_all": {}
}},
_source=["info.id"])
if analyses:
for analysis in analyses:
delete_analysis_and_related_calls(
analysis["_source"]["info"]["id"])
# Paths to clean.
paths = [
os.path.join(CUCKOO_ROOT, "db"),
os.path.join(CUCKOO_ROOT, "log"),
os.path.join(CUCKOO_ROOT, "storage"),
]
# Delete various directories.
for path in paths:
if os.path.isdir(path):
try:
shutil.rmtree(path)
except (IOError, OSError) as e:
log.warning("Error removing directory %s: %s", path, e)
# Delete all compiled Python objects ("*.pyc").
for dirpath, dirnames, filenames in os.walk(CUCKOO_ROOT):
for fname in filenames:
if not fname.endswith(".pyc"):
continue
path = os.path.join(CUCKOO_ROOT, dirpath, fname)
try:
os.unlink(path)
except (IOError, OSError) as e:
log.warning("Error removing file %s: %s", path, e)
def cuckoo_clean_failed_url_tasks():
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for failed tasks.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
if not is_reporting_db_connected():
return
if repconf.mongodb.enabled:
rtmp = mongo_find("analysis", {
"info.category": "url",
"network.http.0": {
"$exists": False
}
}, {
"info.id": 1
},
sort=[("_id", -1)]).limit(100)
elif repconf.elasticsearchdb.enabled:
rtmp = [
d["_source"] for d in all_docs(
index=get_analysis_index(),
query={
"query": {
"bool": {
"must": [{
"exists": {
"field": "network.http"
}
}, {
"match": {
"info.category": "url"
}
}]
}
}
},
_source=["info.id"],
)
]
else:
rtmp = []
if rtmp and len(rtmp) > 0:
resolver_pool.map(lambda tid: delete_data(tid), rtmp)
def cuckoo_clean_sorted_pcap_dump():
"""Clean up failed tasks
It deletes all stored data from file system and configured databases (SQL
and MongoDB for failed tasks.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
if not is_reporting_db_connected():
return
if repconf.mongodb.enabled:
results_db = connect_to_mongo()[mdb]
elif repconf.elasticsearchdb.enabled:
es = connect_to_es()
done = False
while not done:
if repconf.mongodb and repconf.mongodb.enabled:
rtmp = results_db.analysis.find({"network.sorted_pcap_id": {"$exists": True}}, {"info.id": 1}, sort=[("_id", -1)]).limit(
100
)
elif repconf.elasticsearchdb.enabled:
rtmp = [d['_source'] for d in
all_docs(index=get_analysis_index(), query={
"query": {
"exists": {
"field": "network.sorted_pcap_id"
}
}
}, _source=['info.id'])]
else:
rtmp = 0
if rtmp and len(rtmp) > 0:
for e in rtmp:
if e["info"]["id"]:
log.info((e["info"]["id"]))
try:
if repconf.mongodb and repconf.mongodb.enabled:
results_db.analysis.update(
{"info.id": int(e["info"]["id"])},
{"$unset": {"network.sorted_pcap_id": ""}})
elif repconf.elasticsearchdb.enabled:
es.update(
index=e["index"], id=e["info"]["id"],
body={"network.sorted_pcap_id": ""}
)
except Exception:
log.info(("failed to remove sorted pcap from db for id %s" % (e["info"]["id"])))
try:
path = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % (e["info"]["id"]), "dump_sorted.pcap")
os.remove(path)
except Exception as e:
log.info(("failed to remove sorted_pcap from disk %s" % (e)))
else:
done = True
else:
done = True