def cuckoo_dedup_cluster_queue():
"""
Cleans duplicated pending tasks from cluster queue
"""
main_db = Database()
session = main_db.Session()
dist_session = create_session(rep_config.distributed.db, echo=False)
dist_db = dist_session()
hash_dict = dict()
duplicated = session.query(Sample, Task).join(Task).filter(
Sample.id == Task.sample_id,
Task.status == "pending").order_by(Sample.sha256)
for sample, task in duplicated:
try:
# hash -> [[id, file]]
hash_dict.setdefault(sample.sha256, list())
hash_dict[sample.sha256].append((task.id, task.target))
except UnicodeDecodeError:
pass
resolver_pool.map(
lambda sha256: dist_delete_data(hash_dict[sha256][1:], dist_db),
hash_dict)
machinery = Config(cfg.cuckoo.machinery)
disable_x64 = cfg.cuckoo.get("disable_x64", False)
apiconf = Config("api")
rateblock = apiconf.api.get("ratelimit", False)
db = Database()
HAVE_DIST = False
# Distributed CAPE
if repconf.distributed.enabled:
try:
# Tags
from lib.cuckoo.common.dist_db import Machine, create_session
HAVE_DIST = True
session = create_session(repconf.distributed.db)
except Exception as e:
print(e)
ht = False
try:
"""
To enable: sudo apt install apache2-utils
"""
from passlib.apache import HtpasswdFile
HAVE_PASSLIB = True
if apiconf.api.get("users_db") and os.path.exists(apiconf.api.get("users_db")):
ht = HtpasswdFile(apiconf.api.get("users_db"))
from requests.auth import HTTPBasicAuth
except ImportError:
required("requests")
try:
requests.packages.urllib3.disable_warnings()
except AttributeError:
pass
try:
from flask_restful import abort, reqparse
from flask_restful import Api as RestApi, Resource as RestResource
except ImportError:
required("flask-restful")
session = create_session(reporting_conf.distributed.db)
def node_status(url, name, ht_user, ht_pass):
try:
r = requests.get(os.path.join(url, "cuckoo", "status"),
auth=HTTPBasicAuth(ht_user, ht_pass),
verify=False,
timeout=200)
return r.json()["tasks"]
except Exception as e:
log.critical("Possible invalid Cuckoo node (%s): %s", name, e)
return {}
def node_fetch_tasks(status, url, ht_user, ht_pass, action="fetch", since=0):
import requests
except ImportError:
required("requests")
try:
requests.packages.urllib3.disable_warnings()
except AttributeError:
pass
try:
from flask_restful import abort, reqparse
from flask_restful import Api as RestApi, Resource as RestResource
except ImportError:
required("flask-restful")
session = create_session(reporting_conf.distributed.db, echo=False)
def node_status(url, name, ht_user, ht_pass):
try:
r = requests.get(os.path.join(url, "cuckoo", "status"), params={"username": ht_user, "password": ht_pass}, verify=False, timeout=200)
return r.json()["tasks"]
except Exception as e:
log.critical("Possible invalid Cuckoo node (%s): %s", name, e)
return {}
def node_fetch_tasks(status, url, ht_user, ht_pass, action="fetch", since=0):
try:
url = os.path.join(url, "tasks", "list")
params = dict(status=status, ids=True, username=ht_user, password=ht_pass)
if action == "fetch":
params["completed_after"] = since
