def cuckoo_dedup_cluster_queue(): """ Cleans duplicated pending tasks from cluster queue """ main_db = Database() session = main_db.Session() dist_session = create_session(rep_config.distributed.db, echo=False) dist_db = dist_session() hash_dict = dict() duplicated = session.query(Sample, Task).join(Task).filter( Sample.id == Task.sample_id, Task.status == "pending").order_by(Sample.sha256) for sample, task in duplicated: try: # hash -> [[id, file]] hash_dict.setdefault(sample.sha256, list()) hash_dict[sample.sha256].append((task.id, task.target)) except UnicodeDecodeError: pass resolver_pool.map( lambda sha256: dist_delete_data(hash_dict[sha256][1:], dist_db), hash_dict)
machinery = Config(cfg.cuckoo.machinery) disable_x64 = cfg.cuckoo.get("disable_x64", False) apiconf = Config("api") rateblock = apiconf.api.get("ratelimit", False) db = Database() HAVE_DIST = False # Distributed CAPE if repconf.distributed.enabled: try: # Tags from lib.cuckoo.common.dist_db import Machine, create_session HAVE_DIST = True session = create_session(repconf.distributed.db) except Exception as e: print(e) ht = False try: """ To enable: sudo apt install apache2-utils """ from passlib.apache import HtpasswdFile HAVE_PASSLIB = True if apiconf.api.get("users_db") and os.path.exists(apiconf.api.get("users_db")): ht = HtpasswdFile(apiconf.api.get("users_db"))
from requests.auth import HTTPBasicAuth except ImportError: required("requests") try: requests.packages.urllib3.disable_warnings() except AttributeError: pass try: from flask_restful import abort, reqparse from flask_restful import Api as RestApi, Resource as RestResource except ImportError: required("flask-restful") session = create_session(reporting_conf.distributed.db) def node_status(url, name, ht_user, ht_pass): try: r = requests.get(os.path.join(url, "cuckoo", "status"), auth=HTTPBasicAuth(ht_user, ht_pass), verify=False, timeout=200) return r.json()["tasks"] except Exception as e: log.critical("Possible invalid Cuckoo node (%s): %s", name, e) return {} def node_fetch_tasks(status, url, ht_user, ht_pass, action="fetch", since=0):
import requests except ImportError: required("requests") try: requests.packages.urllib3.disable_warnings() except AttributeError: pass try: from flask_restful import abort, reqparse from flask_restful import Api as RestApi, Resource as RestResource except ImportError: required("flask-restful") session = create_session(reporting_conf.distributed.db, echo=False) def node_status(url, name, ht_user, ht_pass): try: r = requests.get(os.path.join(url, "cuckoo", "status"), params={"username": ht_user, "password": ht_pass}, verify=False, timeout=200) return r.json()["tasks"] except Exception as e: log.critical("Possible invalid Cuckoo node (%s): %s", name, e) return {} def node_fetch_tasks(status, url, ht_user, ht_pass, action="fetch", since=0): try: url = os.path.join(url, "tasks", "list") params = dict(status=status, ids=True, username=ht_user, password=ht_pass) if action == "fetch": params["completed_after"] = since