def advance_vote_bounty(self): block_number = self.cur_block pending = len(self.is_voting) if pending >= MAX_OUTSTANDING_VOTES: return events = [] s = DbSession() bounties = s.query(DbBounty).filter_by(status="active") \ .filter(DbBounty.voted.is_(False)) \ .filter((block_number - 60) >= DbBounty.vote_before) \ .filter(DbBounty.truth_value.isnot(None)) \ .with_for_update() for b in bounties: log.warning("%s | %s | Expired vote (%s)", b.guid, block_number, b.vote_before) b.voted = True s.add(b) bounties = s.query(DbBounty).filter_by(status="active") \ .filter(DbBounty.voted.is_(False)) \ .filter(block_number >= DbBounty.vote_after) \ .filter(DbBounty.truth_value.isnot(None)) \ .filter(block_number >= DbBounty.error_delay_block) \ .order_by(DbBounty.id).limit(MAX_OUTSTANDING_VOTES - pending) for b in bounties: _add_event(events, self.is_voting, b.guid, "bounty_vote", b.guid, b.truth_value, b.vote_before) s.commit() s.close() for e, args in events: dispatch_event(e, *args)
def test_expire_verdicts(dispatch_event, db): v = VerdictComponent(Parent()) verdicts.analysis_backends = { u"cuckoo": AnalysisBackend(u"cuckoo", True, 1), u"zer0m0n": AnalysisBackend(u"zer0m0n", False, 1), } expire_at = datetime.datetime.utcnow() - datetime.timedelta(hours=1) mixed = artifact({ u"cuckoo": { "status": JOB_STATUS_PENDING, "expires": expire_at }, u"zer0m0n": { "status": JOB_STATUS_DONE, "expires": expire_at } }) with mixed: v.expire_verdicts() assert dispatch_event.called s = DbSession() try: for av in s.query(DbArtifactVerdict): if av.backend == "cuckoo": assert av.status == JOB_STATUS_FAILED else: assert av.status == JOB_STATUS_DONE finally: s.close() db_clear()
def dashboard_bounties_guid(guid): s = DbSession() b = s.query(DbBounty).filter_by(guid=guid) \ .order_by(DbBounty.id).one_or_none() if b is None: return jsonify({"error": "No such bounty"}), 404 data = _gather_bounty_data(b) s.close() return jsonify(data)
def dashboard_bounties_manual(): """All not-yet-set bounties that need a manual verdict""" s = DbSession() bs = s.query(DbBounty) \ .filter_by(truth_manual=True, voted=False) \ .filter(DbBounty.truth_value.is_(None)) \ .order_by(DbBounty.id) bounties = [_gather_bounty_data(b, False) for b in bs] s.close() return jsonify(bounties)
def dashboard_bounties_pending(): # Bounties that are being processed or need to be submitted s = DbSession() bs = s.query(DbBounty).filter_by(status="active") \ .filter(or_(DbBounty.truth_manual.is_(False), DbBounty.truth_value.isnot(None))) \ .order_by(DbBounty.id) bounties = [_gather_bounty_data(b, False) for b in bs] s.close() return jsonify(bounties)
def verdict_update_async(self, artifact_verdict_id, verdict): """Internal polling has resulted in an artifact verdict.""" s = DbSession() try: av = s.query(DbArtifactVerdict).with_for_update() \ .get(artifact_verdict_id) artifact_id = av.artifact_id if av.status != JOB_STATUS_PENDING: log.warning("Task result for artifact #%s (%s) already made", artifact_id, av.backend) return if verdict is False: log.warning("Task failed for artifact #%s (%s)", artifact_id, av.backend) av.status = JOB_STATUS_FAILED else: log.debug("Task for artifact #%s (%s) complete", artifact_id, av.backend) av.verdict = verdict av.status = JOB_STATUS_DONE s.add(av) s.commit() dispatch_event("verdict_update", artifact_id) finally: s.close()
def dashboard_manual_verdict(guid): """Set manual verdict for a bounty""" verdicts = request.json["verdicts"] app.logger.info("Received verdicts for bounty %s: %s", guid, verdicts) if not isinstance(verdicts, list): abort(400, "Verdicts is not a list") for verdict in verdicts: if verdict is not True and verdict is not False: abort(400, "Invalid verdict value") s = DbSession() try: b = s.query(DbBounty).with_for_update().filter_by(guid=guid).one() if not b.truth_manual: abort(403, "Bounty not in manual mode") if len(verdicts) != b.num_artifacts: abort(400, "This bounty requires %s verdicts" % b.num_artifacts) if b.voted: abort(403, "Bounty vote already submitted") if b.settled: abort(403, "Bounty already settled") b.truth_value = verdicts s.add(b) s.commit() finally: s.close() return jsonify({"status": "OK"})
def pending(): from arbiter.database import DbSession, DbArtifactVerdict s = DbSession() #for av in s.query(DbArtifactVerdict).filter(DbArtifactVerdict.verdict.is_(None)).all(): for av in s.query(DbArtifactVerdict).filter( DbArtifactVerdict.status != 0).all(): status = JOB_STATUS_NAMES.get(av.status, av.status) print("ID: %5s" % av.id, "AVID: %5s" % av.artifact_id, "Backend: %-10s" % av.backend, "S: %-10s" % status, "EXP:", av.expires)
def retry_submissions(self): """ Conditions: * Failure was temporary * Bounty is not about to expire """ s = DbSession() avs = [a.artifact_id for a in s.query(DbArtifactVerdict.artifact_id) \ .filter_by(status=JOB_STATUS_NEW)] s.close() for a in avs: dispatch_event("verdict_jobs", None, a)
def list_artifacts(analysis_backend): s = DbSession() artifacts = [] q = s.query(DbArtifact).filter(~exists().where( and_(DbArtifactVerdict.artifact_id == DbArtifact.id, DbArtifactVerdict.backend == analysis_backend.name))) for artifact in q: artifacts.append(artifact.id) s.close() return jsonify(artifacts)
def _verdict_set(a_id, value): s = DbSession() try: s.query(DbArtifact).filter_by(id=a_id) \ .update({DbArtifact.verdict: value}) s.commit() finally: s.close()
def flush_expired_manual(self): block = self.cur_block s = DbSession() bounties = s.query(DbBounty).filter_by(status="active", settled=False, truth_manual=True, voted=False) \ .filter(block > DbBounty.vote_before).with_for_update() for b in bounties: log.warning("%s | %s | Expired manual voting (%s)", b.guid, block, b.vote_before) b.voted = True s.add(b) s.commit() s.close()
def artifact_datapoints(): unow = datetime.datetime.utcnow() ustart = unow - datetime.timedelta(days=5) start = int(time.mktime(ustart.timetuple())) now = int(time.mktime(unow.timetuple())) data = [] s = DbSession() try: step_time = app.component.artifact_interval rs = s.query(DbArtifact.processed_at_interval, func.count(1)) \ .filter(DbArtifact.processed_at_interval.isnot(None)) \ .filter(DbArtifact.processed_at_interval > start) \ .group_by(DbArtifact.processed_at_interval) \ .order_by(DbArtifact.processed_at_interval) prev = None for r in rs: stamp = r.processed_at_interval for step in missing_time_steps(prev, stamp, step_time): data.append([step, 0]) data.append([stamp, r[1]]) prev = stamp if prev: if (now - prev) > step_time: # We've stopped seeing entries, so end with 0 data.append([prev + step_time, 0]) data.append([now, 0]) #elif data[-1][0] > now: # # Extrapolate the last entry to prevent a "dip" # last = data[-1][0] # boost = (step_time - (last - now)) / float(step_time) # data[-1][1] = data[-1][1] / boost finally: s.close() if len(data) == 1: data.insert(0, [data[0][0] - step_time, 0]) data.sort(key=lambda v: tuple(v)) return jsonify({ "start": data[0][0] if data else start, "end": data[-1][0] if data else now, "data": data, })
def bounty_vote(self, guid, value, vote_before): """Propagate bounty vote value to PolySwarm""" if not value: log.error("%s | Bad bounty_vote call %r", guid, value) self.is_voting.discard(guid) return log.info("%s | %s | Vote on bounty: %s", guid, self.cur_block, vote_show(value)) soft_fail = False try: if self.cur_block <= vote_before: self.polyswarm.vote_bounty(guid, value) else: log.error("%s | %s | Permanent voting error: expired!", self.cur_block, guid) except PolySwarmError as e: if e.status >= 500 and self.cur_block < vote_before: log.error("%s | Temporary voting error: %s", guid, e.message or e.reason) # Server booboo, so try again later soft_fail = True else: log.error("%s | Permanent voting error: %s", guid, e.message or e.reason) # Side-effect: we won't retry except IOError as e: log.error("%s | Temporary voting error: %s", guid, e) soft_fail = True s = DbSession() bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).one() if not bounty.voted: bounty.voted = True if soft_fail: # TODO: WS event bounty.error_delay_block = self.cur_block + 5 bounty.error_retries += 1 if bounty.error_retries >= 3: bounty.status = "aborted" log.error( "%s | %s | Aborted while voting, too many failures", guid, self.cur_block) s.add(bounty) else: log.warning("%s | %s | WARNING: double vote", guid, self.cur_block) s.commit() s.close() self.is_voting.discard(guid)
def bounty_settle(self, guid): """Settle bounty for payout""" log.info("%s | %s | Settle bounty", guid, self.cur_block) failed = False soft_fail = False try: self.polyswarm.settle_bounty(guid) except PolySwarmNotFound: # Record permanent failure log.error("%s | Bounty no longer exists (double submit?)", guid) failed = True except PolySwarmError as e: log.error("%s | Settle error: %s", guid, e) failed = True if "already been settled" not in str(e): soft_fail = True except IOError as e: log.error("%s | Temporary settle error: %s", guid, e) failed = True soft_fail = True s = DbSession() bounty = s.query(DbBounty).with_for_update().filter_by( guid=guid).first() if bounty and not bounty.settled: if failed and soft_fail: bounty.error_delay_block = self.cur_block + 5 bounty.error_retries += 1 if bounty.error_retries >= 3: bounty.status = "aborted" log.error( "%s | %s | Aborted while settling, too many failures", guid, self.cur_block) else: if failed: bounty.status = "aborted" else: bounty.status = "finished" bounty.settled = True s.add(bounty) s.commit() s.close() self.is_settling.discard(guid) if not soft_fail: dispatch_event("bounty_settled", guid)
def verdict_jobs(self, bounty_guid, artifact_id): """Jobs to submit or otherwise check""" submit = [] # Find jobs we need to submit, and mark them s = DbSession() a = s.query(DbArtifact).filter_by(id=artifact_id).one() artifact = Artifact(a.id, a.name, a.hash, "%s/artifact/%s" % (self.url, a.id)) avs = s.query(DbArtifactVerdict).with_for_update() \ .filter_by(artifact_id=artifact.id, status=JOB_STATUS_NEW) for av in avs.all(): submit.append((av.id, av.backend, artifact, av.meta)) av.status = JOB_STATUS_SUBMITTING s.add(av) s.commit() s.close() dispatch_event("verdict_job_submit", artifact_id, submit)
def reset_pending_jobs(): """ Reset jobs that were pending submission. This may result in jobs being submitted twice when the arbiter is restarted. """ log.debug("Reset pending jobs") # TODO: this is incompatible with a multi-process approach s = DbSession() s.query(DbArtifactVerdict) \ .filter_by(status=JOB_STATUS_PENDING) \ .update({DbArtifactVerdict.status: JOB_STATUS_NEW}, synchronize_session=False) s.commit() s.close()
def bounties(): from arbiter.database import DbSession, DbBounty print("Status".ljust(8), "GUID".ljust(36), "MRVS", "<Vote", ">Settle", "Value") S = {True: "*", False: " "} s = DbSession() for b in s.query(DbBounty).order_by(DbBounty.id).all(): if not b.truth_value: value = "-" else: value = "".join(str(v)[:1] for v in b.truth_value) print( b.status.ljust(8), b.guid, S[b.truth_manual] + S[b.revealed] + S[b.voted] + S[b.settled], str(b.vote_before).ljust(5), str(b.settle_block).ljust(7), value.ljust(5), )
def _bounty_check_state(b, should_be_set, should_be_settled=None): s = DbSession() try: if isinstance(b, int): kwargs = {"id": b} else: kwargs = {"guid": b} b = s.query(DbBounty).filter_by(**kwargs).one() is_set = b.truth_value is not None is_settled = b.truth_settled is True finally: s.close() if not should_be_set and is_set: raise ValueError("Bounty %s was set, when it should not be" % b) elif should_be_set and not is_set: raise ValueError("Bounty %s was not set, when it should be" % b) if should_be_settled is None: return if not should_be_settled and is_settled: raise ValueError("Bounty %s was settled, when it should not be" % b) elif should_be_settled and not is_settled: raise ValueError("Bounty %s was not settled, when it should be" % b)
def expire_pending(self): """Expire pending verdict tasks.""" # TODO: preferably replace "arbitrary" timeout window with backend # polling notify_tasks = set() now = datetime.datetime.utcnow() s = DbSession() avs = s.query(DbArtifactVerdict).with_for_update() \ .filter_by(status=JOB_STATUS_PENDING) \ .filter(DbArtifactVerdict.expires < now) for av in avs: log.warning("Job %s expired", av.id) av.status = JOB_STATUS_FAILED # TODO: call backend.cancel_artifact s.add(av) notify_tasks.add(av.artifact_id) s.commit() s.close() for aid in notify_tasks: dispatch_event("verdict_update", aid)
def test_verdict_job_submit(dispatch_event, db): v = VerdictComponent(Parent()) verdicts.analysis_backends = { u"cuckoo": AnalysisBackend(u"cuckoo", True, 1), u"zer0m0n": AnalysisBackend(u"zer0m0n", False, 1), u"cuckscan": AnalysisBackend(u"cuckscan", False, 1), } verdicts.analysis_backends[u"cuckoo"].submit_artifact = lambda a: None verdicts.analysis_backends[u"zer0m0n"].submit_artifact = lambda a: {} verdicts.analysis_backends[u"cuckscan"].submit_artifact = lambda a: 1 pending = artifact({ u"cuckoo": { "status": JOB_STATUS_SUBMITTING }, u"zer0m0n": { "status": JOB_STATUS_SUBMITTING }, u"cuckscan": { "status": JOB_STATUS_SUBMITTING } }) with pending as x: v.verdict_job_submit(x["artifact_id"], x["jobs"]) assert dispatch_event.called s = DbSession() try: for av in s.query(DbArtifactVerdict): if av.backend == "cuckscan": assert av.status == JOB_STATUS_DONE else: assert av.status == JOB_STATUS_PENDING finally: s.close() db_clear()
def bounty_assertions_reveal(self, guid, value): """Reveal assertions. We should have already voted.""" log.debug("%s | Checking assertions", guid) experts_disagree = False assertions = [] try: assertions = self.polyswarm.bounty_assertions(guid) if value: experts_disagree = self._bounty_assertions_disagree( guid, value, assertions) except PolySwarmNotFound: pass except PolySwarmError as e: log.error("%s | Assertion fetch error: %s", guid, e) except Exception as e: # We ignore this for now log.error("Failed to check assertions: %s", e) if assertions: log.debug("%s | %s assertion(s)", guid, len(assertions)) s = DbSession() bounty = s.query(DbBounty).with_for_update() \ .filter_by(guid=guid).one() bounty.revealed = True bounty.assertions = assertions if experts_disagree and not bounty.settled: # Mark as manual so we don't auto-settle #log.warning("%s | Set to manual", guid) #bounty.truth_manual = True pass #settle_block = bounty.settle_block s.add(bounty) s.commit() s.close() self.is_revealing.discard(guid)
def advance_reveal(self): block_number = self.cur_block pending = len(self.is_revealing) if pending >= MAX_OUTSTANDING_REVEALS: return events = [] s = DbSession() bounties = s.query(DbBounty).filter_by(status="active") \ .filter(DbBounty.revealed.is_(False)) \ .filter(block_number >= DbBounty.reveal_block) \ .filter(DbBounty.assertions.is_(None)) \ .order_by(DbBounty.id).limit(MAX_OUTSTANDING_REVEALS - pending) for b in bounties: _add_event(events, self.is_revealing, b.guid, "bounty_assertions_reveal", b.guid, b.truth_value) s.commit() s.close() for e, args in events: dispatch_event(e, *args)
def advance_settle(self): block_number = self.cur_block pending = len(self.is_settling) if pending >= MAX_OUTSTANDING_SETTLES: return events = [] s = DbSession() bounties = s.query(DbBounty).filter_by(status="active") \ .filter(DbBounty.assertions.isnot(None)) \ .filter(DbBounty.settled.is_(False)) \ .filter(block_number >= DbBounty.settle_block) \ .filter(block_number >= DbBounty.error_delay_block) \ .order_by(DbBounty.id).limit(MAX_OUTSTANDING_SETTLES - pending) for b in bounties: _add_event(events, self.is_settling, b.guid, "bounty_settle", b.guid) s.commit() s.close() for e, args in events: dispatch_event(e, *args)
def counters(self): # TODO: update on trigger counters = [] s = DbSession() try: c = s.query(DbBounty.id).filter_by(settled=True).count() counters.append(("counter-bounties-settled", c)) c = s.query(DbArtifact.id).filter_by(processed=False).count() counters.append(("counter-artifacts-processing", c)) counters.append( ("counter-backends-running", len(analysis_backends))) counters.append(("counter-errors", 0)) finally: s.close() for k, c in counters: broadcast(k, c)
def _no_such_bounty(guid): s = DbSession() try: assert s.query(DbBounty).filter_by(guid=guid).count() == 0 finally: s.close()
def _create_bounty(guid, truth_value=None, settled=False, n=0, assertions=None): s = DbSession() b = DbBounty(guid=guid, num_artifacts=n, assertions=n, expires=datetime.datetime.utcnow(), truth_value=truth_value, truth_settled=settled, settle_block=1001) s.add(b) s.flush() b_id = b.id a_ids = [] for i in range(n): a = DbArtifact(bounty_id=b_id, hash="%s" % i, name="%s.txt" % i) s.add(a) s.flush() a_ids.append(a.id) s.commit() s.close() return b_id, a_ids
def bounty_artifact_verdict(self, bounty_id): """Check if bounty can be voted on after artifact update""" s = DbSession() bounty = s.query(DbBounty).with_for_update() \ .filter_by(id=bounty_id).one() if bounty.truth_value is not None: log.warning("%s | Bounty already has truth value, nothing to do", bounty.guid) s.close() return elif bounty.truth_manual: # Already manual, so it won't make a difference s.close() return # This may happen *after* the voting window, in that case mark # the bounty as aborted if self.cur_block and self.cur_block >= bounty.vote_before: guid = None if bounty.status != "aborted": log.error( "%s | Bounty artifact vote came in too late:" " at block %s, voting ended on %s!", bounty.guid, self.cur_block, bounty.vote_before) bounty.status = "aborted" guid = bounty.guid s.add(bounty) s.commit() s.close() if guid: dispatch_event("bounty_aborted", guid) return # Collect votes guid = bounty.guid artifacts = s.query(DbArtifact).filter_by(bounty_id=bounty_id) \ .order_by(DbArtifact.id).all() votes = [] record_value = True can_vote = self.cur_block and self.cur_block >= bounty.vote_after transition_manual = False for artifact in artifacts: if not artifact.processed: log.debug("%s | Artifact #%s still has no vote", bounty.guid, artifact.id) record_value = can_vote = False break elif artifact.verdict is None: log.debug("%s | Artifact #%s has DONTKNOW vote", bounty.guid, artifact.id) #can_vote = False transition_manual = True elif artifact.verdict >= VERDICT_MAYBE: votes.append(True) # Malicious else: votes.append(False) # Safe # Assertions can no longer come in before we've voted ## In case artifact votes came in after settle block #if bounty.assertions and not transition_manual: # transition_manual = self._bounty_assertions_disagree(bounty.guid, votes, bounty.assertions) #vote_before = bounty.vote_before if transition_manual: log.debug("%s | Mark bounty as requiring manual vote", bounty.guid) bounty.truth_manual = True s.add(bounty) s.commit() elif record_value: log.debug("%s | Recording vote: %s", bounty.guid, vote_show(votes)) bounty.truth_value = votes s.add(bounty) s.commit() s.close() #if can_vote and votes and guid not in self.is_voting: # self.is_voting.add(guid) # dispatch_event("bounty_vote", guid, votes, vote_before) if transition_manual: dispatch_event("bounty_manual", guid)
def bounty_with_manifest(self, bounty): """A bounty has become available: register it for processing""" if bounty.get("resolved"): return #if self.first: # self.first = False #else: # return # Download related files try: manifest = ipfs_json(bounty["uri"], cache=False) except IPFSNotFoundError: # Shouldn't happen in production return except: # TODO: track some well-known issues (e.g. disk full, ...) # TODO: right now we'll skip these bounties! log.warning("Couldn't fetch artifact data for bounty %s", bounty["guid"]) return # {hash, name}, both matter manifest = manifest["result"] if not manifest: log.warning("Bounty %s has no artifacts", bounty["guid"]) return # TODO Reintroduce explicit checking of artifact count. # if len(manifest) != bounty["num_artifacts"]: # log.warning( # "Bounty %s has manifest with %s entries, but claims %s", # bounty["guid"], len(manifest), bounty["num_artifacts"]) # return num_artifacts = len(manifest) expiration = int(bounty["expiration"]) s = DbSession() b = DbBounty(guid=bounty["guid"], author=bounty["author"], amount=bounty["amount"], num_artifacts=num_artifacts, error_delay_block=0, expiration_block=expiration, vote_after=expiration + self.polyswarm.reveal_window + 1, vote_before=expiration + self.polyswarm.vote_window, reveal_block=expiration + self.polyswarm.vote_window + self.polyswarm.reveal_window, settle_block=expiration + self.polyswarm.vote_window + self.polyswarm.reveal_window) if self.manual_mode: b.truth_manual = True s.add(b) try: s.flush() except IntegrityError: # log.error("%s => %s", bounty["guid"], e) # Bounty already exists, ignore log.debug("Bounty %s already exists", bounty["guid"]) s.close() return log.info( "%s | New bounty | artifacts=%s expiration=%s vote_before=%s settle=%s", bounty["guid"], len(manifest), expiration, b.vote_before, b.settle_block) # Create jobs for every backend. If new backends join or backends are # removed, tasks are *not* automatically updated. job_ids = [] for artifact in manifest: a = DbArtifact(bounty_id=b.id, hash=artifact["hash"], name=artifact["name"]) s.add(a) s.flush() job_ids.append(a.id) jobs = [] # TODO: analysis_backends may change during different runs for backend in analysis_backends.values(): v = DbArtifactVerdict(artifact_id=a.id, backend=backend.name, status=JOB_STATUS_NEW) s.add(v) jobs.append(v) s.flush() s.commit() s.close() artifacts = [] for i, artifact in enumerate(manifest): # TODO: this is just the way their API works # TODO: parallel download artifacts.append( gevent.spawn(ipfs_download, artifact["hash"], "%s/%s" % (bounty["uri"], i))) gevent.joinall(artifacts) for a in artifacts: if a.exception is not None: # TODO: we should abort here log.warning("Downloading artifacts for %s not succesful!", bounty["guid"]) break # Start submitting the artifacts for job in job_ids: dispatch_event("verdict_jobs", bounty["guid"], job)
def bounty_settle_manual(guid, votes): s = DbSession() bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).first() if not bounty: s.close() raise KeyError("No such bounty") elif bounty.voted or bounty.settled: s.close() raise ValueError("Bounty was already voted on/settled") need = s.query(DbArtifact).filter_by(bounty_id=bounty.id).count() if need != len(votes): s.close() raise ValueError("Need %s vote(s), not %s" % (need, len(votes))) log.info("Manually set bounty %s vote to %s", guid, votes) bounty.truth_value = votes bounty.truth_manual = True s.add(bounty) s.commit() s.close()