def _create_bounty(guid,
truth_value=None,
settled=False,
n=0,
assertions=None):
s = DbSession()
b = DbBounty(guid=guid,
num_artifacts=n,
assertions=n,
expires=datetime.datetime.utcnow(),
truth_value=truth_value,
truth_settled=settled,
settle_block=1001)
s.add(b)
s.flush()
b_id = b.id
a_ids = []
for i in range(n):
a = DbArtifact(bounty_id=b_id, hash="%s" % i, name="%s.txt" % i)
s.add(a)
s.flush()
a_ids.append(a.id)
s.commit()
s.close()
return b_id, a_ids
def verdict_update_async(self, artifact_verdict_id, verdict):
"""Internal polling has resulted in an artifact verdict."""
s = DbSession()
try:
av = s.query(DbArtifactVerdict).with_for_update() \
.get(artifact_verdict_id)
artifact_id = av.artifact_id
if av.status != JOB_STATUS_PENDING:
log.warning("Task result for artifact #%s (%s) already made",
artifact_id, av.backend)
return
if verdict is False:
log.warning("Task failed for artifact #%s (%s)", artifact_id,
av.backend)
av.status = JOB_STATUS_FAILED
else:
log.debug("Task for artifact #%s (%s) complete", artifact_id,
av.backend)
av.verdict = verdict
av.status = JOB_STATUS_DONE
s.add(av)
s.commit()
dispatch_event("verdict_update", artifact_id)
finally:
s.close()
def advance_vote_bounty(self):
block_number = self.cur_block
pending = len(self.is_voting)
if pending >= MAX_OUTSTANDING_VOTES:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.voted.is_(False)) \
.filter((block_number - 60) >= DbBounty.vote_before) \
.filter(DbBounty.truth_value.isnot(None)) \
.with_for_update()
for b in bounties:
log.warning("%s | %s | Expired vote (%s)", b.guid, block_number,
b.vote_before)
b.voted = True
s.add(b)
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.voted.is_(False)) \
.filter(block_number >= DbBounty.vote_after) \
.filter(DbBounty.truth_value.isnot(None)) \
.filter(block_number >= DbBounty.error_delay_block) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_VOTES - pending)
for b in bounties:
_add_event(events, self.is_voting, b.guid, "bounty_vote", b.guid,
b.truth_value, b.vote_before)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def dashboard_manual_verdict(guid):
"""Set manual verdict for a bounty"""
verdicts = request.json["verdicts"]
app.logger.info("Received verdicts for bounty %s: %s", guid, verdicts)
if not isinstance(verdicts, list):
abort(400, "Verdicts is not a list")
for verdict in verdicts:
if verdict is not True and verdict is not False:
abort(400, "Invalid verdict value")
s = DbSession()
try:
b = s.query(DbBounty).with_for_update().filter_by(guid=guid).one()
if not b.truth_manual:
abort(403, "Bounty not in manual mode")
if len(verdicts) != b.num_artifacts:
abort(400, "This bounty requires %s verdicts" % b.num_artifacts)
if b.voted:
abort(403, "Bounty vote already submitted")
if b.settled:
abort(403, "Bounty already settled")
b.truth_value = verdicts
s.add(b)
s.commit()
finally:
s.close()
return jsonify({"status": "OK"})
def flush_expired_manual(self):
block = self.cur_block
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active", settled=False, truth_manual=True, voted=False) \
.filter(block > DbBounty.vote_before).with_for_update()
for b in bounties:
log.warning("%s | %s | Expired manual voting (%s)", b.guid, block,
b.vote_before)
b.voted = True
s.add(b)
s.commit()
s.close()
def bounty_vote(self, guid, value, vote_before):
"""Propagate bounty vote value to PolySwarm"""
if not value:
log.error("%s | Bad bounty_vote call %r", guid, value)
self.is_voting.discard(guid)
return
log.info("%s | %s | Vote on bounty: %s", guid, self.cur_block,
vote_show(value))
soft_fail = False
try:
if self.cur_block <= vote_before:
self.polyswarm.vote_bounty(guid, value)
else:
log.error("%s | %s | Permanent voting error: expired!",
self.cur_block, guid)
except PolySwarmError as e:
if e.status >= 500 and self.cur_block < vote_before:
log.error("%s | Temporary voting error: %s", guid, e.message
or e.reason)
# Server booboo, so try again later
soft_fail = True
else:
log.error("%s | Permanent voting error: %s", guid, e.message
or e.reason)
# Side-effect: we won't retry
except IOError as e:
log.error("%s | Temporary voting error: %s", guid, e)
soft_fail = True
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).one()
if not bounty.voted:
bounty.voted = True
if soft_fail:
# TODO: WS event
bounty.error_delay_block = self.cur_block + 5
bounty.error_retries += 1
if bounty.error_retries >= 3:
bounty.status = "aborted"
log.error(
"%s | %s | Aborted while voting, too many failures",
guid, self.cur_block)
s.add(bounty)
else:
log.warning("%s | %s | WARNING: double vote", guid, self.cur_block)
s.commit()
s.close()
self.is_voting.discard(guid)
def bounty_settle(self, guid):
"""Settle bounty for payout"""
log.info("%s | %s | Settle bounty", guid, self.cur_block)
failed = False
soft_fail = False
try:
self.polyswarm.settle_bounty(guid)
except PolySwarmNotFound:
# Record permanent failure
log.error("%s | Bounty no longer exists (double submit?)", guid)
failed = True
except PolySwarmError as e:
log.error("%s | Settle error: %s", guid, e)
failed = True
if "already been settled" not in str(e):
soft_fail = True
except IOError as e:
log.error("%s | Temporary settle error: %s", guid, e)
failed = True
soft_fail = True
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(
guid=guid).first()
if bounty and not bounty.settled:
if failed and soft_fail:
bounty.error_delay_block = self.cur_block + 5
bounty.error_retries += 1
if bounty.error_retries >= 3:
bounty.status = "aborted"
log.error(
"%s | %s | Aborted while settling, too many failures",
guid, self.cur_block)
else:
if failed:
bounty.status = "aborted"
else:
bounty.status = "finished"
bounty.settled = True
s.add(bounty)
s.commit()
s.close()
self.is_settling.discard(guid)
if not soft_fail:
dispatch_event("bounty_settled", guid)
def verdict_update(self, artifact_id):
"""Recompute final verdict for an artifact and trigger bounty settle if
needed."""
log.debug("Artifact #%s updated", artifact_id)
dispatch_event("metrics_artifact_complete", 1)
s = DbSession()
artifact = s.query(DbArtifact).with_for_update().get(artifact_id)
if artifact.processed:
log.warning("Verdict for artifact #%s already made", artifact_id)
s.close()
return
verdicts = s.query(DbArtifactVerdict).filter_by(
artifact_id=artifact_id)
bounty_id = artifact.bounty_id
incomplete = False
verdict_map = {}
for verdict in verdicts.all():
if verdict.status > JOB_STATUS_DONE:
incomplete = True
verdict_map[verdict.backend] = verdict.verdict
if not incomplete:
log.debug("Verdict for artifact #%s can be made: %r", artifact_id,
verdict_map)
verdict = vote_on_artifact(verdict_map)
log.debug("Verdict for artifact #%s: %r", artifact_id, verdict)
artifact.processed = True
artifact.processed_at = datetime.datetime.utcnow()
artifact.processed_at_interval = interval(time.time(),
self.artifact_interval)
artifact.verdict = verdict
s.add(artifact)
s.commit()
dispatch_event("metrics_artifact_verdict", verdict)
else:
log.debug("Verdict for artifact #%s incomplete", artifact_id)
bounty_id = None
s.close()
if bounty_id is not None:
dispatch_event("bounty_artifact_verdict", bounty_id)
def action_artifact(analysis_backend, artifact_id):
if not isinstance(request.json, dict):
abort(400, "No JSON POST body found")
if "error" in request.json:
pass
if "verdict_value" not in request.json:
abort(400, "Missing verdict_value")
verdict_value = request.json["verdict_value"]
if verdict_value is not None:
verdict_value = int(verdict_value)
if verdict_value < 0 or verdict_value > 100:
abort(400, "Invalid verdict value")
s = DbSession()
verdict = s.query(DbArtifactVerdict) \
.with_for_update().filter(and_(
DbArtifactVerdict.backend == analysis_backend.name,
DbArtifactVerdict.artifact_id == artifact_id
)).first()
if not verdict:
s.close()
abort(404, "Artifact #%d not found" % artifact_id)
if verdict.status == JOB_STATUS_DONE:
s.close()
abort(403, "Verdict for artifact #%s already submitted" % artifact_id)
verdict.status = JOB_STATUS_DONE
verdict.verdict = verdict_value
s.add(verdict)
s.commit()
s.close()
app.logger.debug("Received verdict for artifact #%s from %s", artifact_id,
analysis_backend.name)
dispatch_event("verdict_update", artifact_id)
return jsonify({"status": "OK"})
def expire_pending(self):
"""Expire pending verdict tasks."""
# TODO: preferably replace "arbitrary" timeout window with backend
# polling
notify_tasks = set()
now = datetime.datetime.utcnow()
s = DbSession()
avs = s.query(DbArtifactVerdict).with_for_update() \
.filter_by(status=JOB_STATUS_PENDING) \
.filter(DbArtifactVerdict.expires < now)
for av in avs:
log.warning("Job %s expired", av.id)
av.status = JOB_STATUS_FAILED
# TODO: call backend.cancel_artifact
s.add(av)
notify_tasks.add(av.artifact_id)
s.commit()
s.close()
for aid in notify_tasks:
dispatch_event("verdict_update", aid)
def bounty_assertions_reveal(self, guid, value):
"""Reveal assertions.
We should have already voted."""
log.debug("%s | Checking assertions", guid)
experts_disagree = False
assertions = []
try:
assertions = self.polyswarm.bounty_assertions(guid)
if value:
experts_disagree = self._bounty_assertions_disagree(
guid, value, assertions)
except PolySwarmNotFound:
pass
except PolySwarmError as e:
log.error("%s | Assertion fetch error: %s", guid, e)
except Exception as e:
# We ignore this for now
log.error("Failed to check assertions: %s", e)
if assertions:
log.debug("%s | %s assertion(s)", guid, len(assertions))
s = DbSession()
bounty = s.query(DbBounty).with_for_update() \
.filter_by(guid=guid).one()
bounty.revealed = True
bounty.assertions = assertions
if experts_disagree and not bounty.settled:
# Mark as manual so we don't auto-settle
#log.warning("%s | Set to manual", guid)
#bounty.truth_manual = True
pass
#settle_block = bounty.settle_block
s.add(bounty)
s.commit()
s.close()
self.is_revealing.discard(guid)
def verdict_jobs(self, bounty_guid, artifact_id):
"""Jobs to submit or otherwise check"""
submit = []
# Find jobs we need to submit, and mark them
s = DbSession()
a = s.query(DbArtifact).filter_by(id=artifact_id).one()
artifact = Artifact(a.id, a.name, a.hash,
"%s/artifact/%s" % (self.url, a.id))
avs = s.query(DbArtifactVerdict).with_for_update() \
.filter_by(artifact_id=artifact.id, status=JOB_STATUS_NEW)
for av in avs.all():
submit.append((av.id, av.backend, artifact, av.meta))
av.status = JOB_STATUS_SUBMITTING
s.add(av)
s.commit()
s.close()
dispatch_event("verdict_job_submit", artifact_id, submit)
def __enter__(self):
s = DbSession()
g = str(uuid.uuid4())
b = DbBounty(guid=g,
expires=datetime.datetime.utcnow() +
datetime.timedelta(days=1),
num_artifacts=len(self.with_verdicts),
settle_block=1000)
s.add(b)
s.flush()
bid = b.id
a = DbArtifact(bounty_id=b.id, hash="Q121293810", name="dummy.exe")
s.add(a)
s.flush()
aid = a.id
jobs = []
for backend, attr in self.with_verdicts.items():
av = DbArtifactVerdict(artifact_id=aid,
status=JOB_STATUS_DONE,
backend=backend)
for k, v in attr.items():
setattr(av, k, v)
s.add(av)
s.flush()
jobs.append([av.id, backend, None])
s.commit()
s.close()
return {"bid": bid, "guid": g, "artifact_id": aid, "jobs": jobs}
def bounty_settle_manual(guid, votes):
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).first()
if not bounty:
s.close()
raise KeyError("No such bounty")
elif bounty.voted or bounty.settled:
s.close()
raise ValueError("Bounty was already voted on/settled")
need = s.query(DbArtifact).filter_by(bounty_id=bounty.id).count()
if need != len(votes):
s.close()
raise ValueError("Need %s vote(s), not %s" % (need, len(votes)))
log.info("Manually set bounty %s vote to %s", guid, votes)
bounty.truth_value = votes
bounty.truth_manual = True
s.add(bounty)
s.commit()
s.close()
def bounty_artifact_verdict(self, bounty_id):
"""Check if bounty can be voted on after artifact update"""
s = DbSession()
bounty = s.query(DbBounty).with_for_update() \
.filter_by(id=bounty_id).one()
if bounty.truth_value is not None:
log.warning("%s | Bounty already has truth value, nothing to do",
bounty.guid)
s.close()
return
elif bounty.truth_manual:
# Already manual, so it won't make a difference
s.close()
return
# This may happen *after* the voting window, in that case mark
# the bounty as aborted
if self.cur_block and self.cur_block >= bounty.vote_before:
guid = None
if bounty.status != "aborted":
log.error(
"%s | Bounty artifact vote came in too late:"
" at block %s, voting ended on %s!", bounty.guid,
self.cur_block, bounty.vote_before)
bounty.status = "aborted"
guid = bounty.guid
s.add(bounty)
s.commit()
s.close()
if guid:
dispatch_event("bounty_aborted", guid)
return
# Collect votes
guid = bounty.guid
artifacts = s.query(DbArtifact).filter_by(bounty_id=bounty_id) \
.order_by(DbArtifact.id).all()
votes = []
record_value = True
can_vote = self.cur_block and self.cur_block >= bounty.vote_after
transition_manual = False
for artifact in artifacts:
if not artifact.processed:
log.debug("%s | Artifact #%s still has no vote", bounty.guid,
artifact.id)
record_value = can_vote = False
break
elif artifact.verdict is None:
log.debug("%s | Artifact #%s has DONTKNOW vote", bounty.guid,
artifact.id)
#can_vote = False
transition_manual = True
elif artifact.verdict >= VERDICT_MAYBE:
votes.append(True) # Malicious
else:
votes.append(False) # Safe
# Assertions can no longer come in before we've voted
## In case artifact votes came in after settle block
#if bounty.assertions and not transition_manual:
# transition_manual = self._bounty_assertions_disagree(bounty.guid, votes, bounty.assertions)
#vote_before = bounty.vote_before
if transition_manual:
log.debug("%s | Mark bounty as requiring manual vote", bounty.guid)
bounty.truth_manual = True
s.add(bounty)
s.commit()
elif record_value:
log.debug("%s | Recording vote: %s", bounty.guid, vote_show(votes))
bounty.truth_value = votes
s.add(bounty)
s.commit()
s.close()
#if can_vote and votes and guid not in self.is_voting:
# self.is_voting.add(guid)
# dispatch_event("bounty_vote", guid, votes, vote_before)
if transition_manual:
dispatch_event("bounty_manual", guid)
def bounty_with_manifest(self, bounty):
"""A bounty has become available: register it for processing"""
if bounty.get("resolved"):
return
#if self.first:
# self.first = False
#else:
# return
# Download related files
try:
manifest = ipfs_json(bounty["uri"], cache=False)
except IPFSNotFoundError:
# Shouldn't happen in production
return
except:
# TODO: track some well-known issues (e.g. disk full, ...)
# TODO: right now we'll skip these bounties!
log.warning("Couldn't fetch artifact data for bounty %s",
bounty["guid"])
return
# {hash, name}, both matter
manifest = manifest["result"]
if not manifest:
log.warning("Bounty %s has no artifacts", bounty["guid"])
return
# TODO Reintroduce explicit checking of artifact count.
# if len(manifest) != bounty["num_artifacts"]:
# log.warning(
# "Bounty %s has manifest with %s entries, but claims %s",
# bounty["guid"], len(manifest), bounty["num_artifacts"])
# return
num_artifacts = len(manifest)
expiration = int(bounty["expiration"])
s = DbSession()
b = DbBounty(guid=bounty["guid"],
author=bounty["author"],
amount=bounty["amount"],
num_artifacts=num_artifacts,
error_delay_block=0,
expiration_block=expiration,
vote_after=expiration + self.polyswarm.reveal_window + 1,
vote_before=expiration + self.polyswarm.vote_window,
reveal_block=expiration + self.polyswarm.vote_window +
self.polyswarm.reveal_window,
settle_block=expiration + self.polyswarm.vote_window +
self.polyswarm.reveal_window)
if self.manual_mode:
b.truth_manual = True
s.add(b)
try:
s.flush()
except IntegrityError:
# log.error("%s => %s", bounty["guid"], e)
# Bounty already exists, ignore
log.debug("Bounty %s already exists", bounty["guid"])
s.close()
return
log.info(
"%s | New bounty | artifacts=%s expiration=%s vote_before=%s settle=%s",
bounty["guid"], len(manifest), expiration, b.vote_before,
b.settle_block)
# Create jobs for every backend. If new backends join or backends are
# removed, tasks are *not* automatically updated.
job_ids = []
for artifact in manifest:
a = DbArtifact(bounty_id=b.id,
hash=artifact["hash"],
name=artifact["name"])
s.add(a)
s.flush()
job_ids.append(a.id)
jobs = []
# TODO: analysis_backends may change during different runs
for backend in analysis_backends.values():
v = DbArtifactVerdict(artifact_id=a.id,
backend=backend.name,
status=JOB_STATUS_NEW)
s.add(v)
jobs.append(v)
s.flush()
s.commit()
s.close()
artifacts = []
for i, artifact in enumerate(manifest):
# TODO: this is just the way their API works
# TODO: parallel download
artifacts.append(
gevent.spawn(ipfs_download, artifact["hash"],
"%s/%s" % (bounty["uri"], i)))
gevent.joinall(artifacts)
for a in artifacts:
if a.exception is not None:
# TODO: we should abort here
log.warning("Downloading artifacts for %s not succesful!",
bounty["guid"])
break
# Start submitting the artifacts
for job in job_ids:
dispatch_event("verdict_jobs", bounty["guid"], job)
