def verdict_update_async(self, artifact_verdict_id, verdict):
"""Internal polling has resulted in an artifact verdict."""
s = DbSession()
try:
av = s.query(DbArtifactVerdict).with_for_update() \
.get(artifact_verdict_id)
artifact_id = av.artifact_id
if av.status != JOB_STATUS_PENDING:
log.warning("Task result for artifact #%s (%s) already made",
artifact_id, av.backend)
return
if verdict is False:
log.warning("Task failed for artifact #%s (%s)", artifact_id,
av.backend)
av.status = JOB_STATUS_FAILED
else:
log.debug("Task for artifact #%s (%s) complete", artifact_id,
av.backend)
av.verdict = verdict
av.status = JOB_STATUS_DONE
s.add(av)
s.commit()
dispatch_event("verdict_update", artifact_id)
finally:
s.close()
def advance_vote_bounty(self):
block_number = self.cur_block
pending = len(self.is_voting)
if pending >= MAX_OUTSTANDING_VOTES:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.voted.is_(False)) \
.filter((block_number - 60) >= DbBounty.vote_before) \
.filter(DbBounty.truth_value.isnot(None)) \
.with_for_update()
for b in bounties:
log.warning("%s | %s | Expired vote (%s)", b.guid, block_number,
b.vote_before)
b.voted = True
s.add(b)
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.voted.is_(False)) \
.filter(block_number >= DbBounty.vote_after) \
.filter(DbBounty.truth_value.isnot(None)) \
.filter(block_number >= DbBounty.error_delay_block) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_VOTES - pending)
for b in bounties:
_add_event(events, self.is_voting, b.guid, "bounty_vote", b.guid,
b.truth_value, b.vote_before)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def retry_submissions(self):
"""
Conditions:
* Failure was temporary
* Bounty is not about to expire
"""
s = DbSession()
avs = [a.artifact_id for a in
s.query(DbArtifactVerdict.artifact_id) \
.filter_by(status=JOB_STATUS_NEW)]
s.close()
for a in avs:
dispatch_event("verdict_jobs", None, a)
def bounty_settle(self, guid):
"""Settle bounty for payout"""
log.info("%s | %s | Settle bounty", guid, self.cur_block)
failed = False
soft_fail = False
try:
self.polyswarm.settle_bounty(guid)
except PolySwarmNotFound:
# Record permanent failure
log.error("%s | Bounty no longer exists (double submit?)", guid)
failed = True
except PolySwarmError as e:
log.error("%s | Settle error: %s", guid, e)
failed = True
if "already been settled" not in str(e):
soft_fail = True
except IOError as e:
log.error("%s | Temporary settle error: %s", guid, e)
failed = True
soft_fail = True
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(
guid=guid).first()
if bounty and not bounty.settled:
if failed and soft_fail:
bounty.error_delay_block = self.cur_block + 5
bounty.error_retries += 1
if bounty.error_retries >= 3:
bounty.status = "aborted"
log.error(
"%s | %s | Aborted while settling, too many failures",
guid, self.cur_block)
else:
if failed:
bounty.status = "aborted"
else:
bounty.status = "finished"
bounty.settled = True
s.add(bounty)
s.commit()
s.close()
self.is_settling.discard(guid)
if not soft_fail:
dispatch_event("bounty_settled", guid)
def check_balance(self):
eth_side = 0 #int(self.polyswarm.balance("eth", chain="side"))
eth_home = int(self.polyswarm.balance("eth", chain="home"))
eth = (eth_side, eth_home)
if eth != self.eth_balance:
log.debug("[ETH] Balance: %s", val_readable(eth_home, "eth"))
self.eth_balance = eth
self.changed = True
nct_side = int(self.polyswarm.balance("nct", chain="side"))
nct_home = int(self.polyswarm.balance("nct", chain="home"))
nct = (nct_side, nct_home)
if nct != self.nct_balance:
log.debug("[NCT] Balance: %s / %s", val_readable(nct_side, "nct"),
val_readable(nct_home, "nct"))
self.nct_balance = nct
self.changed = True
dispatch_event("wallet_balance_info", nct, eth)
def advance_reveal(self):
block_number = self.cur_block
pending = len(self.is_revealing)
if pending >= MAX_OUTSTANDING_REVEALS:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.revealed.is_(False)) \
.filter(block_number >= DbBounty.reveal_block) \
.filter(DbBounty.assertions.is_(None)) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_REVEALS - pending)
for b in bounties:
_add_event(events, self.is_revealing, b.guid,
"bounty_assertions_reveal", b.guid, b.truth_value)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def action_artifact(analysis_backend, artifact_id):
if not isinstance(request.json, dict):
abort(400, "No JSON POST body found")
if "error" in request.json:
pass
if "verdict_value" not in request.json:
abort(400, "Missing verdict_value")
verdict_value = request.json["verdict_value"]
if verdict_value is not None:
verdict_value = int(verdict_value)
if verdict_value < 0 or verdict_value > 100:
abort(400, "Invalid verdict value")
s = DbSession()
verdict = s.query(DbArtifactVerdict) \
.with_for_update().filter(and_(
DbArtifactVerdict.backend == analysis_backend.name,
DbArtifactVerdict.artifact_id == artifact_id
)).first()
if not verdict:
s.close()
abort(404, "Artifact #%d not found" % artifact_id)
if verdict.status == JOB_STATUS_DONE:
s.close()
abort(403, "Verdict for artifact #%s already submitted" % artifact_id)
verdict.status = JOB_STATUS_DONE
verdict.verdict = verdict_value
s.add(verdict)
s.commit()
s.close()
app.logger.debug("Received verdict for artifact #%s from %s", artifact_id,
analysis_backend.name)
dispatch_event("verdict_update", artifact_id)
return jsonify({"status": "OK"})
def advance_settle(self):
block_number = self.cur_block
pending = len(self.is_settling)
if pending >= MAX_OUTSTANDING_SETTLES:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.assertions.isnot(None)) \
.filter(DbBounty.settled.is_(False)) \
.filter(block_number >= DbBounty.settle_block) \
.filter(block_number >= DbBounty.error_delay_block) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_SETTLES - pending)
for b in bounties:
_add_event(events, self.is_settling, b.guid, "bounty_settle",
b.guid)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def expire_pending(self):
"""Expire pending verdict tasks."""
# TODO: preferably replace "arbitrary" timeout window with backend
# polling
notify_tasks = set()
now = datetime.datetime.utcnow()
s = DbSession()
avs = s.query(DbArtifactVerdict).with_for_update() \
.filter_by(status=JOB_STATUS_PENDING) \
.filter(DbArtifactVerdict.expires < now)
for av in avs:
log.warning("Job %s expired", av.id)
av.status = JOB_STATUS_FAILED
# TODO: call backend.cancel_artifact
s.add(av)
notify_tasks.add(av.artifact_id)
s.commit()
s.close()
for aid in notify_tasks:
dispatch_event("verdict_update", aid)
def verdict_jobs(self, bounty_guid, artifact_id):
"""Jobs to submit or otherwise check"""
submit = []
# Find jobs we need to submit, and mark them
s = DbSession()
a = s.query(DbArtifact).filter_by(id=artifact_id).one()
artifact = Artifact(a.id, a.name, a.hash,
"%s/artifact/%s" % (self.url, a.id))
avs = s.query(DbArtifactVerdict).with_for_update() \
.filter_by(artifact_id=artifact.id, status=JOB_STATUS_NEW)
for av in avs.all():
submit.append((av.id, av.backend, artifact, av.meta))
av.status = JOB_STATUS_SUBMITTING
s.add(av)
s.commit()
s.close()
dispatch_event("verdict_job_submit", artifact_id, submit)
def verdict_update(self, artifact_id):
"""Recompute final verdict for an artifact and trigger bounty settle if
needed."""
log.debug("Artifact #%s updated", artifact_id)
dispatch_event("metrics_artifact_complete", 1)
s = DbSession()
artifact = s.query(DbArtifact).with_for_update().get(artifact_id)
if artifact.processed:
log.warning("Verdict for artifact #%s already made", artifact_id)
s.close()
return
verdicts = s.query(DbArtifactVerdict).filter_by(
artifact_id=artifact_id)
bounty_id = artifact.bounty_id
incomplete = False
verdict_map = {}
for verdict in verdicts.all():
if verdict.status > JOB_STATUS_DONE:
incomplete = True
verdict_map[verdict.backend] = verdict.verdict
if not incomplete:
log.debug("Verdict for artifact #%s can be made: %r", artifact_id,
verdict_map)
verdict = vote_on_artifact(verdict_map)
log.debug("Verdict for artifact #%s: %r", artifact_id, verdict)
artifact.processed = True
artifact.processed_at = datetime.datetime.utcnow()
artifact.processed_at_interval = interval(time.time(),
self.artifact_interval)
artifact.verdict = verdict
s.add(artifact)
s.commit()
dispatch_event("metrics_artifact_verdict", verdict)
else:
log.debug("Verdict for artifact #%s incomplete", artifact_id)
bounty_id = None
s.close()
if bounty_id is not None:
dispatch_event("bounty_artifact_verdict", bounty_id)
def bounty_artifact_verdict(self, bounty_id):
"""Check if bounty can be voted on after artifact update"""
s = DbSession()
bounty = s.query(DbBounty).with_for_update() \
.filter_by(id=bounty_id).one()
if bounty.truth_value is not None:
log.warning("%s | Bounty already has truth value, nothing to do",
bounty.guid)
s.close()
return
elif bounty.truth_manual:
# Already manual, so it won't make a difference
s.close()
return
# This may happen *after* the voting window, in that case mark
# the bounty as aborted
if self.cur_block and self.cur_block >= bounty.vote_before:
guid = None
if bounty.status != "aborted":
log.error(
"%s | Bounty artifact vote came in too late:"
" at block %s, voting ended on %s!", bounty.guid,
self.cur_block, bounty.vote_before)
bounty.status = "aborted"
guid = bounty.guid
s.add(bounty)
s.commit()
s.close()
if guid:
dispatch_event("bounty_aborted", guid)
return
# Collect votes
guid = bounty.guid
artifacts = s.query(DbArtifact).filter_by(bounty_id=bounty_id) \
.order_by(DbArtifact.id).all()
votes = []
record_value = True
can_vote = self.cur_block and self.cur_block >= bounty.vote_after
transition_manual = False
for artifact in artifacts:
if not artifact.processed:
log.debug("%s | Artifact #%s still has no vote", bounty.guid,
artifact.id)
record_value = can_vote = False
break
elif artifact.verdict is None:
log.debug("%s | Artifact #%s has DONTKNOW vote", bounty.guid,
artifact.id)
#can_vote = False
transition_manual = True
elif artifact.verdict >= VERDICT_MAYBE:
votes.append(True) # Malicious
else:
votes.append(False) # Safe
# Assertions can no longer come in before we've voted
## In case artifact votes came in after settle block
#if bounty.assertions and not transition_manual:
# transition_manual = self._bounty_assertions_disagree(bounty.guid, votes, bounty.assertions)
#vote_before = bounty.vote_before
if transition_manual:
log.debug("%s | Mark bounty as requiring manual vote", bounty.guid)
bounty.truth_manual = True
s.add(bounty)
s.commit()
elif record_value:
log.debug("%s | Recording vote: %s", bounty.guid, vote_show(votes))
bounty.truth_value = votes
s.add(bounty)
s.commit()
s.close()
#if can_vote and votes and guid not in self.is_voting:
# self.is_voting.add(guid)
# dispatch_event("bounty_vote", guid, votes, vote_before)
if transition_manual:
dispatch_event("bounty_manual", guid)
def bounty_with_manifest(self, bounty):
"""A bounty has become available: register it for processing"""
if bounty.get("resolved"):
return
#if self.first:
# self.first = False
#else:
# return
# Download related files
try:
manifest = ipfs_json(bounty["uri"], cache=False)
except IPFSNotFoundError:
# Shouldn't happen in production
return
except:
# TODO: track some well-known issues (e.g. disk full, ...)
# TODO: right now we'll skip these bounties!
log.warning("Couldn't fetch artifact data for bounty %s",
bounty["guid"])
return
# {hash, name}, both matter
manifest = manifest["result"]
if not manifest:
log.warning("Bounty %s has no artifacts", bounty["guid"])
return
# TODO Reintroduce explicit checking of artifact count.
# if len(manifest) != bounty["num_artifacts"]:
# log.warning(
# "Bounty %s has manifest with %s entries, but claims %s",
# bounty["guid"], len(manifest), bounty["num_artifacts"])
# return
num_artifacts = len(manifest)
expiration = int(bounty["expiration"])
s = DbSession()
b = DbBounty(guid=bounty["guid"],
author=bounty["author"],
amount=bounty["amount"],
num_artifacts=num_artifacts,
error_delay_block=0,
expiration_block=expiration,
vote_after=expiration + self.polyswarm.reveal_window + 1,
vote_before=expiration + self.polyswarm.vote_window,
reveal_block=expiration + self.polyswarm.vote_window +
self.polyswarm.reveal_window,
settle_block=expiration + self.polyswarm.vote_window +
self.polyswarm.reveal_window)
if self.manual_mode:
b.truth_manual = True
s.add(b)
try:
s.flush()
except IntegrityError:
# log.error("%s => %s", bounty["guid"], e)
# Bounty already exists, ignore
log.debug("Bounty %s already exists", bounty["guid"])
s.close()
return
log.info(
"%s | New bounty | artifacts=%s expiration=%s vote_before=%s settle=%s",
bounty["guid"], len(manifest), expiration, b.vote_before,
b.settle_block)
# Create jobs for every backend. If new backends join or backends are
# removed, tasks are *not* automatically updated.
job_ids = []
for artifact in manifest:
a = DbArtifact(bounty_id=b.id,
hash=artifact["hash"],
name=artifact["name"])
s.add(a)
s.flush()
job_ids.append(a.id)
jobs = []
# TODO: analysis_backends may change during different runs
for backend in analysis_backends.values():
v = DbArtifactVerdict(artifact_id=a.id,
backend=backend.name,
status=JOB_STATUS_NEW)
s.add(v)
jobs.append(v)
s.flush()
s.commit()
s.close()
artifacts = []
for i, artifact in enumerate(manifest):
# TODO: this is just the way their API works
# TODO: parallel download
artifacts.append(
gevent.spawn(ipfs_download, artifact["hash"],
"%s/%s" % (bounty["uri"], i)))
gevent.joinall(artifacts)
for a in artifacts:
if a.exception is not None:
# TODO: we should abort here
log.warning("Downloading artifacts for %s not succesful!",
bounty["guid"])
break
# Start submitting the artifacts
for job in job_ids:
dispatch_event("verdict_jobs", bounty["guid"], job)
def verdict_job_submit(self, artifact_id, jobs):
completed = 0
tasks = []
task_ids = {}
job_status = {}
exp = datetime.datetime.utcnow() + self.expires
failed = {
DbArtifactVerdict.status: JOB_STATUS_FAILED,
DbArtifactVerdict.meta: None,
DbArtifactVerdict.expires: None
}
try:
for av_id, backend, artifact, previous_task in jobs:
# Just in case a backend is removed
a = analysis_backends.get(backend)
if a:
log.debug("Submitting job #%s to %s", av_id, backend)
task = gevent.spawn(a.submit_artifact, av_id, artifact,
previous_task)
task_ids[id(task)] = av_id
tasks.append(task)
else:
log.warning("%r", backend)
# Collect results
gevent.joinall(tasks)
for task in tasks:
av_id = task_ids[id(task)]
if task.exception is not None:
job_status[av_id] = failed
elif isinstance(task.value, int):
job_status[av_id] = {
DbArtifactVerdict.status: JOB_STATUS_DONE,
DbArtifactVerdict.verdict: task.value,
DbArtifactVerdict.meta: None,
DbArtifactVerdict.expires: None
}
completed += 1
elif "verdict" in task.value:
verdict = task.value.pop("verdict")
job_status[av_id] = {
DbArtifactVerdict.status: JOB_STATUS_DONE,
DbArtifactVerdict.verdict: verdict,
DbArtifactVerdict.meta: task.value,
DbArtifactVerdict.expires: None
}
else:
job_status[av_id] = {
DbArtifactVerdict.status: JOB_STATUS_PENDING,
DbArtifactVerdict.meta: task.value,
DbArtifactVerdict.expires: exp
}
finally:
# Record results
s = DbSession()
reeval = False
for av_id, backend, artifact, previous_task in jobs:
fields = job_status.get(av_id, failed)
status = fields[DbArtifactVerdict.status]
log.debug("Recording job result #%s of %s (r=%s)", av_id,
backend, status)
# The submission process is subject to a race condition where
# we may receive the callback before all submissions are
# complete, so prevent incorrectly updating items.
s.query(DbArtifactVerdict) \
.filter_by(id=av_id, status=JOB_STATUS_SUBMITTING) \
.update(fields, synchronize_session=False)
if status <= JOB_STATUS_DONE:
reeval = True
s.commit()
s.close()
if reeval:
dispatch_event("verdict_update", artifact_id)
dispatch_event("metrics_jobs_submitted", len(jobs))
if completed:
dispatch_event("metrics_artifact_complete", completed)