def advance_vote_bounty(self):
block_number = self.cur_block
pending = len(self.is_voting)
if pending >= MAX_OUTSTANDING_VOTES:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.voted.is_(False)) \
.filter((block_number - 60) >= DbBounty.vote_before) \
.filter(DbBounty.truth_value.isnot(None)) \
.with_for_update()
for b in bounties:
log.warning("%s | %s | Expired vote (%s)", b.guid, block_number,
b.vote_before)
b.voted = True
s.add(b)
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.voted.is_(False)) \
.filter(block_number >= DbBounty.vote_after) \
.filter(DbBounty.truth_value.isnot(None)) \
.filter(block_number >= DbBounty.error_delay_block) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_VOTES - pending)
for b in bounties:
_add_event(events, self.is_voting, b.guid, "bounty_vote", b.guid,
b.truth_value, b.vote_before)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def __enter__(self):
s = DbSession()
g = str(uuid.uuid4())
b = DbBounty(guid=g,
expires=datetime.datetime.utcnow() +
datetime.timedelta(days=1),
num_artifacts=len(self.with_verdicts),
settle_block=1000)
s.add(b)
s.flush()
bid = b.id
a = DbArtifact(bounty_id=b.id, hash="Q121293810", name="dummy.exe")
s.add(a)
s.flush()
aid = a.id
jobs = []
for backend, attr in self.with_verdicts.items():
av = DbArtifactVerdict(artifact_id=aid,
status=JOB_STATUS_DONE,
backend=backend)
for k, v in attr.items():
setattr(av, k, v)
s.add(av)
s.flush()
jobs.append([av.id, backend, None])
s.commit()
s.close()
return {"bid": bid, "guid": g, "artifact_id": aid, "jobs": jobs}
def dashboard_manual_verdict(guid):
"""Set manual verdict for a bounty"""
verdicts = request.json["verdicts"]
app.logger.info("Received verdicts for bounty %s: %s", guid, verdicts)
if not isinstance(verdicts, list):
abort(400, "Verdicts is not a list")
for verdict in verdicts:
if verdict is not True and verdict is not False:
abort(400, "Invalid verdict value")
s = DbSession()
try:
b = s.query(DbBounty).with_for_update().filter_by(guid=guid).one()
if not b.truth_manual:
abort(403, "Bounty not in manual mode")
if len(verdicts) != b.num_artifacts:
abort(400, "This bounty requires %s verdicts" % b.num_artifacts)
if b.voted:
abort(403, "Bounty vote already submitted")
if b.settled:
abort(403, "Bounty already settled")
b.truth_value = verdicts
s.add(b)
s.commit()
finally:
s.close()
return jsonify({"status": "OK"})
def verdict_update_async(self, artifact_verdict_id, verdict):
"""Internal polling has resulted in an artifact verdict."""
s = DbSession()
try:
av = s.query(DbArtifactVerdict).with_for_update() \
.get(artifact_verdict_id)
artifact_id = av.artifact_id
if av.status != JOB_STATUS_PENDING:
log.warning("Task result for artifact #%s (%s) already made",
artifact_id, av.backend)
return
if verdict is False:
log.warning("Task failed for artifact #%s (%s)", artifact_id,
av.backend)
av.status = JOB_STATUS_FAILED
else:
log.debug("Task for artifact #%s (%s) complete", artifact_id,
av.backend)
av.verdict = verdict
av.status = JOB_STATUS_DONE
s.add(av)
s.commit()
dispatch_event("verdict_update", artifact_id)
finally:
s.close()
def _create_bounty(guid,
truth_value=None,
settled=False,
n=0,
assertions=None):
s = DbSession()
b = DbBounty(guid=guid,
num_artifacts=n,
assertions=n,
expires=datetime.datetime.utcnow(),
truth_value=truth_value,
truth_settled=settled,
settle_block=1001)
s.add(b)
s.flush()
b_id = b.id
a_ids = []
for i in range(n):
a = DbArtifact(bounty_id=b_id, hash="%s" % i, name="%s.txt" % i)
s.add(a)
s.flush()
a_ids.append(a.id)
s.commit()
s.close()
return b_id, a_ids
def test_expire_verdicts(dispatch_event, db):
v = VerdictComponent(Parent())
verdicts.analysis_backends = {
u"cuckoo": AnalysisBackend(u"cuckoo", True, 1),
u"zer0m0n": AnalysisBackend(u"zer0m0n", False, 1),
}
expire_at = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
mixed = artifact({
u"cuckoo": {
"status": JOB_STATUS_PENDING,
"expires": expire_at
},
u"zer0m0n": {
"status": JOB_STATUS_DONE,
"expires": expire_at
}
})
with mixed:
v.expire_verdicts()
assert dispatch_event.called
s = DbSession()
try:
for av in s.query(DbArtifactVerdict):
if av.backend == "cuckoo":
assert av.status == JOB_STATUS_FAILED
else:
assert av.status == JOB_STATUS_DONE
finally:
s.close()
db_clear()
def _verdict_set(a_id, value):
s = DbSession()
try:
s.query(DbArtifact).filter_by(id=a_id) \
.update({DbArtifact.verdict: value})
s.commit()
finally:
s.close()
def dashboard_bounties_guid(guid):
s = DbSession()
b = s.query(DbBounty).filter_by(guid=guid) \
.order_by(DbBounty.id).one_or_none()
if b is None:
return jsonify({"error": "No such bounty"}), 404
data = _gather_bounty_data(b)
s.close()
return jsonify(data)
def dashboard_bounties_pending():
# Bounties that are being processed or need to be submitted
s = DbSession()
bs = s.query(DbBounty).filter_by(status="active") \
.filter(or_(DbBounty.truth_manual.is_(False),
DbBounty.truth_value.isnot(None))) \
.order_by(DbBounty.id)
bounties = [_gather_bounty_data(b, False) for b in bs]
s.close()
return jsonify(bounties)
def dashboard_bounties_manual():
"""All not-yet-set bounties that need a manual verdict"""
s = DbSession()
bs = s.query(DbBounty) \
.filter_by(truth_manual=True, voted=False) \
.filter(DbBounty.truth_value.is_(None)) \
.order_by(DbBounty.id)
bounties = [_gather_bounty_data(b, False) for b in bs]
s.close()
return jsonify(bounties)
def pending():
from arbiter.database import DbSession, DbArtifactVerdict
s = DbSession()
#for av in s.query(DbArtifactVerdict).filter(DbArtifactVerdict.verdict.is_(None)).all():
for av in s.query(DbArtifactVerdict).filter(
DbArtifactVerdict.status != 0).all():
status = JOB_STATUS_NAMES.get(av.status, av.status)
print("ID: %5s" % av.id, "AVID: %5s" % av.artifact_id,
"Backend: %-10s" % av.backend, "S: %-10s" % status, "EXP:",
av.expires)
def flush_expired_manual(self):
block = self.cur_block
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active", settled=False, truth_manual=True, voted=False) \
.filter(block > DbBounty.vote_before).with_for_update()
for b in bounties:
log.warning("%s | %s | Expired manual voting (%s)", b.guid, block,
b.vote_before)
b.voted = True
s.add(b)
s.commit()
s.close()
def retry_submissions(self):
"""
Conditions:
* Failure was temporary
* Bounty is not about to expire
"""
s = DbSession()
avs = [a.artifact_id for a in
s.query(DbArtifactVerdict.artifact_id) \
.filter_by(status=JOB_STATUS_NEW)]
s.close()
for a in avs:
dispatch_event("verdict_jobs", None, a)
def reset_pending_jobs():
"""
Reset jobs that were pending submission. This may result in jobs being
submitted twice when the arbiter is restarted.
"""
log.debug("Reset pending jobs")
# TODO: this is incompatible with a multi-process approach
s = DbSession()
s.query(DbArtifactVerdict) \
.filter_by(status=JOB_STATUS_PENDING) \
.update({DbArtifactVerdict.status: JOB_STATUS_NEW}, synchronize_session=False)
s.commit()
s.close()
def bounty_vote(self, guid, value, vote_before):
"""Propagate bounty vote value to PolySwarm"""
if not value:
log.error("%s | Bad bounty_vote call %r", guid, value)
self.is_voting.discard(guid)
return
log.info("%s | %s | Vote on bounty: %s", guid, self.cur_block,
vote_show(value))
soft_fail = False
try:
if self.cur_block <= vote_before:
self.polyswarm.vote_bounty(guid, value)
else:
log.error("%s | %s | Permanent voting error: expired!",
self.cur_block, guid)
except PolySwarmError as e:
if e.status >= 500 and self.cur_block < vote_before:
log.error("%s | Temporary voting error: %s", guid, e.message
or e.reason)
# Server booboo, so try again later
soft_fail = True
else:
log.error("%s | Permanent voting error: %s", guid, e.message
or e.reason)
# Side-effect: we won't retry
except IOError as e:
log.error("%s | Temporary voting error: %s", guid, e)
soft_fail = True
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).one()
if not bounty.voted:
bounty.voted = True
if soft_fail:
# TODO: WS event
bounty.error_delay_block = self.cur_block + 5
bounty.error_retries += 1
if bounty.error_retries >= 3:
bounty.status = "aborted"
log.error(
"%s | %s | Aborted while voting, too many failures",
guid, self.cur_block)
s.add(bounty)
else:
log.warning("%s | %s | WARNING: double vote", guid, self.cur_block)
s.commit()
s.close()
self.is_voting.discard(guid)
def list_artifacts(analysis_backend):
s = DbSession()
artifacts = []
q = s.query(DbArtifact).filter(~exists().where(
and_(DbArtifactVerdict.artifact_id == DbArtifact.id,
DbArtifactVerdict.backend == analysis_backend.name)))
for artifact in q:
artifacts.append(artifact.id)
s.close()
return jsonify(artifacts)
def bounty_settle(self, guid):
"""Settle bounty for payout"""
log.info("%s | %s | Settle bounty", guid, self.cur_block)
failed = False
soft_fail = False
try:
self.polyswarm.settle_bounty(guid)
except PolySwarmNotFound:
# Record permanent failure
log.error("%s | Bounty no longer exists (double submit?)", guid)
failed = True
except PolySwarmError as e:
log.error("%s | Settle error: %s", guid, e)
failed = True
if "already been settled" not in str(e):
soft_fail = True
except IOError as e:
log.error("%s | Temporary settle error: %s", guid, e)
failed = True
soft_fail = True
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(
guid=guid).first()
if bounty and not bounty.settled:
if failed and soft_fail:
bounty.error_delay_block = self.cur_block + 5
bounty.error_retries += 1
if bounty.error_retries >= 3:
bounty.status = "aborted"
log.error(
"%s | %s | Aborted while settling, too many failures",
guid, self.cur_block)
else:
if failed:
bounty.status = "aborted"
else:
bounty.status = "finished"
bounty.settled = True
s.add(bounty)
s.commit()
s.close()
self.is_settling.discard(guid)
if not soft_fail:
dispatch_event("bounty_settled", guid)
def artifact_datapoints():
unow = datetime.datetime.utcnow()
ustart = unow - datetime.timedelta(days=5)
start = int(time.mktime(ustart.timetuple()))
now = int(time.mktime(unow.timetuple()))
data = []
s = DbSession()
try:
step_time = app.component.artifact_interval
rs = s.query(DbArtifact.processed_at_interval, func.count(1)) \
.filter(DbArtifact.processed_at_interval.isnot(None)) \
.filter(DbArtifact.processed_at_interval > start) \
.group_by(DbArtifact.processed_at_interval) \
.order_by(DbArtifact.processed_at_interval)
prev = None
for r in rs:
stamp = r.processed_at_interval
for step in missing_time_steps(prev, stamp, step_time):
data.append([step, 0])
data.append([stamp, r[1]])
prev = stamp
if prev:
if (now - prev) > step_time:
# We've stopped seeing entries, so end with 0
data.append([prev + step_time, 0])
data.append([now, 0])
#elif data[-1][0] > now:
# # Extrapolate the last entry to prevent a "dip"
# last = data[-1][0]
# boost = (step_time - (last - now)) / float(step_time)
# data[-1][1] = data[-1][1] / boost
finally:
s.close()
if len(data) == 1:
data.insert(0, [data[0][0] - step_time, 0])
data.sort(key=lambda v: tuple(v))
return jsonify({
"start": data[0][0] if data else start,
"end": data[-1][0] if data else now,
"data": data,
})
def verdict_update(self, artifact_id):
"""Recompute final verdict for an artifact and trigger bounty settle if
needed."""
log.debug("Artifact #%s updated", artifact_id)
dispatch_event("metrics_artifact_complete", 1)
s = DbSession()
artifact = s.query(DbArtifact).with_for_update().get(artifact_id)
if artifact.processed:
log.warning("Verdict for artifact #%s already made", artifact_id)
s.close()
return
verdicts = s.query(DbArtifactVerdict).filter_by(
artifact_id=artifact_id)
bounty_id = artifact.bounty_id
incomplete = False
verdict_map = {}
for verdict in verdicts.all():
if verdict.status > JOB_STATUS_DONE:
incomplete = True
verdict_map[verdict.backend] = verdict.verdict
if not incomplete:
log.debug("Verdict for artifact #%s can be made: %r", artifact_id,
verdict_map)
verdict = vote_on_artifact(verdict_map)
log.debug("Verdict for artifact #%s: %r", artifact_id, verdict)
artifact.processed = True
artifact.processed_at = datetime.datetime.utcnow()
artifact.processed_at_interval = interval(time.time(),
self.artifact_interval)
artifact.verdict = verdict
s.add(artifact)
s.commit()
dispatch_event("metrics_artifact_verdict", verdict)
else:
log.debug("Verdict for artifact #%s incomplete", artifact_id)
bounty_id = None
s.close()
if bounty_id is not None:
dispatch_event("bounty_artifact_verdict", bounty_id)
def advance_reveal(self):
block_number = self.cur_block
pending = len(self.is_revealing)
if pending >= MAX_OUTSTANDING_REVEALS:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.revealed.is_(False)) \
.filter(block_number >= DbBounty.reveal_block) \
.filter(DbBounty.assertions.is_(None)) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_REVEALS - pending)
for b in bounties:
_add_event(events, self.is_revealing, b.guid,
"bounty_assertions_reveal", b.guid, b.truth_value)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def counters(self):
# TODO: update on trigger
counters = []
s = DbSession()
try:
c = s.query(DbBounty.id).filter_by(settled=True).count()
counters.append(("counter-bounties-settled", c))
c = s.query(DbArtifact.id).filter_by(processed=False).count()
counters.append(("counter-artifacts-processing", c))
counters.append(
("counter-backends-running", len(analysis_backends)))
counters.append(("counter-errors", 0))
finally:
s.close()
for k, c in counters:
broadcast(k, c)
def action_artifact(analysis_backend, artifact_id):
if not isinstance(request.json, dict):
abort(400, "No JSON POST body found")
if "error" in request.json:
pass
if "verdict_value" not in request.json:
abort(400, "Missing verdict_value")
verdict_value = request.json["verdict_value"]
if verdict_value is not None:
verdict_value = int(verdict_value)
if verdict_value < 0 or verdict_value > 100:
abort(400, "Invalid verdict value")
s = DbSession()
verdict = s.query(DbArtifactVerdict) \
.with_for_update().filter(and_(
DbArtifactVerdict.backend == analysis_backend.name,
DbArtifactVerdict.artifact_id == artifact_id
)).first()
if not verdict:
s.close()
abort(404, "Artifact #%d not found" % artifact_id)
if verdict.status == JOB_STATUS_DONE:
s.close()
abort(403, "Verdict for artifact #%s already submitted" % artifact_id)
verdict.status = JOB_STATUS_DONE
verdict.verdict = verdict_value
s.add(verdict)
s.commit()
s.close()
app.logger.debug("Received verdict for artifact #%s from %s", artifact_id,
analysis_backend.name)
dispatch_event("verdict_update", artifact_id)
return jsonify({"status": "OK"})
def expire_pending(self):
"""Expire pending verdict tasks."""
# TODO: preferably replace "arbitrary" timeout window with backend
# polling
notify_tasks = set()
now = datetime.datetime.utcnow()
s = DbSession()
avs = s.query(DbArtifactVerdict).with_for_update() \
.filter_by(status=JOB_STATUS_PENDING) \
.filter(DbArtifactVerdict.expires < now)
for av in avs:
log.warning("Job %s expired", av.id)
av.status = JOB_STATUS_FAILED
# TODO: call backend.cancel_artifact
s.add(av)
notify_tasks.add(av.artifact_id)
s.commit()
s.close()
for aid in notify_tasks:
dispatch_event("verdict_update", aid)
def advance_settle(self):
block_number = self.cur_block
pending = len(self.is_settling)
if pending >= MAX_OUTSTANDING_SETTLES:
return
events = []
s = DbSession()
bounties = s.query(DbBounty).filter_by(status="active") \
.filter(DbBounty.assertions.isnot(None)) \
.filter(DbBounty.settled.is_(False)) \
.filter(block_number >= DbBounty.settle_block) \
.filter(block_number >= DbBounty.error_delay_block) \
.order_by(DbBounty.id).limit(MAX_OUTSTANDING_SETTLES - pending)
for b in bounties:
_add_event(events, self.is_settling, b.guid, "bounty_settle",
b.guid)
s.commit()
s.close()
for e, args in events:
dispatch_event(e, *args)
def bounty_assertions_reveal(self, guid, value):
"""Reveal assertions.
We should have already voted."""
log.debug("%s | Checking assertions", guid)
experts_disagree = False
assertions = []
try:
assertions = self.polyswarm.bounty_assertions(guid)
if value:
experts_disagree = self._bounty_assertions_disagree(
guid, value, assertions)
except PolySwarmNotFound:
pass
except PolySwarmError as e:
log.error("%s | Assertion fetch error: %s", guid, e)
except Exception as e:
# We ignore this for now
log.error("Failed to check assertions: %s", e)
if assertions:
log.debug("%s | %s assertion(s)", guid, len(assertions))
s = DbSession()
bounty = s.query(DbBounty).with_for_update() \
.filter_by(guid=guid).one()
bounty.revealed = True
bounty.assertions = assertions
if experts_disagree and not bounty.settled:
# Mark as manual so we don't auto-settle
#log.warning("%s | Set to manual", guid)
#bounty.truth_manual = True
pass
#settle_block = bounty.settle_block
s.add(bounty)
s.commit()
s.close()
self.is_revealing.discard(guid)
def verdict_jobs(self, bounty_guid, artifact_id):
"""Jobs to submit or otherwise check"""
submit = []
# Find jobs we need to submit, and mark them
s = DbSession()
a = s.query(DbArtifact).filter_by(id=artifact_id).one()
artifact = Artifact(a.id, a.name, a.hash,
"%s/artifact/%s" % (self.url, a.id))
avs = s.query(DbArtifactVerdict).with_for_update() \
.filter_by(artifact_id=artifact.id, status=JOB_STATUS_NEW)
for av in avs.all():
submit.append((av.id, av.backend, artifact, av.meta))
av.status = JOB_STATUS_SUBMITTING
s.add(av)
s.commit()
s.close()
dispatch_event("verdict_job_submit", artifact_id, submit)
def bounties():
from arbiter.database import DbSession, DbBounty
print("Status".ljust(8), "GUID".ljust(36), "MRVS", "<Vote", ">Settle",
"Value")
S = {True: "*", False: " "}
s = DbSession()
for b in s.query(DbBounty).order_by(DbBounty.id).all():
if not b.truth_value:
value = "-"
else:
value = "".join(str(v)[:1] for v in b.truth_value)
print(
b.status.ljust(8),
b.guid,
S[b.truth_manual] + S[b.revealed] + S[b.voted] + S[b.settled],
str(b.vote_before).ljust(5),
str(b.settle_block).ljust(7),
value.ljust(5),
)
def _bounty_check_state(b, should_be_set, should_be_settled=None):
s = DbSession()
try:
if isinstance(b, int):
kwargs = {"id": b}
else:
kwargs = {"guid": b}
b = s.query(DbBounty).filter_by(**kwargs).one()
is_set = b.truth_value is not None
is_settled = b.truth_settled is True
finally:
s.close()
if not should_be_set and is_set:
raise ValueError("Bounty %s was set, when it should not be" % b)
elif should_be_set and not is_set:
raise ValueError("Bounty %s was not set, when it should be" % b)
if should_be_settled is None:
return
if not should_be_settled and is_settled:
raise ValueError("Bounty %s was settled, when it should not be" % b)
elif should_be_settled and not is_settled:
raise ValueError("Bounty %s was not settled, when it should be" % b)
def bounty_settle_manual(guid, votes):
s = DbSession()
bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).first()
if not bounty:
s.close()
raise KeyError("No such bounty")
elif bounty.voted or bounty.settled:
s.close()
raise ValueError("Bounty was already voted on/settled")
need = s.query(DbArtifact).filter_by(bounty_id=bounty.id).count()
if need != len(votes):
s.close()
raise ValueError("Need %s vote(s), not %s" % (need, len(votes)))
log.info("Manually set bounty %s vote to %s", guid, votes)
bounty.truth_value = votes
bounty.truth_manual = True
s.add(bounty)
s.commit()
s.close()
def test_verdict_job_submit(dispatch_event, db):
v = VerdictComponent(Parent())
verdicts.analysis_backends = {
u"cuckoo": AnalysisBackend(u"cuckoo", True, 1),
u"zer0m0n": AnalysisBackend(u"zer0m0n", False, 1),
u"cuckscan": AnalysisBackend(u"cuckscan", False, 1),
}
verdicts.analysis_backends[u"cuckoo"].submit_artifact = lambda a: None
verdicts.analysis_backends[u"zer0m0n"].submit_artifact = lambda a: {}
verdicts.analysis_backends[u"cuckscan"].submit_artifact = lambda a: 1
pending = artifact({
u"cuckoo": {
"status": JOB_STATUS_SUBMITTING
},
u"zer0m0n": {
"status": JOB_STATUS_SUBMITTING
},
u"cuckscan": {
"status": JOB_STATUS_SUBMITTING
}
})
with pending as x:
v.verdict_job_submit(x["artifact_id"], x["jobs"])
assert dispatch_event.called
s = DbSession()
try:
for av in s.query(DbArtifactVerdict):
if av.backend == "cuckscan":
assert av.status == JOB_STATUS_DONE
else:
assert av.status == JOB_STATUS_PENDING
finally:
s.close()
db_clear()