Exemple #1
0
 def advance_vote_bounty(self):
     block_number = self.cur_block
     pending = len(self.is_voting)
     if pending >= MAX_OUTSTANDING_VOTES:
         return
     events = []
     s = DbSession()
     bounties = s.query(DbBounty).filter_by(status="active") \
         .filter(DbBounty.voted.is_(False)) \
         .filter((block_number - 60) >= DbBounty.vote_before) \
         .filter(DbBounty.truth_value.isnot(None)) \
         .with_for_update()
     for b in bounties:
         log.warning("%s | %s | Expired vote (%s)", b.guid, block_number,
                     b.vote_before)
         b.voted = True
         s.add(b)
     bounties = s.query(DbBounty).filter_by(status="active") \
         .filter(DbBounty.voted.is_(False)) \
         .filter(block_number >= DbBounty.vote_after) \
         .filter(DbBounty.truth_value.isnot(None)) \
         .filter(block_number >= DbBounty.error_delay_block) \
         .order_by(DbBounty.id).limit(MAX_OUTSTANDING_VOTES - pending)
     for b in bounties:
         _add_event(events, self.is_voting, b.guid, "bounty_vote", b.guid,
                    b.truth_value, b.vote_before)
     s.commit()
     s.close()
     for e, args in events:
         dispatch_event(e, *args)
def test_expire_verdicts(dispatch_event, db):
    v = VerdictComponent(Parent())
    verdicts.analysis_backends = {
        u"cuckoo": AnalysisBackend(u"cuckoo", True, 1),
        u"zer0m0n": AnalysisBackend(u"zer0m0n", False, 1),
    }

    expire_at = datetime.datetime.utcnow() - datetime.timedelta(hours=1)

    mixed = artifact({
        u"cuckoo": {
            "status": JOB_STATUS_PENDING,
            "expires": expire_at
        },
        u"zer0m0n": {
            "status": JOB_STATUS_DONE,
            "expires": expire_at
        }
    })
    with mixed:
        v.expire_verdicts()
        assert dispatch_event.called

        s = DbSession()
        try:
            for av in s.query(DbArtifactVerdict):
                if av.backend == "cuckoo":
                    assert av.status == JOB_STATUS_FAILED
                else:
                    assert av.status == JOB_STATUS_DONE
        finally:
            s.close()
    db_clear()
Exemple #3
0
def dashboard_bounties_guid(guid):
    s = DbSession()
    b = s.query(DbBounty).filter_by(guid=guid) \
        .order_by(DbBounty.id).one_or_none()
    if b is None:
        return jsonify({"error": "No such bounty"}), 404
    data = _gather_bounty_data(b)
    s.close()
    return jsonify(data)
Exemple #4
0
def dashboard_bounties_manual():
    """All not-yet-set bounties that need a manual verdict"""
    s = DbSession()
    bs = s.query(DbBounty) \
        .filter_by(truth_manual=True, voted=False) \
        .filter(DbBounty.truth_value.is_(None)) \
        .order_by(DbBounty.id)
    bounties = [_gather_bounty_data(b, False) for b in bs]
    s.close()
    return jsonify(bounties)
Exemple #5
0
def dashboard_bounties_pending():
    # Bounties that are being processed or need to be submitted
    s = DbSession()
    bs = s.query(DbBounty).filter_by(status="active") \
        .filter(or_(DbBounty.truth_manual.is_(False),
                    DbBounty.truth_value.isnot(None))) \
        .order_by(DbBounty.id)
    bounties = [_gather_bounty_data(b, False) for b in bs]
    s.close()
    return jsonify(bounties)
Exemple #6
0
 def verdict_update_async(self, artifact_verdict_id, verdict):
     """Internal polling has resulted in an artifact verdict."""
     s = DbSession()
     try:
         av = s.query(DbArtifactVerdict).with_for_update() \
             .get(artifact_verdict_id)
         artifact_id = av.artifact_id
         if av.status != JOB_STATUS_PENDING:
             log.warning("Task result for artifact #%s (%s) already made",
                         artifact_id, av.backend)
             return
         if verdict is False:
             log.warning("Task failed for artifact #%s (%s)", artifact_id,
                         av.backend)
             av.status = JOB_STATUS_FAILED
         else:
             log.debug("Task for artifact #%s (%s) complete", artifact_id,
                       av.backend)
             av.verdict = verdict
             av.status = JOB_STATUS_DONE
         s.add(av)
         s.commit()
         dispatch_event("verdict_update", artifact_id)
     finally:
         s.close()
Exemple #7
0
def dashboard_manual_verdict(guid):
    """Set manual verdict for a bounty"""
    verdicts = request.json["verdicts"]
    app.logger.info("Received verdicts for bounty %s: %s", guid, verdicts)
    if not isinstance(verdicts, list):
        abort(400, "Verdicts is not a list")
    for verdict in verdicts:
        if verdict is not True and verdict is not False:
            abort(400, "Invalid verdict value")

    s = DbSession()
    try:
        b = s.query(DbBounty).with_for_update().filter_by(guid=guid).one()
        if not b.truth_manual:
            abort(403, "Bounty not in manual mode")
        if len(verdicts) != b.num_artifacts:
            abort(400, "This bounty requires %s verdicts" % b.num_artifacts)
        if b.voted:
            abort(403, "Bounty vote already submitted")
        if b.settled:
            abort(403, "Bounty already settled")
        b.truth_value = verdicts
        s.add(b)
        s.commit()
    finally:
        s.close()

    return jsonify({"status": "OK"})
Exemple #8
0
def pending():
    from arbiter.database import DbSession, DbArtifactVerdict

    s = DbSession()
    #for av in s.query(DbArtifactVerdict).filter(DbArtifactVerdict.verdict.is_(None)).all():
    for av in s.query(DbArtifactVerdict).filter(
            DbArtifactVerdict.status != 0).all():
        status = JOB_STATUS_NAMES.get(av.status, av.status)
        print("ID: %5s" % av.id, "AVID: %5s" % av.artifact_id,
              "Backend: %-10s" % av.backend, "S: %-10s" % status, "EXP:",
              av.expires)
Exemple #9
0
 def retry_submissions(self):
     """
     Conditions:
     * Failure was temporary
     * Bounty is not about to expire
     """
     s = DbSession()
     avs = [a.artifact_id for a in
            s.query(DbArtifactVerdict.artifact_id) \
            .filter_by(status=JOB_STATUS_NEW)]
     s.close()
     for a in avs:
         dispatch_event("verdict_jobs", None, a)
Exemple #10
0
def list_artifacts(analysis_backend):
    s = DbSession()
    artifacts = []

    q = s.query(DbArtifact).filter(~exists().where(
        and_(DbArtifactVerdict.artifact_id == DbArtifact.id,
             DbArtifactVerdict.backend == analysis_backend.name)))

    for artifact in q:
        artifacts.append(artifact.id)

    s.close()

    return jsonify(artifacts)
Exemple #11
0
def _verdict_set(a_id, value):
    s = DbSession()
    try:
        s.query(DbArtifact).filter_by(id=a_id) \
            .update({DbArtifact.verdict: value})
        s.commit()
    finally:
        s.close()
Exemple #12
0
 def flush_expired_manual(self):
     block = self.cur_block
     s = DbSession()
     bounties = s.query(DbBounty).filter_by(status="active", settled=False, truth_manual=True, voted=False) \
         .filter(block > DbBounty.vote_before).with_for_update()
     for b in bounties:
         log.warning("%s | %s | Expired manual voting (%s)", b.guid, block,
                     b.vote_before)
         b.voted = True
         s.add(b)
     s.commit()
     s.close()
Exemple #13
0
def artifact_datapoints():
    unow = datetime.datetime.utcnow()
    ustart = unow - datetime.timedelta(days=5)
    start = int(time.mktime(ustart.timetuple()))
    now = int(time.mktime(unow.timetuple()))
    data = []

    s = DbSession()
    try:
        step_time = app.component.artifact_interval
        rs = s.query(DbArtifact.processed_at_interval, func.count(1)) \
            .filter(DbArtifact.processed_at_interval.isnot(None)) \
            .filter(DbArtifact.processed_at_interval > start) \
            .group_by(DbArtifact.processed_at_interval) \
            .order_by(DbArtifact.processed_at_interval)

        prev = None
        for r in rs:
            stamp = r.processed_at_interval
            for step in missing_time_steps(prev, stamp, step_time):
                data.append([step, 0])
            data.append([stamp, r[1]])
            prev = stamp

        if prev:
            if (now - prev) > step_time:
                # We've stopped seeing entries, so end with 0
                data.append([prev + step_time, 0])
                data.append([now, 0])
            #elif data[-1][0] > now:
            #    # Extrapolate the last entry to prevent a "dip"
            #    last = data[-1][0]
            #    boost = (step_time - (last - now)) / float(step_time)
            #    data[-1][1] = data[-1][1] / boost

    finally:
        s.close()

    if len(data) == 1:
        data.insert(0, [data[0][0] - step_time, 0])
    data.sort(key=lambda v: tuple(v))

    return jsonify({
        "start": data[0][0] if data else start,
        "end": data[-1][0] if data else now,
        "data": data,
    })
Exemple #14
0
    def bounty_vote(self, guid, value, vote_before):
        """Propagate bounty vote value to PolySwarm"""
        if not value:
            log.error("%s | Bad bounty_vote call %r", guid, value)
            self.is_voting.discard(guid)
            return

        log.info("%s | %s | Vote on bounty: %s", guid, self.cur_block,
                 vote_show(value))
        soft_fail = False
        try:
            if self.cur_block <= vote_before:
                self.polyswarm.vote_bounty(guid, value)
            else:
                log.error("%s | %s | Permanent voting error: expired!",
                          self.cur_block, guid)
        except PolySwarmError as e:
            if e.status >= 500 and self.cur_block < vote_before:
                log.error("%s | Temporary voting error: %s", guid, e.message
                          or e.reason)
                # Server booboo, so try again later
                soft_fail = True
            else:
                log.error("%s | Permanent voting error: %s", guid, e.message
                          or e.reason)
                # Side-effect: we won't retry
        except IOError as e:
            log.error("%s | Temporary voting error: %s", guid, e)
            soft_fail = True

        s = DbSession()
        bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).one()
        if not bounty.voted:
            bounty.voted = True
            if soft_fail:
                # TODO: WS event
                bounty.error_delay_block = self.cur_block + 5
                bounty.error_retries += 1
                if bounty.error_retries >= 3:
                    bounty.status = "aborted"
                    log.error(
                        "%s | %s | Aborted while voting, too many failures",
                        guid, self.cur_block)
            s.add(bounty)
        else:
            log.warning("%s | %s | WARNING: double vote", guid, self.cur_block)
        s.commit()
        s.close()

        self.is_voting.discard(guid)
Exemple #15
0
    def bounty_settle(self, guid):
        """Settle bounty for payout"""

        log.info("%s | %s | Settle bounty", guid, self.cur_block)
        failed = False
        soft_fail = False
        try:
            self.polyswarm.settle_bounty(guid)
        except PolySwarmNotFound:
            # Record permanent failure
            log.error("%s | Bounty no longer exists (double submit?)", guid)
            failed = True
        except PolySwarmError as e:
            log.error("%s | Settle error: %s", guid, e)
            failed = True
            if "already been settled" not in str(e):
                soft_fail = True
        except IOError as e:
            log.error("%s | Temporary settle error: %s", guid, e)
            failed = True
            soft_fail = True

        s = DbSession()
        bounty = s.query(DbBounty).with_for_update().filter_by(
            guid=guid).first()
        if bounty and not bounty.settled:
            if failed and soft_fail:
                bounty.error_delay_block = self.cur_block + 5
                bounty.error_retries += 1
                if bounty.error_retries >= 3:
                    bounty.status = "aborted"
                    log.error(
                        "%s | %s | Aborted while settling, too many failures",
                        guid, self.cur_block)
            else:
                if failed:
                    bounty.status = "aborted"
                else:
                    bounty.status = "finished"
                bounty.settled = True
            s.add(bounty)
        s.commit()
        s.close()

        self.is_settling.discard(guid)
        if not soft_fail:
            dispatch_event("bounty_settled", guid)
Exemple #16
0
    def verdict_jobs(self, bounty_guid, artifact_id):
        """Jobs to submit or otherwise check"""
        submit = []

        # Find jobs we need to submit, and mark them
        s = DbSession()
        a = s.query(DbArtifact).filter_by(id=artifact_id).one()
        artifact = Artifact(a.id, a.name, a.hash,
                            "%s/artifact/%s" % (self.url, a.id))
        avs = s.query(DbArtifactVerdict).with_for_update() \
            .filter_by(artifact_id=artifact.id, status=JOB_STATUS_NEW)

        for av in avs.all():
            submit.append((av.id, av.backend, artifact, av.meta))
            av.status = JOB_STATUS_SUBMITTING
            s.add(av)

        s.commit()
        s.close()

        dispatch_event("verdict_job_submit", artifact_id, submit)
Exemple #17
0
def reset_pending_jobs():
    """
    Reset jobs that were pending submission.  This may result in jobs being
    submitted twice when the arbiter is restarted.
    """
    log.debug("Reset pending jobs")
    # TODO: this is incompatible with a multi-process approach

    s = DbSession()
    s.query(DbArtifactVerdict) \
        .filter_by(status=JOB_STATUS_PENDING) \
        .update({DbArtifactVerdict.status: JOB_STATUS_NEW}, synchronize_session=False)
    s.commit()
    s.close()
Exemple #18
0
def bounties():
    from arbiter.database import DbSession, DbBounty

    print("Status".ljust(8), "GUID".ljust(36), "MRVS", "<Vote", ">Settle",
          "Value")

    S = {True: "*", False: " "}

    s = DbSession()
    for b in s.query(DbBounty).order_by(DbBounty.id).all():
        if not b.truth_value:
            value = "-"
        else:
            value = "".join(str(v)[:1] for v in b.truth_value)
        print(
            b.status.ljust(8),
            b.guid,
            S[b.truth_manual] + S[b.revealed] + S[b.voted] + S[b.settled],
            str(b.vote_before).ljust(5),
            str(b.settle_block).ljust(7),
            value.ljust(5),
        )
Exemple #19
0
def _bounty_check_state(b, should_be_set, should_be_settled=None):
    s = DbSession()
    try:
        if isinstance(b, int):
            kwargs = {"id": b}
        else:
            kwargs = {"guid": b}
        b = s.query(DbBounty).filter_by(**kwargs).one()
        is_set = b.truth_value is not None
        is_settled = b.truth_settled is True
    finally:
        s.close()
    if not should_be_set and is_set:
        raise ValueError("Bounty %s was set, when it should not be" % b)
    elif should_be_set and not is_set:
        raise ValueError("Bounty %s was not set, when it should be" % b)
    if should_be_settled is None:
        return
    if not should_be_settled and is_settled:
        raise ValueError("Bounty %s was settled, when it should not be" % b)
    elif should_be_settled and not is_settled:
        raise ValueError("Bounty %s was not settled, when it should be" % b)
Exemple #20
0
 def expire_pending(self):
     """Expire pending verdict tasks."""
     # TODO: preferably replace "arbitrary" timeout window with backend
     # polling
     notify_tasks = set()
     now = datetime.datetime.utcnow()
     s = DbSession()
     avs = s.query(DbArtifactVerdict).with_for_update() \
         .filter_by(status=JOB_STATUS_PENDING) \
         .filter(DbArtifactVerdict.expires < now)
     for av in avs:
         log.warning("Job %s expired", av.id)
         av.status = JOB_STATUS_FAILED
         # TODO: call backend.cancel_artifact
         s.add(av)
         notify_tasks.add(av.artifact_id)
     s.commit()
     s.close()
     for aid in notify_tasks:
         dispatch_event("verdict_update", aid)
def test_verdict_job_submit(dispatch_event, db):
    v = VerdictComponent(Parent())
    verdicts.analysis_backends = {
        u"cuckoo": AnalysisBackend(u"cuckoo", True, 1),
        u"zer0m0n": AnalysisBackend(u"zer0m0n", False, 1),
        u"cuckscan": AnalysisBackend(u"cuckscan", False, 1),
    }

    verdicts.analysis_backends[u"cuckoo"].submit_artifact = lambda a: None
    verdicts.analysis_backends[u"zer0m0n"].submit_artifact = lambda a: {}
    verdicts.analysis_backends[u"cuckscan"].submit_artifact = lambda a: 1

    pending = artifact({
        u"cuckoo": {
            "status": JOB_STATUS_SUBMITTING
        },
        u"zer0m0n": {
            "status": JOB_STATUS_SUBMITTING
        },
        u"cuckscan": {
            "status": JOB_STATUS_SUBMITTING
        }
    })
    with pending as x:
        v.verdict_job_submit(x["artifact_id"], x["jobs"])
        assert dispatch_event.called
        s = DbSession()
        try:
            for av in s.query(DbArtifactVerdict):
                if av.backend == "cuckscan":
                    assert av.status == JOB_STATUS_DONE
                else:
                    assert av.status == JOB_STATUS_PENDING
        finally:
            s.close()
    db_clear()
Exemple #22
0
    def bounty_assertions_reveal(self, guid, value):
        """Reveal assertions.
        We should have already voted."""
        log.debug("%s | Checking assertions", guid)

        experts_disagree = False
        assertions = []
        try:
            assertions = self.polyswarm.bounty_assertions(guid)
            if value:
                experts_disagree = self._bounty_assertions_disagree(
                    guid, value, assertions)
        except PolySwarmNotFound:
            pass
        except PolySwarmError as e:
            log.error("%s | Assertion fetch error: %s", guid, e)
        except Exception as e:
            # We ignore this for now
            log.error("Failed to check assertions: %s", e)

        if assertions:
            log.debug("%s | %s assertion(s)", guid, len(assertions))

        s = DbSession()
        bounty = s.query(DbBounty).with_for_update() \
            .filter_by(guid=guid).one()
        bounty.revealed = True
        bounty.assertions = assertions

        if experts_disagree and not bounty.settled:
            # Mark as manual so we don't auto-settle
            #log.warning("%s | Set to manual", guid)
            #bounty.truth_manual = True
            pass

        #settle_block = bounty.settle_block
        s.add(bounty)
        s.commit()
        s.close()

        self.is_revealing.discard(guid)
Exemple #23
0
 def advance_reveal(self):
     block_number = self.cur_block
     pending = len(self.is_revealing)
     if pending >= MAX_OUTSTANDING_REVEALS:
         return
     events = []
     s = DbSession()
     bounties = s.query(DbBounty).filter_by(status="active") \
         .filter(DbBounty.revealed.is_(False)) \
         .filter(block_number >= DbBounty.reveal_block) \
         .filter(DbBounty.assertions.is_(None)) \
         .order_by(DbBounty.id).limit(MAX_OUTSTANDING_REVEALS - pending)
     for b in bounties:
         _add_event(events, self.is_revealing, b.guid,
                    "bounty_assertions_reveal", b.guid, b.truth_value)
     s.commit()
     s.close()
     for e, args in events:
         dispatch_event(e, *args)
Exemple #24
0
 def advance_settle(self):
     block_number = self.cur_block
     pending = len(self.is_settling)
     if pending >= MAX_OUTSTANDING_SETTLES:
         return
     events = []
     s = DbSession()
     bounties = s.query(DbBounty).filter_by(status="active") \
         .filter(DbBounty.assertions.isnot(None)) \
         .filter(DbBounty.settled.is_(False)) \
         .filter(block_number >= DbBounty.settle_block) \
         .filter(block_number >= DbBounty.error_delay_block) \
         .order_by(DbBounty.id).limit(MAX_OUTSTANDING_SETTLES - pending)
     for b in bounties:
         _add_event(events, self.is_settling, b.guid, "bounty_settle",
                    b.guid)
     s.commit()
     s.close()
     for e, args in events:
         dispatch_event(e, *args)
Exemple #25
0
    def counters(self):
        # TODO: update on trigger
        counters = []
        s = DbSession()
        try:
            c = s.query(DbBounty.id).filter_by(settled=True).count()
            counters.append(("counter-bounties-settled", c))

            c = s.query(DbArtifact.id).filter_by(processed=False).count()
            counters.append(("counter-artifacts-processing", c))

            counters.append(
                ("counter-backends-running", len(analysis_backends)))
            counters.append(("counter-errors", 0))
        finally:
            s.close()

        for k, c in counters:
            broadcast(k, c)
Exemple #26
0
def _no_such_bounty(guid):
    s = DbSession()
    try:
        assert s.query(DbBounty).filter_by(guid=guid).count() == 0
    finally:
        s.close()
Exemple #27
0
def _create_bounty(guid,
                   truth_value=None,
                   settled=False,
                   n=0,
                   assertions=None):
    s = DbSession()
    b = DbBounty(guid=guid,
                 num_artifacts=n,
                 assertions=n,
                 expires=datetime.datetime.utcnow(),
                 truth_value=truth_value,
                 truth_settled=settled,
                 settle_block=1001)
    s.add(b)
    s.flush()
    b_id = b.id
    a_ids = []
    for i in range(n):
        a = DbArtifact(bounty_id=b_id, hash="%s" % i, name="%s.txt" % i)
        s.add(a)
        s.flush()
        a_ids.append(a.id)
    s.commit()
    s.close()
    return b_id, a_ids
Exemple #28
0
    def bounty_artifact_verdict(self, bounty_id):
        """Check if bounty can be voted on after artifact update"""

        s = DbSession()
        bounty = s.query(DbBounty).with_for_update() \
            .filter_by(id=bounty_id).one()
        if bounty.truth_value is not None:
            log.warning("%s | Bounty already has truth value, nothing to do",
                        bounty.guid)
            s.close()
            return
        elif bounty.truth_manual:
            # Already manual, so it won't make a difference
            s.close()
            return

        # This may happen *after* the voting window, in that case mark
        # the bounty as aborted
        if self.cur_block and self.cur_block >= bounty.vote_before:
            guid = None
            if bounty.status != "aborted":
                log.error(
                    "%s | Bounty artifact vote came in too late:"
                    " at block %s, voting ended on %s!", bounty.guid,
                    self.cur_block, bounty.vote_before)
                bounty.status = "aborted"
                guid = bounty.guid
                s.add(bounty)
                s.commit()
            s.close()
            if guid:
                dispatch_event("bounty_aborted", guid)
            return

        # Collect votes
        guid = bounty.guid
        artifacts = s.query(DbArtifact).filter_by(bounty_id=bounty_id) \
            .order_by(DbArtifact.id).all()
        votes = []
        record_value = True
        can_vote = self.cur_block and self.cur_block >= bounty.vote_after
        transition_manual = False
        for artifact in artifacts:
            if not artifact.processed:
                log.debug("%s | Artifact #%s still has no vote", bounty.guid,
                          artifact.id)
                record_value = can_vote = False
                break
            elif artifact.verdict is None:
                log.debug("%s | Artifact #%s has DONTKNOW vote", bounty.guid,
                          artifact.id)
                #can_vote = False
                transition_manual = True
            elif artifact.verdict >= VERDICT_MAYBE:
                votes.append(True)  # Malicious
            else:
                votes.append(False)  # Safe

        # Assertions can no longer come in before we've voted
        ## In case artifact votes came in after settle block
        #if bounty.assertions and not transition_manual:
        #    transition_manual = self._bounty_assertions_disagree(bounty.guid, votes, bounty.assertions)
        #vote_before = bounty.vote_before

        if transition_manual:
            log.debug("%s | Mark bounty as requiring manual vote", bounty.guid)
            bounty.truth_manual = True
            s.add(bounty)
            s.commit()
        elif record_value:
            log.debug("%s | Recording vote: %s", bounty.guid, vote_show(votes))
            bounty.truth_value = votes
            s.add(bounty)
            s.commit()
        s.close()
        #if can_vote and votes and guid not in self.is_voting:
        #    self.is_voting.add(guid)
        #    dispatch_event("bounty_vote", guid, votes, vote_before)
        if transition_manual:
            dispatch_event("bounty_manual", guid)
Exemple #29
0
    def bounty_with_manifest(self, bounty):
        """A bounty has become available: register it for processing"""

        if bounty.get("resolved"):
            return

        #if self.first:
        #    self.first = False
        #else:
        #    return

        # Download related files
        try:
            manifest = ipfs_json(bounty["uri"], cache=False)
        except IPFSNotFoundError:
            # Shouldn't happen in production
            return
        except:
            # TODO: track some well-known issues (e.g. disk full, ...)
            # TODO: right now we'll skip these bounties!
            log.warning("Couldn't fetch artifact data for bounty %s",
                        bounty["guid"])
            return

        # {hash, name}, both matter
        manifest = manifest["result"]
        if not manifest:
            log.warning("Bounty %s has no artifacts", bounty["guid"])
            return

        # TODO Reintroduce explicit checking of artifact count.
        # if len(manifest) != bounty["num_artifacts"]:
        #     log.warning(
        #         "Bounty %s has manifest with %s entries, but claims %s",
        #         bounty["guid"], len(manifest), bounty["num_artifacts"])
        #     return
        num_artifacts = len(manifest)
        expiration = int(bounty["expiration"])

        s = DbSession()
        b = DbBounty(guid=bounty["guid"],
                     author=bounty["author"],
                     amount=bounty["amount"],
                     num_artifacts=num_artifacts,
                     error_delay_block=0,
                     expiration_block=expiration,
                     vote_after=expiration + self.polyswarm.reveal_window + 1,
                     vote_before=expiration + self.polyswarm.vote_window,
                     reveal_block=expiration + self.polyswarm.vote_window +
                     self.polyswarm.reveal_window,
                     settle_block=expiration + self.polyswarm.vote_window +
                     self.polyswarm.reveal_window)

        if self.manual_mode:
            b.truth_manual = True
        s.add(b)
        try:
            s.flush()
        except IntegrityError:
            # log.error("%s => %s", bounty["guid"], e)
            # Bounty already exists, ignore
            log.debug("Bounty %s already exists", bounty["guid"])
            s.close()
            return

        log.info(
            "%s | New bounty | artifacts=%s expiration=%s vote_before=%s settle=%s",
            bounty["guid"], len(manifest), expiration, b.vote_before,
            b.settle_block)

        # Create jobs for every backend.  If new backends join or backends are
        # removed, tasks are *not* automatically updated.
        job_ids = []
        for artifact in manifest:
            a = DbArtifact(bounty_id=b.id,
                           hash=artifact["hash"],
                           name=artifact["name"])
            s.add(a)
            s.flush()
            job_ids.append(a.id)

            jobs = []
            # TODO: analysis_backends may change during different runs
            for backend in analysis_backends.values():
                v = DbArtifactVerdict(artifact_id=a.id,
                                      backend=backend.name,
                                      status=JOB_STATUS_NEW)
                s.add(v)
                jobs.append(v)
            s.flush()
        s.commit()
        s.close()

        artifacts = []
        for i, artifact in enumerate(manifest):
            # TODO: this is just the way their API works
            # TODO: parallel download
            artifacts.append(
                gevent.spawn(ipfs_download, artifact["hash"],
                             "%s/%s" % (bounty["uri"], i)))
        gevent.joinall(artifacts)
        for a in artifacts:
            if a.exception is not None:
                # TODO: we should abort here
                log.warning("Downloading artifacts for %s not succesful!",
                            bounty["guid"])
                break

        # Start submitting the artifacts
        for job in job_ids:
            dispatch_event("verdict_jobs", bounty["guid"], job)
Exemple #30
0
def bounty_settle_manual(guid, votes):
    s = DbSession()
    bounty = s.query(DbBounty).with_for_update().filter_by(guid=guid).first()

    if not bounty:
        s.close()
        raise KeyError("No such bounty")
    elif bounty.voted or bounty.settled:
        s.close()
        raise ValueError("Bounty was already voted on/settled")

    need = s.query(DbArtifact).filter_by(bounty_id=bounty.id).count()
    if need != len(votes):
        s.close()
        raise ValueError("Need %s vote(s), not %s" % (need, len(votes)))

    log.info("Manually set bounty %s vote to %s", guid, votes)
    bounty.truth_value = votes
    bounty.truth_manual = True
    s.add(bounty)
    s.commit()
    s.close()