Exemple #1
0
    def listen(cls):
        trail_blocks = Conf.get('trail_blocks')
        assert trail_blocks >= 0
        assert trail_blocks < 25

        steemd = SteemClient.instance()
        hive_head = Blocks.head_num()
        for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap=40):
            start_time = time.perf_counter()

            query("START TRANSACTION")
            num = Blocks.process(block)
            follows = Follow.flush(trx=False)
            accts = Accounts.flush(trx=False, period=8)
            CachedPost.dirty_paidouts(block['timestamp'])
            cnt = CachedPost.flush(trx=False)
            query("COMMIT")

            ms = (time.perf_counter() - start_time) * 1000
            print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
                  "% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
                  % (num, block['timestamp'], len(block['transactions']),
                     cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
                     accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))

            # once per hour, update accounts
            if num % 1200 == 0:
                Accounts.dirty_oldest(10000)
                Accounts.flush(trx=True)
                #Accounts.update_ranks()

            # once a minute, update chain props
            if num % 20 == 0:
                cls._update_chain_state(steemd)
Exemple #2
0
    def listen(self):
        """Live (block following) mode."""
        trail_blocks = self._conf.get('trail_blocks')
        assert trail_blocks >= 0
        assert trail_blocks <= 100

        # debug: no max gap if disable_sync in effect
        max_gap = None if self._conf.get('test_disable_sync') else 100

        steemd = self._steem
        hive_head = Blocks.head_num()

        for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
                                          max_gap):
            start_time = perf()

            self._db.query("START TRANSACTION")
            num = Blocks.process(block)
            follows = Follow.flush(trx=False)
            accts = Accounts.flush(steemd, trx=False, spread=8)
            CachedPost.dirty_paidouts(block['timestamp'])
            cnt = CachedPost.flush(steemd, trx=False)
            self._db.query("COMMIT")

            ms = (perf() - start_time) * 1000
            log.info(
                "[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
                "% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows"
                " --% 5dms%s", num, block['timestamp'],
                len(block['transactions']), cnt['insert'], cnt['update'],
                cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows,
                ms, ' SLOW' if ms > 1000 else '')

            if num % 1200 == 0:  #1hr
                log.warning("head block %d @ %s", num, block['timestamp'])
                log.info("[LIVE] hourly stats")
                Accounts.fetch_ranks()
                #Community.recalc_pending_payouts()
            if num % 200 == 0:  #10min
                Community.recalc_pending_payouts()
            if num % 100 == 0:  #5min
                log.info("[LIVE] 5-min stats")
                Accounts.dirty_oldest(500)
            if num % 20 == 0:  #1min
                self._update_chain_state()
Exemple #3
0
    def listen(cls):
        """Live (block following) mode."""
        trail_blocks = Conf.get('trail_blocks')
        assert trail_blocks >= 0
        assert trail_blocks <= 100

        # debug: no max gap if disable_sync in effect
        max_gap = None if Conf.get('disable_sync') else 100

        steemd = SteemClient.instance()
        hive_head = Blocks.head_num()

        for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
                                          max_gap):
            start_time = perf()

            query("START TRANSACTION")
            num = Blocks.process(block)
            follows = Follow.flush(trx=False)
            accts = Accounts.flush(trx=False, spread=8)
            CachedPost.dirty_paidouts(block['timestamp'])
            cnt = CachedPost.flush(trx=False)
            query("COMMIT")

            ms = (perf() - start_time) * 1000
            log.info(
                "[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
                "% 3d payouts,% 3d votes,% 3d accts,% 3d follows --% 5dms%s",
                num, block['timestamp'], len(block['transactions']),
                cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
                accts, follows, int(ms), ' SLOW' if ms > 1000 else '')

            # once per hour, update accounts
            if num % 1200 == 0:
                Accounts.dirty_oldest(10000)
                Accounts.flush(trx=True)
                #Accounts.update_ranks()

            # once a minute, update chain props
            if num % 20 == 0:
                cls._update_chain_state(steemd)
Exemple #4
0
def listen_steemd(trail_blocks=0, max_gap=50):
    assert trail_blocks >= 0
    assert trail_blocks < 25

    # db state
    db_last = Blocks.last()
    last_block = db_last['num']
    last_hash = db_last['hash']

    # chain state
    steemd = get_adapter()
    head_block = steemd.head_block()
    next_expected = time.time()

    # loop state
    tries = 0
    queue = []

    # TODO: detect missed blocks by looking at block timestamps.
    #       this would be an even more efficient way to track slots.
    while True:
        assert not last_block > head_block

        # fast fwd head block if slots missed
        curr_time = time.time()
        while curr_time >= next_expected:
            head_block += 1
            next_expected += 3

        # if gap too large, abort. if caught up, wait.
        gap = head_block - last_block
        if gap > max_gap:
            print("[LIVE] gap too large: %d -- abort listen mode" % gap)
            return
        elif gap > 0:
            print("[LIVE] %d blocks behind..." % gap)
        elif gap == 0:
            time.sleep(next_expected - curr_time)
            head_block += 1
            next_expected += 3

        # get the target block; if DNE, pause and retry
        block_num = last_block + 1
        block = steemd.get_block(block_num)
        if not block:
            tries += 1
            print("[LIVE] block %d unavailable (try %d). delay 1s. head: %d/%d."
                  % (block_num, tries, head_block, steemd.head_block()))
            #assert tries < 12, "could not fetch block %s" % block_num
            assert tries < 240, "could not fetch block %s" % block_num #74
            time.sleep(1)      # pause for 1s; and,
            next_expected += 1 # delay schedule 1s
            continue
        last_block = block_num
        tries = 0

        # ensure this block links to our last; otherwise, blow up. see #59
        if last_hash != block['previous']:
            if queue:
                print("[FORK] Fork encountered. Emptying queue to retry!")
                return
            raise Exception("Unlinkable block: have %s, got %s -> %s)"
                            % (last_hash, block['previous'], block['block_id']))
        last_hash = block['block_id']

        # buffer until queue full
        queue.append(block)
        if len(queue) <= trail_blocks:
            continue


        # buffer primed; process head of queue
        # ------------------------------------

        block = queue.pop(0)

        start_time = time.perf_counter()

        query("START TRANSACTION")
        num = Blocks.process(block)
        follows = Follow.flush(trx=False)
        accts = Accounts.flush(trx=False, period=8)
        CachedPost.dirty_paidouts(block['timestamp'])
        cnt = CachedPost.flush(trx=False)
        query("COMMIT")

        ms = (time.perf_counter() - start_time) * 1000
        print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
              "% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
              % (num, block['timestamp'], len(block['transactions']),
                 cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
                 accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))

        # once per hour, update accounts
        if num % 1200 == 0:
            Accounts.dirty_oldest(10000)
            Accounts.flush(trx=True)
            #Accounts.update_ranks()

        # once a minute, update chain props
        if num % 20 == 0:
            update_chain_state()