Example #1
0
 def __init__(self, load=False):
     self.outpoints = UTXO_Map()
     self.block_name = ZERO_NAME
     self.height = -1
     self.total = 0
     self.lost = 0
     self.fees = 0
     if load:
         from __main__ import G
         save_path = os.path.join(G.args.base, self.save_path)
         self.load_state(save_path)
Example #2
0
 def __init__ (self, load=False):
     self.outpoints = UTXO_Map()
     self.block_name = ZERO_NAME
     self.height = -1
     self.total = 0
     self.lost = 0
     self.fees = 0
     if load:
         from __main__ import G
         save_path = os.path.join (G.args.base, self.save_path)
         self.load_state (save_path)
Example #3
0
def catch_up(G):

    db = G.block_db

    def get_names():
        "get the chain of all block names, ignoring forks"
        if not db.num_block:
            return []
        else:
            names = list(db.num_block[db.last_block])
            # XXX handle this case
            assert (len(names) == 1)
            b = db[names[0]]
            r = []
            name = b.name
            while 1:
                r.append(name)
                name = db.prev[name]
                if name == ZERO_NAME:
                    break
            r.reverse()
            return r

    ledger = LedgerState(load=True)

    if len(ledger.outpoints) == 0:
        LOG('no cache')
        ledger.outpoints = UTXO_Scan_Map()
        fast_scan = True
    else:
        fast_scan = False

    t0 = timer()
    names = get_names()
    #if fast_scan:
    #    # TRIM FOR TESTING ONLY
    #    names = names[:225430]
    # drop back by a 20-block horizon
    most_names = names[:-20]
    i = 0
    fed = 0
    for name in most_names:
        if i == ledger.height + 1:
            if i % 1000 == 0:
                LOG('scan', i)
            block = db[name]
            ledger.feed_block(block, i)
            fed += 1
        elif i <= ledger.height:
            pass
        else:
            LOG('block too high?')
            import pdb
            pdb.set_trace()
        i += 1
        coro.yield_slice()

    LOG('total/lost/fees', ledger.total, ledger.lost, ledger.fees)
    LOG('scan', t0.end(), fed)
    if fed > 150:
        LOG('saving', repr(ledger.block_name))
        ledger.save_state()
    if fast_scan:
        LOG('fast scan done, reloading')
        ledger.outpoints = None
        ledger.outpoints = UTXO_Map()
        ledger.load_state()

    LOG('topping off recent blocks')
    G.recent_blocks = RecentBlocks(ledger, db)
    names = db.next(ledger.block_name)
    while names:
        name = names.pop()
        LOG('add', repr(name))
        G.recent_blocks.new_block(db[name])
        names += db.next(name)

    if __name__ == '__main__':
        coro.set_exit()

    return G.recent_blocks
Example #4
0
class LedgerState:

    save_path = 'utxo.bin'
    do_yields = True

    def __init__(self, load=False):
        self.outpoints = UTXO_Map()
        self.block_name = ZERO_NAME
        self.height = -1
        self.total = 0
        self.lost = 0
        self.fees = 0
        if load:
            from __main__ import G
            save_path = os.path.join(G.args.base, self.save_path)
            self.load_state(save_path)

    def clone(self):
        ob = LedgerState()
        ob.block_name = self.block_name
        ob.height = self.height
        ob.total = self.total
        ob.lost = self.lost
        ob.fees = self.fees
        ob.outpoints = self.outpoints.copy()
        return ob

    def extend(self, block, height, verify=True):
        ob = self.clone()
        ob.feed_block(block, height, verify)
        return ob

    def get_total_outpoints(self):
        total = 0
        for k, v in self.outpoints:
            total += len(v)
        return total

    cache_version = 3

    def save_state(self):
        from coro.asn1.data_file import DataFileWriter
        from __main__ import G
        save_path = os.path.join(G.args.base, self.save_path)
        f = open(save_path + '.tmp', 'wb')
        df = DataFileWriter(f)
        t0 = timer()
        df.write_object([
            self.cache_version, self.height,
            str(self.block_name), self.total, self.lost, self.fees,
            len(self.outpoints)
        ])
        n = 0
        for item in self.outpoints:
            df.write_object(item)
            n += 1
            if n % 1000 == 999:
                coro.yield_slice()
        f.close()
        os.rename(save_path + '.tmp', save_path)
        LOG('saved outpoints', len(self.outpoints), n, t0.end())

    def load_state(self, path=None):
        from coro.asn1.data_file import DataFileReader
        from __main__ import G
        if path is None:
            path = os.path.join(G.args.base, self.save_path)
        LOG('cache', 'start')
        t0 = timer()
        try:
            f = open(path, 'rb')
            df = DataFileReader(f)
            info = df.read_object()
            if info[0] < self.cache_version:
                LOG('old cache version, ignoring')
                return
            assert (info[0] == self.cache_version)  # version
            [
                _, self.height, self.block_name, self.total, self.lost,
                self.fees, size
            ] = info
            LOG('cache', self.height, size)
            self.block_name = Name(self.block_name)
            n = [0]
            df.next = df.read_object
            self.outpoints.build(df, size)
            f.close()
            LOG('cache', 'stop', len(self.outpoints), n[0])
            LOG('cache', self.height, repr(self.block_name))
        except IOError:
            pass
        LOG('cache', 'stop', t0.end())

    def store_outputs(self, tx):
        output_sum = 0
        outputs = []
        for i, (amt, lock_script) in enumerate(tx.outputs):
            #if len(lock_script) > 500:
            #    W ('%r len(script) = %d\n' % (tx.name, len(lock_script)))
            if not is_unspendable(lock_script):
                outputs.append((i, amt, lock_script))
            output_sum += amt
        self.outpoints.new_entry(str(tx.name), outputs)
        self.total += output_sum
        return output_sum

    def get_utxo(self, name, index):
        return self.outpoints.get_utxo(name, index)

    def feed_tx(self, index, tx, timestamp, verify=False):
        input_sum = 0
        for j in range(len(tx.inputs)):
            (outpoint, index), script, sequence = tx.inputs[j]
            outstr = str(outpoint)
            amt, lock_script = self.outpoints.pop_utxo(outstr, index)
            if verify:
                try:
                    tx.verify(j, lock_script, timestamp)
                except VerifyError:
                    self.outpoints.new_entry(outstr,
                                             [(index, amt, lock_script)])
                    raise
            input_sum += amt
            if self.do_yields and j % 20 == 19:
                coro.yield_slice()
        output_sum = self.store_outputs(tx)
        return input_sum, output_sum

    def feed_block(self, b, height, verify=False):
        if b.prev_block != self.block_name:
            raise ValueError(b.prev_block, self.block_name)
        # assume coinbase is ok for now
        tx0 = b.transactions[0]
        reward0 = self.store_outputs(tx0)
        fees = 0
        for i, tx in enumerate(b.transactions):
            if i == 0:
                continue
            input_sum, output_sum = self.feed_tx(i, tx, b.timestamp, verify)
            fees += input_sum - output_sum
            self.total -= input_sum
        self.fees += fees
        reward1 = compute_reward(height)
        if reward1 + fees != reward0:
            lost = (reward1 + fees) - reward0
            #W ('reward mismatch height=%d lost=%s\n' % (height, lost))
            self.lost += lost
        self.height = height
        self.block_name = b.name
Example #5
0
class LedgerState:

    save_path = 'utxo.bin'
    do_yields = True

    def __init__ (self, load=False):
        self.outpoints = UTXO_Map()
        self.block_name = ZERO_NAME
        self.height = -1
        self.total = 0
        self.lost = 0
        self.fees = 0
        if load:
            from __main__ import G
            save_path = os.path.join (G.args.base, self.save_path)
            self.load_state (save_path)

    def clone (self):
        ob = LedgerState()
        ob.block_name = self.block_name
        ob.height = self.height
        ob.total = self.total
        ob.lost = self.lost
        ob.fees = self.fees
        ob.outpoints = self.outpoints.copy()
        return ob

    def extend (self, block, height, verify=True):
        ob = self.clone()
        ob.feed_block (block, height, verify)
        return ob

    def get_total_outpoints (self):
        total = 0
        for k, v in self.outpoints:
            total += len(v)
        return total

    cache_version = 2

    def save_state (self):
        from coro.asn1.data_file import DataFileWriter
        from __main__ import G
        save_path = os.path.join (G.args.base, self.save_path)
        f = open (save_path + '.tmp', 'wb')
        df = DataFileWriter (f)
        t0 = timer()
        df.write_object ([
            self.cache_version,
            self.height,
            str(self.block_name),
            self.total,
            self.lost,
            self.fees,
            len(self.outpoints)
        ])
        n = 0
        for item in self.outpoints:
            df.write_object (item)
            n += 1
            if n % 1000 == 999:
                coro.yield_slice()
        f.close()
        os.rename (save_path + '.tmp', save_path)
        W ('[saved outpoints %d/%d entries %.02fs]' % (len(self.outpoints), n, t0.end()))

    def load_state (self, path=None):
        from coro.asn1.data_file import DataFileReader
        from __main__ import G
        if path is None:
            path = os.path.join (G.args.base, self.save_path)
        W ('loading outpoints cache...')
        t0 = timer()
        try:
            f = open (path, 'rb')
            df = DataFileReader (f)
            info = df.read_object()
            assert (info[0] == self.cache_version)  # version
            [_, self.height, self.block_name, self.total, self.lost, self.fees, size] = info
            W (' height = %d ...' % (self.height,))
            self.block_name = Name (self.block_name)
            n = [0]
            def gen():
                while 1:
                    try:
                        x = df.read_object()
                        n[0] += 1
                        yield x
                    except EOFError:
                        break
            self.outpoints.build (gen(), size)
            f.close()
            W ('[loaded outpoints %d/%d entries]' % (len(self.outpoints),n[0]))
            W ('\nlast block: %d %064x\n' % (self.height, self.block_name))
        except IOError:
            pass
        W ('...done (%.2fs)\n' % (t0.end(),))

    def store_outputs (self, tx):
        output_sum = 0
        i = 0
        outputs = []
        for amt, pk_script in tx.outputs:
            #if len(pk_script) > 500:
            #    W ('%r len(script) = %d\n' % (tx.name, len(pk_script)))
            outputs.append ((i, amt, pk_script))
            if amt > 0:
                output_sum += amt
            i += 1
        self.outpoints.new_entry (str(tx.name), outputs)
        self.total += output_sum
        return output_sum

    def get_utxo (self, name, index):
        return self.outpoints.get_utxo (name, index)

    def feed_block (self, b, height, verify=False):
        if b.prev_block != self.block_name:
            raise ValueError (b.prev_block, self.block_name)
        # assume coinbase is ok for now
        tx0 = b.transactions[0]
        reward0 = self.store_outputs (tx0)
        fees = 0
        for i, tx in enumerate (b.transactions):
            if i == 0:
                continue
            # verify each transaction
            # first, we need the output script for each of the inputs
            input_sum = 0
            for j in range (len (tx.inputs)):
                (outpoint, index), script, sequence = tx.inputs[j]
                amt, oscript = self.outpoints.pop_utxo (str(outpoint), index)
                #W ('.')
                if verify:
                    tx.verify (j, oscript, b.timestamp)
                # XXX if it fails to verify, put it back!
                input_sum += amt
            output_sum = self.store_outputs (tx)
            fees += input_sum - output_sum
            self.total -= input_sum
            if self.do_yields and i % 50 == 0:
                coro.yield_slice()
        self.fees += fees
        reward1 = compute_reward (height)
        if reward1 + fees != reward0:
            lost = (reward1 + fees) - reward0
            #W ('reward mismatch height=%d lost=%s\n' % (height, lost))
            self.lost += lost
        self.height = height
        self.block_name = b.name
Example #6
0
def catch_up(G):

    db = G.block_db

    def get_names():
        "get the chain of all block names, ignoring forks"
        if not db.num_block:
            return []
        else:
            names = list(db.num_block[db.last_block])
            # XXX handle this case
            assert (len(names) == 1)
            b = db[names[0]]
            r = []
            name = b.name
            while 1:
                r.append(name)
                name = db.prev[name]
                if name == ZERO_NAME:
                    break
            r.reverse()
            return r

    ledger = LedgerState(load=True)

    if len(ledger.outpoints) == 0:
        W('no outpoints cache.  performing fast scan [30-45 minutes]\n')
        ledger.outpoints = UTXO_Scan_Map()
        fast_scan = True
    else:
        fast_scan = False

    t0 = timer()
    names = get_names()
    #if fast_scan:
    #    # TRIM FOR TESTING ONLY
    #    names = names[:225430]
    # drop back by a 20-block horizon
    most_names = names[:-20]
    i = 0
    fed = 0
    for name in most_names:
        if i == ledger.height + 1:
            if i % 1000 == 0:
                W('%d ' % (i, ))
            block = db[name]
            ledger.feed_block(block, i)
            fed += 1
        elif i <= ledger.height:
            pass
        else:
            W('oops, block too high?\n')
            import pdb
            pdb.set_trace()
        i += 1
        coro.yield_slice()

    W('\n')
    W('       total=%20s\n' % bcrepr(ledger.total + ledger.lost))
    W('        live=%20s\n' % bcrepr(ledger.total))
    W('        lost=%20s\n' % bcrepr(ledger.lost))
    W('        fees=%20s\n' % bcrepr(ledger.fees))
    W('(%.2fs to scan %d blocks into ledger)\n' % (t0.end(), fed))
    if fed > 150:
        W('saving... ledger.block_name = %064x\n' % (ledger.block_name, ))
        ledger.save_state()
    if fast_scan:
        W('done with fast scan, reloading...\n')
        ledger.outpoints = None
        ledger.outpoints = UTXO_Map()
        ledger.load_state()

    W('topping off recent_blocks...\n')
    G.recent_blocks = RecentBlocks(ledger, db)
    names = db.next(ledger.block_name)
    while names:
        name = names.pop()
        W('adding %r\n' % (name, ))
        G.recent_blocks.new_block(db[name])
        names += db.next(name)

    if __name__ == '__main__':
        coro.set_exit()

    return G.recent_blocks
Example #7
0
class LedgerState:

    save_path = 'utxo.bin'
    do_yields = True

    def __init__(self, load=False):
        self.outpoints = UTXO_Map()
        self.block_name = ZERO_NAME
        self.height = -1
        self.total = 0
        self.lost = 0
        self.fees = 0
        if load:
            from __main__ import G
            save_path = os.path.join(G.args.base, self.save_path)
            self.load_state(save_path)

    def clone(self):
        ob = LedgerState()
        ob.block_name = self.block_name
        ob.height = self.height
        ob.total = self.total
        ob.lost = self.lost
        ob.fees = self.fees
        ob.outpoints = self.outpoints.copy()
        return ob

    def extend(self, block, height, verify=True):
        ob = self.clone()
        ob.feed_block(block, height, verify)
        return ob

    def get_total_outpoints(self):
        total = 0
        for k, v in self.outpoints:
            total += len(v)
        return total

    cache_version = 2

    def save_state(self):
        from coro.asn1.data_file import DataFileWriter
        from __main__ import G
        save_path = os.path.join(G.args.base, self.save_path)
        f = open(save_path + '.tmp', 'wb')
        df = DataFileWriter(f)
        t0 = timer()
        df.write_object([
            self.cache_version, self.height,
            str(self.block_name), self.total, self.lost, self.fees,
            len(self.outpoints)
        ])
        n = 0
        for item in self.outpoints:
            df.write_object(item)
            n += 1
            if n % 1000 == 999:
                coro.yield_slice()
        f.close()
        os.rename(save_path + '.tmp', save_path)
        W('[saved outpoints %d/%d entries %.02fs]' %
          (len(self.outpoints), n, t0.end()))

    def load_state(self, path=None):
        from coro.asn1.data_file import DataFileReader
        from __main__ import G
        if path is None:
            path = os.path.join(G.args.base, self.save_path)
        W('loading outpoints cache...')
        t0 = timer()
        try:
            f = open(path, 'rb')
            df = DataFileReader(f)
            info = df.read_object()
            assert (info[0] == self.cache_version)  # version
            [
                _, self.height, self.block_name, self.total, self.lost,
                self.fees, size
            ] = info
            W(' height = %d ...' % (self.height, ))
            self.block_name = Name(self.block_name)
            n = [0]

            def gen():
                while 1:
                    try:
                        x = df.read_object()
                        n[0] += 1
                        yield x
                    except EOFError:
                        break

            self.outpoints.build(gen(), size)
            f.close()
            W('[loaded outpoints %d/%d entries]' % (len(self.outpoints), n[0]))
            W('\nlast block: %d %064x\n' % (self.height, self.block_name))
        except IOError:
            pass
        W('...done (%.2fs)\n' % (t0.end(), ))

    def store_outputs(self, tx):
        output_sum = 0
        i = 0
        outputs = []
        for amt, pk_script in tx.outputs:
            #if len(pk_script) > 500:
            #    W ('%r len(script) = %d\n' % (tx.name, len(pk_script)))
            outputs.append((i, amt, pk_script))
            if amt > 0:
                output_sum += amt
            i += 1
        self.outpoints.new_entry(str(tx.name), outputs)
        self.total += output_sum
        return output_sum

    def get_utxo(self, name, index):
        return self.outpoints.get_utxo(name, index)

    def feed_block(self, b, height, verify=False):
        if b.prev_block != self.block_name:
            raise ValueError(b.prev_block, self.block_name)
        # assume coinbase is ok for now
        tx0 = b.transactions[0]
        reward0 = self.store_outputs(tx0)
        fees = 0
        for i, tx in enumerate(b.transactions):
            if i == 0:
                continue
            # verify each transaction
            # first, we need the output script for each of the inputs
            input_sum = 0
            for j in range(len(tx.inputs)):
                (outpoint, index), script, sequence = tx.inputs[j]
                amt, oscript = self.outpoints.pop_utxo(str(outpoint), index)
                #W ('.')
                if verify:
                    tx.verify(j, oscript, b.timestamp)
                # XXX if it fails to verify, put it back!
                input_sum += amt
            output_sum = self.store_outputs(tx)
            fees += input_sum - output_sum
            self.total -= input_sum
            if self.do_yields and i % 50 == 0:
                coro.yield_slice()
        self.fees += fees
        reward1 = compute_reward(height)
        if reward1 + fees != reward0:
            lost = (reward1 + fees) - reward0
            #W ('reward mismatch height=%d lost=%s\n' % (height, lost))
            self.lost += lost
        self.height = height
        self.block_name = b.name
Example #8
0
class LedgerState:

    save_path = 'utxo.bin'
    do_yields = True

    def __init__ (self, load=False):
        self.outpoints = UTXO_Map()
        self.block_name = ZERO_NAME
        self.height = -1
        self.total = 0
        self.lost = 0
        self.fees = 0
        if load:
            from __main__ import G
            save_path = os.path.join (G.args.base, self.save_path)
            self.load_state (save_path)

    def clone (self):
        ob = LedgerState()
        ob.block_name = self.block_name
        ob.height = self.height
        ob.total = self.total
        ob.lost = self.lost
        ob.fees = self.fees
        ob.outpoints = self.outpoints.copy()
        return ob

    def extend (self, block, height, verify=True):
        ob = self.clone()
        ob.feed_block (block, height, verify)
        return ob

    def get_total_outpoints (self):
        total = 0
        for k, v in self.outpoints:
            total += len(v)
        return total

    cache_version = 3

    def save_state (self):
        from coro.asn1.data_file import DataFileWriter
        from __main__ import G
        save_path = os.path.join (G.args.base, self.save_path)
        f = open (save_path + '.tmp', 'wb')
        df = DataFileWriter (f)
        t0 = timer()
        df.write_object ([
            self.cache_version,
            self.height,
            str(self.block_name),
            self.total,
            self.lost,
            self.fees,
            len(self.outpoints)
        ])
        n = 0
        for item in self.outpoints:
            df.write_object (item)
            n += 1
            if n % 1000 == 999:
                coro.yield_slice()
        f.close()
        os.rename (save_path + '.tmp', save_path)
        LOG ('saved outpoints', len(self.outpoints), n, t0.end())

    def load_state (self, path=None):
        from coro.asn1.data_file import DataFileReader
        from __main__ import G
        if path is None:
            path = os.path.join (G.args.base, self.save_path)
        LOG ('cache', 'start')
        t0 = timer()
        try:
            f = open (path, 'rb')
            df = DataFileReader (f)
            info = df.read_object()
            if info[0] < self.cache_version:
                LOG ('old cache version, ignoring')
                return
            assert (info[0] == self.cache_version)  # version
            [_, self.height, self.block_name, self.total, self.lost, self.fees, size] = info
            LOG ('cache', self.height, size)
            self.block_name = Name (self.block_name)
            n = [0]
            df.next = df.read_object
            self.outpoints.build (df, size)
            f.close()
            LOG ('cache', 'stop', len(self.outpoints), n[0])
            LOG ('cache', self.height, repr(self.block_name))
        except IOError:
            pass
        LOG ('cache', 'stop', t0.end())

    def store_outputs (self, tx):
        output_sum = 0
        outputs = []
        for i, (amt, lock_script) in enumerate (tx.outputs):
            #if len(lock_script) > 500:
            #    W ('%r len(script) = %d\n' % (tx.name, len(lock_script)))
            if not is_unspendable (lock_script):
                outputs.append ((i, amt, lock_script))
            output_sum += amt
        self.outpoints.new_entry (str(tx.name), outputs)
        self.total += output_sum
        return output_sum

    def get_utxo (self, name, index):
        return self.outpoints.get_utxo (name, index)

    def feed_tx (self, index, tx, timestamp, verify=False):
        input_sum = 0
        for j in range (len (tx.inputs)):
            (outpoint, index), script, sequence = tx.inputs[j]
            outstr = str(outpoint)
            amt, lock_script = self.outpoints.pop_utxo (outstr, index)
            if verify:
                try:
                    tx.verify (j, lock_script, timestamp)
                except VerifyError:
                    self.outpoints.new_entry (outstr, [(index, amt, lock_script)])
                    raise
            input_sum += amt
            if self.do_yields and j % 20 == 19:
                coro.yield_slice()
        output_sum = self.store_outputs (tx)
        return input_sum, output_sum

    def feed_block (self, b, height, verify=False):
        if b.prev_block != self.block_name:
            raise ValueError (b.prev_block, self.block_name)
        # assume coinbase is ok for now
        tx0 = b.transactions[0]
        reward0 = self.store_outputs (tx0)
        fees = 0
        for i, tx in enumerate (b.transactions):
            if i == 0:
                continue
            input_sum, output_sum = self.feed_tx (i, tx, b.timestamp, verify)
            fees += input_sum - output_sum
            self.total -= input_sum
        self.fees += fees
        reward1 = compute_reward (height)
        if reward1 + fees != reward0:
            lost = (reward1 + fees) - reward0
            #W ('reward mismatch height=%d lost=%s\n' % (height, lost))
            self.lost += lost
        self.height = height
        self.block_name = b.name