def main(): start_time = datetime.now() #block_id, block_hash, depth = (0, '\0' * 32, 0) #block_id, block_hash, depth = (257339, binascii.unhexlify('610ebdcb90530fe6f5c7911479ce8b257738f048390adb5d102cf91200000000'), 12054) db_session = db.Session() res = db.engine.execute('update block set prev_block_id = p.id from block p where block.prev_block_hash = p.block_hash and block.prev_block_id is null') print '%r prev_block_id set' % (res.rowcount,) db_session.commit() db_session.query(db.Block).filter(db.Block.prev_block_hash == 32 * '\x00').update(values={'depth': 0}) ChildBlock = aliased(db.Block, name='child_block') ParentBlock = aliased(db.Block, name='parent_block') prev = None found = True while found: found = False for block_id, block_hash, depth in ( db_session.query( ParentBlock.id, ParentBlock.block_hash, ParentBlock.depth ).join( ChildBlock, ChildBlock.prev_block_id == ParentBlock.id ).filter( (ChildBlock.depth.is_(None)) & (ParentBlock.depth.isnot(None)) ) ): if prev == block_id: break prev = block_id found = True while True: depth += 1 res = db_session.query(db.Block.id, db.Block.block_hash).filter(db.Block.prev_block_id == block_id).first() if res is None: break #prev_block_id = block_id block_id, block_hash = res db_session.query(db.Block).filter(db.Block.id == block_id).update(values={'depth': depth}) # , 'prev_block_id': prev_block_id}) #if depth % 20 == 0: # db_session.commit() db_session.commit() print('%7i %7i %s' % (depth, block_id, binascii.hexlify(block_hash))) #db_session.commit() # the above worked the last time I tried it but previously I needed the below to update some blocks while True: res = db.engine.execute('update block set depth = p.depth + 1 from block p where block.prev_block_id = p.id and block.depth is null and p.depth is not null returning block.id, block.depth, block.block_hash') if res.rowcount == 0: break blk_id, depth, blk_hash = res.first() print('%i %7i %7i %s' % (res.rowcount, blk_id, depth, binascii.hexlify(blk_hash))) db_session.commit()
def __init__(self): self.db_session = db.Session() self.queue = multiprocessing.Queue() self.num_processed = multiprocessing.Value('i', 0) self.total_to_process = self.db_session.query(db.TxIn.id).filter( db.TxIn.txout_id.is_(None) & (db.TxIn.previous_output_transaction_hash != 32 * '\x00')).count() self.total_blocks = self.total_to_process / BLOCK self.blocks_processed = multiprocessing.Value('i', 0) self.shutdown_event = multiprocessing.Event() self.queued_blocks = 0 self.start_time = datetime.now()
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.create_table('txin_txout', sa.Column('txin_id', sa.Integer(), nullable=False), sa.Column('txout_id', sa.Integer(), nullable=False), sa.ForeignKeyConstraint( ['txin_id'], ['txin.id'], ), sa.ForeignKeyConstraint( ['txout_id'], ['txout.id'], ), sa.PrimaryKeyConstraint('txin_id', 'txout_id')) # INSERT INTO txin_txout (txin_id, txout_id) SELECT id, txout_id FROM txin WHERE txout_id IS NOT NULL conn = op.get_bind() session = db.Session(bind=conn) num_to_insert = session.query(sa.func.count(db.TxIn.id)).filter( db.TxIn.txout_id.isnot(None)).scalar() print('Going to insert %u records into txin_txout' % (num_to_insert, )) start_time = datetime.now() num_inserted = 0 for chunk in chunked(session.query(db.TxIn.id, db.TxIn.txout_id).filter( db.TxIn.txout_id.isnot(None)).yield_per(CHUNK_SIZE), chunk_size=CHUNK_SIZE): query_start = datetime.now() conn.execute(db.TxIn_TxOut.__table__.insert().values(chunk)) query_end = datetime.now() num_inserted += len(chunk) tot_time = query_end - start_time avg_time = tot_time / num_inserted print( '%u / %u %.3f%% done, %u inserted, %s for query, %s total, %s avg, ~%s remaining' % (num_inserted, num_to_insert, num_inserted * 100.0 / num_to_insert, len(chunk), query_end - query_start, tot_time, avg_time, avg_time * (num_to_insert - num_inserted))) # Create indexes after inserting data op.create_index(op.f('ix_txin_txout_txin_id'), 'txin_txout', ['txin_id'], unique=False) op.create_index('ix_txin_txout_txin_id_txout_id', 'txin_txout', ['txin_id', 'txout_id'], unique=True) op.create_index(op.f('ix_txin_txout_txout_id'), 'txin_txout', ['txout_id'], unique=False)
def run(self): self.db_session = db.Session() try: while not self.db_write_thread_stop_event.is_set(): try: if self.block_queue.empty(): time.sleep(1) else: try: blktmpfilename = self.block_queue.get(timeout=1) except Queue.Empty: continue log.info('Reading block file %s', blktmpfilename) try: with open(blktmpfilename, 'rb') as blktmpfile: data = blktmpfile.read() except IOError: log.exception('IOError reading blockfile %s', blktmpfilename) continue (msg, _) = protocol.Message.parse(data) assert not _, _ self.write_block_to_db(msg) os.remove(blktmpfilename) except Exception: log.exception( 'Exception in db_write_loop, creating a new session') self.block_queue.put(blktmpfilename) try: db.reconnect() except Exception: log.exception('Exception reconnecting, ignoring') log.info('Reconnected') except: self.block_queue.put(blktmpfilename) raise except: log.exception('exception in db_write_loop') raise finally: log.info('db_write_loop shutting down IOLoop') self.ioloop.shutdown() log.info('db_write_loop finished')
def __init__(self): self.db_session = db.Session() #STATEFILE = 'txout_id' #if not os.path.exists(STATEFILE): # with open(STATEFILE, 'w') as f: # f.write(str(self.db_session.query(sql_functions.min(db.TxOut.id)).scalar())) #with open(STATEFILE, 'r') as f: # self.min_id = int(f.read()) #self.min_id = 97465110 - BLOCK * 100 #self.min_id = self.db_session.query(sql_functions.min(db.TxOut.id)).scalar() self.queue = multiprocessing.Queue() self.num_processed = multiprocessing.Value('i', 0) self.total_to_process = self.db_session.query(db.TxOut.id).filter( # db.TxOut.id >= self.min_id db.TxOut.spent.is_(None) | db.TxOut.spent.is_(False)).count() self.total_blocks = self.total_to_process / BLOCK self.blocks_processed = multiprocessing.Value('i', 0) self.shutdown_event = multiprocessing.Event() self.queued_blocks = 0 self.start_time = datetime.now()
def __init__(self, read_blocktmp_files=True): super(IOLoop, self).__init__() self.sock = None self.out_queue = Queue.Queue() self.waiting_for = {} self.stored = {} self.max_height = multiprocessing.Value(ctypes.c_ulong, 0) self.db_session = db.Session() self.db_write_loop = DBWriteLoop(self) max_height = self.db_session.query(sql_functions.max( db.Block.depth)).scalar() if max_height is not None: self.max_height.value = max_height self.known_blocks = set( block.block_hash for block in self.db_session.query(db.Block.block_hash).all()) self._prev_block_hashes = set() if read_blocktmp_files: for blktmpfilename in glob.glob('blocktmp/*.rawblk'): self.db_write_loop.queue_block(blktmpfilename) log.info('Reading blockfile %s', blktmpfilename) try: with open(blktmpfilename, 'rb') as blktmpfile: data = blktmpfile.read() except IOError: log.exception('IOError reading blockfile %s', blktmpfilename) continue (msg, _) = protocol.Message.parse(data) assert not _, _ self.known_blocks.add(msg.block_hash) self._prev_block_hashes.add(msg.prev_block_hash) self.num_blocks = multiprocessing.Value(ctypes.c_ulonglong, len(self.known_blocks)) log.info('Block database starting with %r blocks', self.num_blocks.value) self.process_queue = Queue.Queue() self.process_thread = None self.write_thread = None self.read_thread = None self.shutdown_event = multiprocessing.Event() self._internal_shutdown_event = threading.Event() self.message_timeout = MESSAGE_TIMEOUT self.ping_timing = SECONDS_BETWEEN_PINGS self.last_ping = None self.last_pong = None self.last_message = None self.remote_addr = ('10.0.42.253', 8333) self.remote_addr = ('127.0.0.1', 8333) local_addr = [ addrs for i, addrs in ((i, [ addr for addr in addrs[netifaces.AF_INET] if 'peer' not in addr ]) for i, addrs in ((i, netifaces.ifaddresses(i)) for i in netifaces.interfaces()) if netifaces.AF_INET in addrs) if addrs ][0][0] self.local_addr = local_addr['addr'] self.local_port = 8334