def upgrade_db(self): self.logger.info('DB version: {:d}'.format(self.db_version)) self.logger.info('Upgrading your DB; this can take some time...') def upgrade_u_prefix(prefix): count = 0 with self.utxo_db.write_batch() as batch: batch_delete = batch.delete batch_put = batch.put # Key: b'u' + address_hashX + tx_idx + tx_num for db_key, db_value in self.utxo_db.iterator(prefix=prefix): if len(db_key) != 18: break count += 1 batch_delete(db_key) batch_put(db_key[:14] + b'\0\0' + db_key[14:], db_value) return count last = time.time() count = 0 for cursor in range(65536): prefix = b'u' + pack_be_uint16(cursor) count += upgrade_u_prefix(prefix) now = time.time() if now > last + 10: last = now self.logger.info(f'DB 1 of 2: {count:,d} entries updated, ' f'{cursor * 100 / 65536:.1f}% complete') self.logger.info('DB 1 of 2 upgraded successfully') def upgrade_h_prefix(prefix): count = 0 with self.utxo_db.write_batch() as batch: batch_delete = batch.delete batch_put = batch.put # Key: b'h' + compressed_tx_hash + tx_idx + tx_num for db_key, db_value in self.utxo_db.iterator(prefix=prefix): if len(db_key) != 11: break count += 1 batch_delete(db_key) batch_put(db_key[:7] + b'\0\0' + db_key[7:], db_value) return count last = time.time() count = 0 for cursor in range(65536): prefix = b'h' + pack_be_uint16(cursor) count += upgrade_h_prefix(prefix) now = time.time() if now > last + 10: last = now self.logger.info(f'DB 2 of 2: {count:,d} entries updated, ' f'{cursor * 100 / 65536:.1f}% complete') self.db_version = max(self.DB_VERSIONS) with self.utxo_db.write_batch() as batch: self.write_utxo_state(batch) self.logger.info('DB 2 of 2 upgraded successfully')
def check_hashX_compaction(history): history.max_hist_row_entries = 40 row_size = history.max_hist_row_entries * 4 full_hist = array.array('I', range(100)).tobytes() hashX = urandom(HASHX_LEN) pairs = ((1, 20), (26, 50), (56, 30)) cum = 0 hist_list = [] hist_map = {} for flush_count, count in pairs: key = hashX + pack_be_uint16(flush_count) hist = full_hist[cum * 4:(cum + count) * 4] hist_map[key] = hist hist_list.append(hist) cum += count write_items = [] keys_to_delete = set() write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) # Check results for sanity assert write_size == len(full_hist) assert len(write_items) == 3 assert len(keys_to_delete) == 3 assert len(hist_map) == len(pairs) for n, item in enumerate(write_items): assert item == (hashX + pack_be_uint16(n), full_hist[n * row_size:(n + 1) * row_size]) for flush_count, count in pairs: assert hashX + pack_be_uint16(flush_count) in keys_to_delete # Check re-compaction is null hist_map = {key: value for key, value in write_items} hist_list = [value for key, value in write_items] write_items.clear() keys_to_delete.clear() write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) assert write_size == 0 assert len(write_items) == 0 assert len(keys_to_delete) == 0 assert len(hist_map) == len(pairs) # Check re-compaction adding a single tx writes the one row hist_list[-1] += array.array('I', [100]).tobytes() write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) assert write_size == len(hist_list[-1]) assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])] assert len(keys_to_delete) == 1 assert write_items[0][0] in keys_to_delete assert len(hist_map) == len(pairs)
def _compact_history(self, limit): '''Inner loop of history compaction. Loops until limit bytes have been processed. ''' keys_to_delete = set() write_items = [] # A list of (key, value) pairs write_size = 0 # Loop over 2-byte prefixes cursor = self.comp_cursor while write_size < limit and cursor < 65536: prefix = pack_be_uint16(cursor) write_size += self._compact_prefix(prefix, write_items, keys_to_delete) cursor += 1 max_rows = self.comp_flush_count + 1 self._flush_compaction(cursor, write_items, keys_to_delete) self.logger.info('history compaction: wrote {:,d} rows ({:.1f} MB), ' 'removed {:,d} rows, largest: {:,d}, {:.1f}% complete' .format(len(write_items), write_size / 1000000, len(keys_to_delete), max_rows, 100 * cursor / 65536)) return write_size
def _compact_hashY_topic(self, hashY_topic, hist_map, hist_list, write_items, keys_to_delete): '''Compres history for a hashY. hist_list is an ordered list of the histories to be compressed.''' # History entries (tx numbers) are 4 bytes each. Distribute # over rows of up to 50KB in size. A fixed row size means # future compactions will not need to update the first N - 1 # rows. max_row_size = self.max_hist_row_entries * 4 full_hist = b''.join(hist_list) nrows = (len(full_hist) + max_row_size - 1) // max_row_size if nrows > 4: self.logger.info('hashY {} is large: {:,d} entries across ' '{:,d} rows'.format(hash_to_hex_str(hashY_topic), len(full_hist) // 4, nrows)) # Find what history needs to be written, and what keys need to # be deleted. Start by assuming all keys are to be deleted, # and then remove those that are the same on-disk as when # compacted. write_size = 0 keys_to_delete.update(hist_map) for n, chunk in enumerate(util.chunks(full_hist, max_row_size)): key = hashY_topic + pack_be_uint16(n) if hist_map.get(key) == chunk: keys_to_delete.remove(key) else: write_items.append((key, chunk)) write_size += len(chunk) assert n + 1 == nrows self.comp_flush_count = max(self.comp_flush_count, n) return write_size
def serialize(self): assert (len(self.proTxHash) == 32 and len(self.ipAddress) == 16 and len(self.inputsHash) == 32 and len(self.payloadSig) == 96) return ( pack_le_uint16(self.version) + # version self.proTxHash + # proTxHash self.ipAddress + # ipAddress pack_be_uint16(self.port) + # port pack_varbytes(self.scriptOperatorPayout) + # scriptOperatorPayout self.inputsHash + # inputsHash self.payloadSig # payloadSig )
def upgrade_cursor(cursor): count = 0 prefix = pack_be_uint16(cursor) key_len = HASHX_LEN + 2 chunks = util.chunks with self.db.write_batch() as batch: batch_put = batch.put for key, hist in self.db.iterator(prefix=prefix): # Ignore non-history entries if len(key) != key_len: continue count += 1 hist = b''.join(item + b'\0' for item in chunks(hist, 4)) batch_put(key, hist) self.upgrade_cursor = cursor self.write_state(batch) return count
def serialize(self): assert (len(self.ipAddress) == 16 and len(self.KeyIdOwner) == 20 and len(self.PubKeyOperator) == 48 and len(self.KeyIdVoting) == 20 and len(self.inputsHash) == 32) return (pack_le_uint16(self.version) + # version pack_le_uint16(self.type) + # type pack_le_uint16(self.mode) + # mode self.collateralOutpoint.serialize() + # collateralOutpoint self.ipAddress + # ipAddress pack_be_uint16(self.port) + # port self.KeyIdOwner + # KeyIdOwner self.PubKeyOperator + # PubKeyOperator self.KeyIdVoting + # KeyIdVoting pack_le_uint16(self.operatorReward) + # operatorReward pack_varbytes(self.scriptPayout) + # scriptPayout self.inputsHash + # inputsHash pack_varbytes(self.payloadSig) # payloadSig )
def flush(self): start_time = time.monotonic() self.flush_count += 1 flush_id = pack_be_uint16(self.flush_count) unflushed = self.unflushed with self.db.write_batch() as batch: for hashX in sorted(unflushed): key = hashX + flush_id batch.put(key, bytes(unflushed[hashX])) self.write_state(batch) count = len(unflushed) unflushed.clear() self.unflushed_count = 0 if self.db.for_sync: elapsed = time.monotonic() - start_time self.logger.info(f'flushed history in {elapsed:.1f}s ' f'for {count:,d} addrs')
def flush(self): start_time = time.time() self.flush_count += 1 flush_id = pack_be_uint16(self.flush_count) unflushed = self.unflushed with self.db.write_batch() as batch: for hashY in sorted(unflushed): key = hashY + flush_id # 把二维数据按照一维数组存储 batch.put(key, b''.join([x.tobytes() for x in unflushed[hashY]])) self.write_state(batch) count = len(unflushed) unflushed.clear() self.unflushed_count = 0 if self.db.for_sync: elapsed = time.time() - start_time self.logger.info(f'flushed eventlog in {elapsed:.1f}s ' f'for {count:,d} addrs')
def upgrade_db(self): self.logger.info(f'UTXO DB version: {self.db_version}') self.logger.info('Upgrading your DB; this can take some time...') def upgrade_u_prefix(prefix): count = 0 with self.utxo_db.write_batch() as batch: batch_delete = batch.delete batch_put = batch.put # Key: b'u' + address_hashX + tx_idx + tx_num for db_key, db_value in self.utxo_db.iterator(prefix=prefix): if len(db_key) == 21: return break if self.db_version == 6: for db_key, db_value in self.utxo_db.iterator( prefix=prefix): count += 1 batch_delete(db_key) batch_put(db_key[:14] + b'\0\0' + db_key[14:] + b'\0', db_value) else: for db_key, db_value in self.utxo_db.iterator( prefix=prefix): count += 1 batch_delete(db_key) batch_put(db_key + b'\0', db_value) return count last = time.time() count = 0 for cursor in range(65536): prefix = b'u' + pack_be_uint16(cursor) count += upgrade_u_prefix(prefix) now = time.time() if now > last + 10: last = now self.logger.info(f'DB 1 of 3: {count:,d} entries updated, ' f'{cursor * 100 / 65536:.1f}% complete') self.logger.info('DB 1 of 3 upgraded successfully') def upgrade_h_prefix(prefix): count = 0 with self.utxo_db.write_batch() as batch: batch_delete = batch.delete batch_put = batch.put # Key: b'h' + compressed_tx_hash + tx_idx + tx_num for db_key, db_value in self.utxo_db.iterator(prefix=prefix): if len(db_key) == 14: return break if self.db_version == 6: for db_key, db_value in self.utxo_db.iterator( prefix=prefix): count += 1 batch_delete(db_key) batch_put(db_key[:7] + b'\0\0' + db_key[7:] + b'\0', db_value) else: for db_key, db_value in self.utxo_db.iterator( prefix=prefix): count += 1 batch_delete(db_key) batch_put(db_key + b'\0', db_value) return count last = time.time() count = 0 for cursor in range(65536): prefix = b'h' + pack_be_uint16(cursor) count += upgrade_h_prefix(prefix) now = time.time() if now > last + 10: last = now self.logger.info(f'DB 2 of 3: {count:,d} entries updated, ' f'{cursor * 100 / 65536:.1f}% complete') # Upgrade tx_counts file size = (self.db_height + 1) * 8 tx_counts = self.tx_counts_file.read(0, size) if len(tx_counts) == (self.db_height + 1) * 4: tx_counts = array('I', tx_counts) tx_counts = array('Q', tx_counts) self.tx_counts_file.write(0, tx_counts.tobytes()) self.db_version = max(self.DB_VERSIONS) with self.utxo_db.write_batch() as batch: self.write_utxo_state(batch) self.logger.info('DB 2 of 3 upgraded successfully')