def check_hashX_compaction(history):
history.max_hist_row_entries = 40
row_size = history.max_hist_row_entries * 5
full_hist = b''.join(pack_le_uint64(tx_num)[:5] for tx_num in range(100))
hashX = urandom(HASHX_LEN)
pairs = ((1, 20), (26, 50), (56, 30))
cum = 0
hist_list = []
hist_map = {}
for flush_count, count in pairs:
key = hashX + pack_be_uint16(flush_count)
hist = full_hist[cum * 5:(cum + count) * 5]
hist_map[key] = hist
hist_list.append(hist)
cum += count
write_items = []
keys_to_delete = set()
write_size = history._compact_hashX(hashX, hist_map, hist_list,
write_items, keys_to_delete)
# Check results for sanity
assert write_size == len(full_hist)
assert len(write_items) == 3
assert len(keys_to_delete) == 3
assert len(hist_map) == len(pairs)
for n, item in enumerate(write_items):
assert item == (hashX + pack_be_uint16(n),
full_hist[n * row_size:(n + 1) * row_size])
for flush_count, count in pairs:
assert hashX + pack_be_uint16(flush_count) in keys_to_delete
# Check re-compaction is null
hist_map = {key: value for key, value in write_items}
hist_list = [value for key, value in write_items]
write_items.clear()
keys_to_delete.clear()
write_size = history._compact_hashX(hashX, hist_map, hist_list,
write_items, keys_to_delete)
assert write_size == 0
assert len(write_items) == 0
assert len(keys_to_delete) == 0
assert len(hist_map) == len(pairs)
# Check re-compaction adding a single tx writes the one row
hist_list[-1] += array.array('I', [100]).tobytes()
write_size = history._compact_hashX(hashX, hist_map, hist_list,
write_items, keys_to_delete)
assert write_size == len(hist_list[-1])
assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])]
assert len(keys_to_delete) == 1
assert write_items[0][0] in keys_to_delete
assert len(hist_map) == len(pairs)
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.ipAddress) == 16
and len(self.inputsHash) == 32
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
self.proTxHash + # proTxHash
self.ipAddress + # ipAddress
pack_be_uint16(self.port) + # port
pack_varbytes(self.scriptOperatorPayout) + # scriptOperatorPayout
self.inputsHash + # inputsHash
self.payloadSig # payloadSig
)
def serialize(self):
assert (len(self.ipAddress) == 16
and len(self.KeyIdOwner) == 20
and len(self.PubKeyOperator) == 48
and len(self.KeyIdVoting) == 20
and len(self.inputsHash) == 32)
return (
pack_le_uint16(self.version) + # version
pack_le_uint16(self.type) + # type
pack_le_uint16(self.mode) + # mode
self.collateralOutpoint.serialize() + # collateralOutpoint
self.ipAddress + # ipAddress
pack_be_uint16(self.port) + # port
self.KeyIdOwner + # KeyIdOwner
self.PubKeyOperator + # PubKeyOperator
self.KeyIdVoting + # KeyIdVoting
pack_le_uint16(self.operatorReward) + # operatorReward
pack_varbytes(self.scriptPayout) + # scriptPayout
self.inputsHash + # inputsHash
pack_varbytes(self.payloadSig) # payloadSig
)
def upgrade_db(self):
self.logger.info(f'UTXO DB version: {self.db_version}')
self.logger.info('Upgrading your DB; this can take some time...')
def upgrade_u_prefix(prefix):
count = 0
with self.utxo_db.write_batch() as batch:
batch_delete = batch.delete
batch_put = batch.put
# Key: b'u' + address_hashX + tx_idx + tx_num
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
if len(db_key) == 21:
return
break
if self.db_version == 6:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key[:14] + b'\0\0' + db_key[14:] + b'\0',
db_value)
else:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key + b'\0', db_value)
return count
last = time.time()
count = 0
for cursor in range(65536):
prefix = b'u' + pack_be_uint16(cursor)
count += upgrade_u_prefix(prefix)
now = time.time()
if now > last + 10:
last = now
self.logger.info(f'DB 1 of 3: {count:,d} entries updated, '
f'{cursor * 100 / 65536:.1f}% complete')
self.logger.info('DB 1 of 3 upgraded successfully')
def upgrade_h_prefix(prefix):
count = 0
with self.utxo_db.write_batch() as batch:
batch_delete = batch.delete
batch_put = batch.put
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
if len(db_key) == 14:
return
break
if self.db_version == 6:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key[:7] + b'\0\0' + db_key[7:] + b'\0',
db_value)
else:
for db_key, db_value in self.utxo_db.iterator(
prefix=prefix):
count += 1
batch_delete(db_key)
batch_put(db_key + b'\0', db_value)
return count
last = time.time()
count = 0
for cursor in range(65536):
prefix = b'h' + pack_be_uint16(cursor)
count += upgrade_h_prefix(prefix)
now = time.time()
if now > last + 10:
last = now
self.logger.info(f'DB 2 of 3: {count:,d} entries updated, '
f'{cursor * 100 / 65536:.1f}% complete')
# Upgrade tx_counts file
size = (self.db_height + 1) * 8
tx_counts = self.tx_counts_file.read(0, size)
if len(tx_counts) == (self.db_height + 1) * 4:
tx_counts = array.array('I', tx_counts)
tx_counts = array.array('Q', tx_counts)
self.tx_counts_file.write(0, tx_counts.tobytes())
self.db_version = max(self.DB_VERSIONS)
with self.utxo_db.write_batch() as batch:
self.write_utxo_state(batch)
self.logger.info('DB 2 of 3 upgraded successfully')