def create_histories(history, hashX_count=100): '''Creates a bunch of random transaction histories, and write them to disk in a series of small flushes.''' hashXs = [urandom(HASHX_LEN) for n in range(hashX_count)] mk_array = lambda: array.array('Q') histories = {hashX: mk_array() for hashX in hashXs} unflushed = history.unflushed tx_num = 0 while hashXs: tx_numb = pack_le_uint64(tx_num)[:5] hash_indexes = set( random.randrange(len(hashXs)) for n in range(1 + random.randrange(4))) for index in hash_indexes: histories[hashXs[index]].append(tx_num) unflushed[hashXs[index]].extend(tx_numb) tx_num += 1 # Occasionally flush and drop a random hashX if non-empty if random.random() < 0.1: history.flush() index = random.randrange(0, len(hashXs)) if histories[hashXs[index]]: del hashXs[index] return histories
def add_unflushed(self, hashXs_by_tx, first_tx_num): unflushed = self.unflushed count = 0 for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num): tx_numb = pack_le_uint64(tx_num)[:5] hashXs = set(hashXs) for hashX in hashXs: unflushed[hashX].extend(tx_numb) count += len(hashXs) self.unflushed_count += count
def fs_update_header_offsets(self, offset_start, height_start, headers): if self.coin.STATIC_BLOCK_HEADERS: return offset = offset_start offsets = [] for h in headers: offset += len(h) offsets.append(pack_le_uint64(offset)) # For each header we get the offset of the next header, hence we # start writing from the next height pos = (height_start + 1) * 8 self.headers_offsets_file.write(pos, b''.join(offsets))
def check_hashX_compaction(history): history.max_hist_row_entries = 40 row_size = history.max_hist_row_entries * 5 full_hist = b''.join(pack_le_uint64(tx_num)[:5] for tx_num in range(100)) hashX = urandom(HASHX_LEN) pairs = ((1, 20), (26, 50), (56, 30)) cum = 0 hist_list = [] hist_map = {} for flush_count, count in pairs: key = hashX + pack_be_uint16(flush_count) hist = full_hist[cum * 5:(cum + count) * 5] hist_map[key] = hist hist_list.append(hist) cum += count write_items = [] keys_to_delete = set() write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) # Check results for sanity assert write_size == len(full_hist) assert len(write_items) == 3 assert len(keys_to_delete) == 3 assert len(hist_map) == len(pairs) for n, item in enumerate(write_items): assert item == (hashX + pack_be_uint16(n), full_hist[n * row_size:(n + 1) * row_size]) for flush_count, count in pairs: assert hashX + pack_be_uint16(flush_count) in keys_to_delete # Check re-compaction is null hist_map = {key: value for key, value in write_items} hist_list = [value for key, value in write_items] write_items.clear() keys_to_delete.clear() write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) assert write_size == 0 assert len(write_items) == 0 assert len(keys_to_delete) == 0 assert len(hist_map) == len(pairs) # Check re-compaction adding a single tx writes the one row hist_list[-1] += array.array('I', [100]).tobytes() write_size = history._compact_hashX(hashX, hist_map, hist_list, write_items, keys_to_delete) assert write_size == len(hist_list[-1]) assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])] assert len(keys_to_delete) == 1 assert write_items[0][0] in keys_to_delete assert len(hist_map) == len(pairs)
def serialize_unsigned(self, payload_version): return b''.join((pack_varbytes(self.code), self.cid, self.did if payload_version > 0 else b'', pack_varbytes(self.nickname), pack_varbytes(self.url), pack_le_uint64(self.location)))
def serialize(self, payload_version): return b''.join( (pack_varbytes(self.ownerpb), pack_varbytes(self.nodepb), pack_varbytes(self.nickname), pack_varbytes(self.url), pack_le_uint64(self.location), pack_varbytes(self.netaddress), pack_varbytes(self.signature)))