async def run(self):
for chunk in chunks(self.generate_links(), self.config.chunk_size):
tasks = await self.create_task_group(chunk)
await self.process_task_group(tasks)
self.cleanup_task_group()
await self.close_sessions()
await self.conn.close()
def _compact_hashX(self, hashX, hist_map, hist_list, write_items,
keys_to_delete):
'''Compres history for a hashX. hist_list is an ordered list of
the histories to be compressed.'''
# History entries (tx numbers) are 4 bytes each. Distribute
# over rows of up to 50KB in size. A fixed row size means
# future compactions will not need to update the first N - 1
# rows.
max_row_size = self.max_hist_row_entries * 4
full_hist = b''.join(hist_list)
nrows = (len(full_hist) + max_row_size - 1) // max_row_size
if nrows > 4:
self.log_info(
'hashX {} is large: {:,d} entries across {:,d} rows'.format(
hash_to_str(hashX),
len(full_hist) // 4, nrows))
# Find what history needs to be written, and what keys need to
# be deleted. Start by assuming all keys are to be deleted,
# and then remove those that are the same on-disk as when
# compacted.
write_size = 0
keys_to_delete.update(hist_map)
for n, chunk in enumerate(util.chunks(full_hist, max_row_size)):
key = hashX + pack('>H', n)
if hist_map.get(key) == chunk:
keys_to_delete.remove(key)
else:
write_items.append((key, chunk))
write_size += len(chunk)
assert n + 1 == nrows
self.comp_flush_count = max(self.comp_flush_count, n)
return write_size
async def reorg_chain(self, count=None):
'''Handle a chain reorganisation.
Count is the number of blocks to simulate a reorg, or None for
a real reorg.'''
if count is None:
self.logger.info('chain reorg detected')
else:
self.logger.info('faking a reorg of {:,d} blocks'.format(count))
await self.controller.run_in_executor(self.flush, True)
hashes, start, count = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_str(hash) for hash in reversed(hashes)]
# get saved evntlog hashYs
if hashes:
eventlog_hashYs = reduce(
operator.add, [self.get_block_hashYs(x) for x in hashes])
else:
eventlog_hashYs = []
self.logger.info('chain reorg eventlog_hashYs {} {}'.format(
eventlog_hashYs, hashes))
for hex_hashes in chunks(hashes, 50):
blocks = await self.daemon.raw_blocks(hex_hashes)
await self.controller.run_in_executor(self.backup_blocks, blocks,
eventlog_hashYs)
await self.prefetcher.reset_height()
def fetch_videos(self, ids, batch_size=20):
result = {}
batches = util.chunks(ids, batch_size)
for batch in batches:
response = self.fetch(','.join(batch))
data = self.save_videos(response)
result.update(data)
return result
def _upload_bfmr(self, numbers) -> None:
former_headless = self.driver_creator.args.headless
self.driver_creator.args.headless = False
driver = self._login_bfmr()
try:
for batch in util.chunks(numbers, 100):
self._upload_bfmr_batch(driver, batch)
finally:
driver.quit()
self.driver_creator.args.headless = former_headless
def fetch_channels(self, ids):
batches = util.chunks(ids, 30)
results = []
for batch in batches:
request = self.get_youtube().channels().list(
part="snippet,contentDetails",
id=','.join(batch),
)
response = request.execute()
results += [Channel(item) for item in response.get('items')]
return results
def pool_gen_global_site_channels(pool, master):
config = master.config
site = master.global_site
pages = []
out_dir = "%s/global" % (config.out_dir)
results = []
if site.gen_channel_html and config.channel:
util.mkdir("%s/channels" % out_dir)
chunks = util.chunks(site.groups, 10)
for chunk in chunks:
results.append(
pool.apply_async(single_channel_pages, (site, config, chunk)))
return results
async def reorg_chain(self, count=None):
'''Handle a chain reorganisation.
Count is the number of blocks to simulate a reorg, or None for
a real reorg.'''
if count is None:
self.logger.info('chain reorg detected')
else:
self.logger.info('faking a reorg of {:,d} blocks'.format(count))
await self.controller.run_in_executor(self.flush, True)
hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_str(hash) for hash in reversed(hashes)]
for hex_hashes in chunks(hashes, 50):
blocks = await self.daemon.raw_blocks(hex_hashes)
await self.controller.run_in_executor(self.backup_blocks, blocks)
await self.prefetcher.reset_height()
def test_chunks():
assert list(util.chunks([1, 2, 3, 4, 5], 2)) == [[1, 2], [3, 4], [5]]
def _upload_bfmr(self, numbers) -> None:
for batch in util.chunks(numbers, 100):
self._upload_bfmr_batch(batch)
