class TestBlobManager(AsyncioTestCase): async def setup_blob_manager(self, save_blobs=True): tmp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(tmp_dir)) self.config = Config(save_blobs=save_blobs) self.storage = SQLiteStorage(self.config, os.path.join(tmp_dir, "lbrynet.sqlite")) self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config) await self.storage.open() async def test_memory_blobs_arent_verifie_but_real_ones_are(self): for save_blobs in (False, True): await self.setup_blob_manager(save_blobs=save_blobs) # add a blob file blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" blob_bytes = b'1' * ((2 * 2 ** 20) - 1) blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes)) blob.save_verified_blob(blob_bytes) self.assertTrue(blob.get_is_verified()) self.blob_manager.blob_completed(blob) self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs) async def test_sync_blob_file_manager_on_startup(self): await self.setup_blob_manager(save_blobs=True) # add a blob file blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed" blob_bytes = b'1' * ((2 * 2 ** 20) - 1) with open(os.path.join(self.blob_manager.blob_dir, blob_hash), 'wb') as f: f.write(blob_bytes) # it should not have been added automatically on startup await self.blob_manager.setup() self.assertSetEqual(self.blob_manager.completed_blob_hashes, set()) # make sure we can add the blob await self.blob_manager.blob_completed(self.blob_manager.get_blob(blob_hash, len(blob_bytes))) self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash}) # stop the blob manager and restart it, make sure the blob is there self.blob_manager.stop() self.assertSetEqual(self.blob_manager.completed_blob_hashes, set()) await self.blob_manager.setup() self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash}) # test that the blob is removed upon the next startup after the file being manually deleted self.blob_manager.stop() # manually delete the blob file and restart the blob manager os.remove(os.path.join(self.blob_manager.blob_dir, blob_hash)) await self.blob_manager.setup() self.assertSetEqual(self.blob_manager.completed_blob_hashes, set()) # check that the deleted blob was updated in the database self.assertEqual( 'pending', ( await self.storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash) ) )
class BlobComponent(Component): component_name = BLOB_COMPONENT depends_on = [DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) self.blob_manager: typing.Optional[BlobManager] = None @property def component(self) -> typing.Optional[BlobManager]: return self.blob_manager async def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) data_store = None if DHT_COMPONENT not in self.component_manager.skip_components: dht_node: Node = self.component_manager.get_component( DHT_COMPONENT) if dht_node: data_store = dht_node.protocol.data_store blob_dir = os.path.join(self.conf.data_dir, 'blobfiles') if not os.path.isdir(blob_dir): os.mkdir(blob_dir) self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store) return await self.blob_manager.setup() async def stop(self): self.blob_manager.stop() async def get_status(self): count = 0 if self.blob_manager: count = len(self.blob_manager.completed_blob_hashes) return { 'finished_blobs': count, 'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status }