class TestBlobManager(AsyncioTestCase):
async def setup_blob_manager(self, save_blobs=True):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.config = Config(save_blobs=save_blobs)
self.storage = SQLiteStorage(self.config, os.path.join(tmp_dir, "lbrynet.sqlite"))
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
async def test_sync_blob_file_manager_on_startup(self):
await self.setup_blob_manager(save_blobs=True)
# add a blob file
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
with open(os.path.join(self.blob_manager.blob_dir, blob_hash), 'wb') as f:
f.write(blob_bytes)
# it should not have been added automatically on startup
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
# make sure we can add the blob
await self.blob_manager.blob_completed(self.blob_manager.get_blob(blob_hash, len(blob_bytes)))
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
# stop the blob manager and restart it, make sure the blob is there
self.blob_manager.stop()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
# test that the blob is removed upon the next startup after the file being manually deleted
self.blob_manager.stop()
# manually delete the blob file and restart the blob manager
os.remove(os.path.join(self.blob_manager.blob_dir, blob_hash))
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
# check that the deleted blob was updated in the database
self.assertEqual(
'pending', (
await self.storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
)
)
async def test_host_same_blob_to_multiple_peers_at_once(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2**20) - 1)
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_conf = Config()
second_client_storage = SQLiteStorage(
second_client_conf,
os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobManager(self.loop, second_client_dir,
second_client_storage,
second_client_conf)
server_from_second_client = make_kademlia_peer(b'1' * 48,
"127.0.0.1",
tcp_port=33333,
allow_localhost=True)
await second_client_storage.open()
await second_client_blob_manager.setup()
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
second_client_blob = second_client_blob_manager.get_blob(blob_hash)
# download the blob
await asyncio.gather(
request_blob(self.loop, second_client_blob,
server_from_second_client.address,
server_from_second_client.tcp_port, 2, 3),
self._test_transfer_blob(blob_hash))
await second_client_blob.verified.wait()
self.assertTrue(second_client_blob.get_is_verified())
async def main(blob_hash: str, url: str):
conf = Config()
loop = asyncio.get_running_loop()
host_url, port = url.split(":")
try:
host = None
if ipaddress.ip_address(host_url):
host = host_url
except ValueError:
host = None
if not host:
host_info = await loop.getaddrinfo(
host_url, 'https',
proto=socket.IPPROTO_TCP,
)
host = host_info[0][4][0]
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
await storage.open()
await blob_manager.setup()
blob = blob_manager.get_blob(blob_hash)
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
conf.blob_download_timeout)
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
f"keep connection: {keep}")
if blob.get_is_verified():
await blob_manager.delete_blobs([blob.blob_hash])
print(f"deleted {blob_hash}")
async def test_old_key_sort_sd_blob(self):
loop = asyncio.get_event_loop()
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.conf = Config()
storage = SQLiteStorage(self.conf, ":memory:")
await storage.open()
blob_manager = BlobManager(loop, tmp_dir, storage, self.conf)
sd_bytes = b'{"stream_name": "4f62616d6120446f6e6b65792d322e73746c", "blobs": [{"length": 1153488, "blob_num' \
b'": 0, "blob_hash": "9fa32a249ce3f2d4e46b78599800f368b72f2a7f22b81df443c7f6bdbef496bd61b4c0079c7' \
b'3d79c8bb9be9a6bf86592", "iv": "0bf348867244019c9e22196339016ea6"}, {"length": 0, "blob_num": 1,' \
b' "iv": "9f36abae16955463919b07ed530a3d18"}], "stream_type": "lbryfile", "key": "a03742b87628aa7' \
b'228e48f1dcd207e48", "suggested_file_name": "4f62616d6120446f6e6b65792d322e73746c", "stream_hash' \
b'": "b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e1' \
b'60207"}'
sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2'
stream_hash = 'b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e160207'
blob = blob_manager.get_blob(sd_hash)
blob.set_length(len(sd_bytes))
writer = blob.get_blob_writer()
writer.write(sd_bytes)
await blob.verified.wait()
descriptor = await StreamDescriptor.from_stream_descriptor_blob(
loop, blob_manager.blob_dir, blob
)
self.assertEqual(stream_hash, descriptor.get_stream_hash())
self.assertEqual(sd_hash, descriptor.calculate_old_sort_sd_hash())
self.assertNotEqual(sd_hash, descriptor.calculate_sd_hash())
class TestReflector(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.key = b'deadbeef' * 4
self.cleartext = os.urandom(20000000)
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.conf = Config()
self.storage = SQLiteStorage(self.conf,
os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage,
self.conf)
self.addCleanup(self.blob_manager.stop)
self.stream_manager = StreamManager(self.loop, Config(),
self.blob_manager, None,
self.storage, None)
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
self.server_conf = Config()
self.server_storage = SQLiteStorage(
self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
await self.server_storage.open()
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir,
self.server_storage,
self.server_conf)
self.addCleanup(self.server_blob_manager.stop)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
# create the stream
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream_manager.config.reflect_streams = False
self.stream = await self.stream_manager.create(file_path)
async def _test_reflect_stream(self,
response_chunk_size=50,
partial_needs=False):
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=response_chunk_size,
partial_needs=partial_needs)
reflector.start_server(5566, '127.0.0.1')
if partial_needs:
server_blob = self.server_blob_manager.get_blob(
self.stream.sd_hash)
client_blob = self.blob_manager.get_blob(self.stream.sd_hash)
with client_blob.reader_context() as handle:
server_blob.set_length(client_blob.get_length())
writer = server_blob.get_blob_writer('nobody', 0)
writer.write(handle.read())
self.server_blob_manager.blob_completed(server_blob)
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertEqual(100, self.stream.reflector_progress)
if partial_needs:
self.assertFalse(self.stream.is_fully_reflected)
send_more = await self.stream.upload_to_reflector(
'127.0.0.1', 5566)
self.assertGreater(len(send_more), 0)
sent.extend(send_more)
sent.append(self.stream.sd_hash)
self.assertSetEqual(
set(sent),
set(
map(
lambda b: b.blob_hash, self.stream.descriptor.blobs[:-1] +
[self.blob_manager.get_blob(self.stream.sd_hash)])))
send_more = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertEqual(len(send_more), 0)
self.assertTrue(self.stream.is_fully_reflected)
server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash)
self.assertTrue(server_sd_blob.get_is_verified())
self.assertEqual(server_sd_blob.length, server_sd_blob.length)
for blob in self.stream.descriptor.blobs[:-1]:
server_blob = self.server_blob_manager.get_blob(blob.blob_hash)
self.assertTrue(server_blob.get_is_verified())
self.assertEqual(server_blob.length, blob.length)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertListEqual(sent, [])
async def test_reflect_stream(self):
return await asyncio.wait_for(
self._test_reflect_stream(response_chunk_size=50),
3,
loop=self.loop)
async def test_reflect_stream_but_reflector_changes_its_mind(self):
return await asyncio.wait_for(
self._test_reflect_stream(partial_needs=True), 3, loop=self.loop)
async def test_reflect_stream_small_response_chunks(self):
return await asyncio.wait_for(
self._test_reflect_stream(response_chunk_size=30),
3,
loop=self.loop)
async def test_announces(self):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce,
"sd blob not set to announce")
self.assertNotIn(self.stream.descriptor.blobs[0].blob_hash,
to_announce, "head blob set to announce")
async def test_result_from_disconnect_mid_sd_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=50,
stop_event=stop,
incoming_event=incoming)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(
self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
stop.set()
# this used to raise (and then propagate) a CancelledError
self.assertListEqual(await reflect_task, [])
self.assertFalse(self.stream.is_fully_reflected)
self.assertFalse(
self.server_blob_manager.get_blob(
self.stream.sd_hash).get_is_verified())
async def test_result_from_disconnect_after_sd_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=50,
stop_event=stop,
incoming_event=incoming,
not_incoming_event=not_incoming)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(
self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
stop.set()
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash], sent)
self.assertTrue(
self.server_blob_manager.get_blob(
self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_after_data_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=50,
stop_event=stop,
incoming_event=incoming,
not_incoming_event=not_incoming)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(
self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
await not_incoming.wait()
stop.set()
sent = await reflect_task
self.assertListEqual(
[self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash],
sent)
self.assertTrue(
self.server_blob_manager.get_blob(
self.stream.sd_hash).get_is_verified())
self.assertTrue(
self.server_blob_manager.get_blob(
self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_mid_data_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=50,
stop_event=stop,
incoming_event=incoming,
not_incoming_event=not_incoming)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(
self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(
self.server_blob_manager.get_blob(
self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(
self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_delete_file_during_reflector_upload(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=50,
stop_event=stop,
incoming_event=incoming,
not_incoming_event=not_incoming)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(
self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash], sent)
self.assertTrue(
self.server_blob_manager.get_blob(
self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(
self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
class TestStreamAssembler(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.key = b'deadbeef' * 4
self.cleartext = os.urandom(20000000)
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.conf = Config()
self.storage = SQLiteStorage(self.conf,
os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage,
self.conf)
self.stream_manager = StreamManager(self.loop, Config(),
self.blob_manager, None,
self.storage, None)
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
self.server_conf = Config()
self.server_storage = SQLiteStorage(
self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
await self.server_storage.open()
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir,
self.server_storage,
self.server_conf)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
# create the stream
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream = await self.stream_manager.create_stream(file_path)
async def _test_reflect_stream(self, response_chunk_size):
reflector = ReflectorServer(self.server_blob_manager,
response_chunk_size=response_chunk_size)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertSetEqual(
set(sent),
set(
map(
lambda b: b.blob_hash, self.stream.descriptor.blobs[:-1] +
[self.blob_manager.get_blob(self.stream.sd_hash)])))
server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash)
self.assertTrue(server_sd_blob.get_is_verified())
self.assertEqual(server_sd_blob.length, server_sd_blob.length)
for blob in self.stream.descriptor.blobs[:-1]:
server_blob = self.server_blob_manager.get_blob(blob.blob_hash)
self.assertTrue(server_blob.get_is_verified())
self.assertEqual(server_blob.length, blob.length)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertListEqual(sent, [])
async def test_reflect_stream(self):
return await asyncio.wait_for(
self._test_reflect_stream(response_chunk_size=50),
3,
loop=self.loop)
async def test_reflect_stream_small_response_chunks(self):
return await asyncio.wait_for(
self._test_reflect_stream(response_chunk_size=30),
3,
loop=self.loop)
async def test_announces(self):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce,
"sd blob not set to announce")
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce,
"head blob not set to announce")
