def download_it(peer, timeout, blob_hash): tmp_dir = yield threads.deferToThread(tempfile.mkdtemp) storage = SQLiteStorage(tmp_dir, reactor) yield storage.setup() tmp_blob_manager = DiskBlobManager(tmp_dir, storage) config = {'auto_connect': True} if conf.settings['lbryum_wallet_dir']: config['lbryum_wallet_dir'] = conf.settings['lbryum_wallet_dir'] config['use_keyring'] = False config['blockchain_name'] = conf.settings['blockchain_name'] config['lbryum_servers'] = [] wallet = LbryWalletManager.from_lbrynet_config(config, storage) downloader = SinglePeerDownloader() downloader.setup(wallet) try: blob_downloaded = yield downloader.download_blob_from_peer( peer, timeout, blob_hash, tmp_blob_manager) if blob_downloaded: log.info("SUCCESS!") blob = yield tmp_blob_manager.get_blob(blob_hash) pprint(blob) if not blob.verified: log.error("except that its not verified....") else: reader = BlobStreamDescriptorReader(blob) info = None for x in range(0, 3): try: info = yield reader.get_info() except ValueError: pass if info: break # there's some kind of race condition where it sometimes doesnt write the blob to disk in time time.sleep(0.1) if info is not None: pprint(info) for content_blob in info['blobs']: if 'blob_hash' in content_blob: yield download_it(peer, timeout, content_blob['blob_hash']) else: log.error("Download failed") finally: yield tmp_blob_manager.stop() yield threads.deferToThread(shutil.rmtree, tmp_dir) defer.returnValue(True)
def download_it(peer, timeout, blob_hash): tmp_dir = yield threads.deferToThread(tempfile.mkdtemp) announcer = DummyHashAnnouncer() tmp_blob_manager = DiskBlobManager(announcer, tmp_dir, tmp_dir) config = {'auto_connect': True} if conf.settings['lbryum_wallet_dir']: config['lbryum_path'] = conf.settings['lbryum_wallet_dir'] storage = Wallet.InMemoryStorage() wallet = Wallet.LBRYumWallet(storage, config) downloader = SinglePeerDownloader() downloader.setup(wallet) try: blob_downloaded = yield downloader.download_blob_from_peer( peer, timeout, blob_hash, tmp_blob_manager) if blob_downloaded: log.info("SUCCESS!") blob = yield tmp_blob_manager.get_blob(blob_hash) pprint(blob) if not blob.verified: log.error("except that its not verified....") else: reader = BlobStreamDescriptorReader(blob) info = None for x in range(0, 3): try: info = yield reader.get_info() except ValueError: pass if info: break time.sleep( 0.1 ) # there's some kind of race condition where it sometimes doesnt write the blob to disk in time if info is not None: pprint(info) for content_blob in info['blobs']: if 'blob_hash' in content_blob: yield download_it(peer, timeout, content_blob['blob_hash']) else: log.error("Download failed") finally: yield tmp_blob_manager.stop() yield threads.deferToThread(shutil.rmtree, tmp_dir) defer.returnValue(True)
class BlobManagerTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): conf.initialize_settings(False) self.blob_dir = tempfile.mkdtemp() self.db_dir = tempfile.mkdtemp() self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(self.db_dir)) self.peer = Peer('somehost', 22) yield self.bm.storage.setup() @defer.inlineCallbacks def tearDown(self): yield self.bm.stop() yield self.bm.storage.stop() shutil.rmtree(self.blob_dir) shutil.rmtree(self.db_dir) @defer.inlineCallbacks def _create_and_add_blob(self, should_announce=False): # create and add blob to blob manager data_len = random.randint(1, 1000) data = b''.join( random.choice(string.ascii_lowercase).encode() for _ in range(data_len)) hashobj = get_lbry_hash_obj() hashobj.update(data) out = hashobj.hexdigest() blob_hash = out # create new blob yield self.bm.setup() blob = yield self.bm.get_blob(blob_hash, len(data)) writer, finished_d = yield blob.open_for_writing(self.peer) yield writer.write(data) yield self.bm.blob_completed(blob, should_announce) # check to see if blob is there self.assertTrue(os.path.isfile(os.path.join(self.blob_dir, blob_hash))) blobs = yield self.bm.get_all_verified_blobs() self.assertTrue(blob_hash in blobs) defer.returnValue(blob_hash) @defer.inlineCallbacks def test_create_blob(self): blob_hashes = [] # create a bunch of blobs for i in range(0, 10): blob_hash = yield self._create_and_add_blob() blob_hashes.append(blob_hash) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(10, len(blobs)) @defer.inlineCallbacks def test_delete_blob(self): # create blob blob_hash = yield self._create_and_add_blob() blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 1) # delete blob yield self.bm.delete_blobs([blob_hash]) self.assertFalse(os.path.isfile(os.path.join(self.blob_dir, blob_hash))) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 0) blobs = yield self.bm.storage.get_all_blob_hashes() self.assertEqual(len(blobs), 0) self.assertFalse(blob_hash in self.bm.blobs) # delete blob that was already deleted once yield self.bm.delete_blobs([blob_hash]) # delete blob that does not exist, nothing will # happen blob_hash = random_lbry_hash() yield self.bm.delete_blobs([blob_hash]) @defer.inlineCallbacks def test_delete_open_blob(self): # Test that a blob that is opened for writing will not be deleted # create blobs blob_hashes = [] for i in range(0, 10): blob_hash = yield self._create_and_add_blob() blob_hashes.append(blob_hash) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 10) # open the last blob blob = yield self.bm.get_blob(blob_hashes[-1]) w, finished_d = yield blob.open_for_writing(self.peer) # schedule a close, just to leave the reactor clean finished_d.addBoth(lambda x: None) self.addCleanup(w.close) # delete the last blob and check if it still exists yield self.bm.delete_blobs([blob_hash]) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 10) self.assertTrue(blob_hashes[-1] in blobs) self.assertTrue( os.path.isfile(os.path.join(self.blob_dir, blob_hashes[-1]))) @defer.inlineCallbacks def test_should_announce(self): # create blob with should announce blob_hash = yield self._create_and_add_blob(should_announce=True) out = yield self.bm.get_should_announce(blob_hash) self.assertTrue(out) count = yield self.bm.count_should_announce_blobs() self.assertEqual(1, count) # set should announce to False yield self.bm.set_should_announce(blob_hash, should_announce=False) out = yield self.bm.get_should_announce(blob_hash) self.assertFalse(out) count = yield self.bm.count_should_announce_blobs() self.assertEqual(0, count)
class BlobManagerTest(unittest.TestCase): def setUp(self): conf.initialize_settings() self.blob_dir = tempfile.mkdtemp() self.db_dir = tempfile.mkdtemp() hash_announcer = DummyHashAnnouncer() self.bm = DiskBlobManager(hash_announcer, self.blob_dir, self.db_dir) self.peer = Peer('somehost', 22) def tearDown(self): self.bm.stop() # BlobFile will try to delete itself in _close_writer # thus when calling rmtree we may get a FileNotFoundError # for the blob file shutil.rmtree(self.blob_dir, ignore_errors=True) shutil.rmtree(self.db_dir) @defer.inlineCallbacks def _create_and_add_blob(self): # create and add blob to blob manager data_len = random.randint(1, 1000) data = ''.join( random.choice(string.lowercase) for data_len in range(data_len)) hashobj = get_lbry_hash_obj() hashobj.update(data) out = hashobj.hexdigest() blob_hash = out # create new blob yield self.bm.setup() blob = yield self.bm.get_blob(blob_hash, len(data)) writer, finished_d = yield blob.open_for_writing(self.peer) yield writer.write(data) yield self.bm.blob_completed(blob) yield self.bm.add_blob_to_upload_history(blob_hash, 'test', len(data)) # check to see if blob is there self.assertTrue(os.path.isfile(os.path.join(self.blob_dir, blob_hash))) blobs = yield self.bm.get_all_verified_blobs() self.assertTrue(blob_hash in blobs) defer.returnValue(blob_hash) @defer.inlineCallbacks def test_create_blob(self): blob_hashes = [] # create a bunch of blobs for i in range(0, 10): blob_hash = yield self._create_and_add_blob() blob_hashes.append(blob_hash) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(10, len(blobs)) @defer.inlineCallbacks def test_delete_blob(self): # create blob blob_hash = yield self._create_and_add_blob() blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 1) # delete blob yield self.bm.delete_blobs([blob_hash]) self.assertFalse(os.path.isfile(os.path.join(self.blob_dir, blob_hash))) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 0) blobs = yield self.bm._get_all_blob_hashes() self.assertEqual(len(blobs), 0) # delete blob that does not exist, nothing will # happen blob_hash = random_lbry_hash() out = yield self.bm.delete_blobs([blob_hash]) @defer.inlineCallbacks def test_delete_open_blob(self): # Test that a blob that is opened for writing will not be deleted # create blobs blob_hashes = [] for i in range(0, 10): blob_hash = yield self._create_and_add_blob() blob_hashes.append(blob_hash) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 10) # open the last blob blob = yield self.bm.get_blob(blob_hashes[-1]) writer, finished_d = yield blob.open_for_writing(self.peer) # delete the last blob and check if it still exists out = yield self.bm.delete_blobs([blob_hash]) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 10) self.assertTrue(blob_hashes[-1] in blobs) self.assertTrue( os.path.isfile(os.path.join(self.blob_dir, blob_hashes[-1])))
class BlobManagerTest(unittest.TestCase): def setUp(self): conf.initialize_settings() self.blob_dir = tempfile.mkdtemp() self.db_dir = tempfile.mkdtemp() hash_announcer = DummyHashAnnouncer() self.bm = DiskBlobManager(hash_announcer, self.blob_dir, self.db_dir) self.peer = Peer('somehost', 22) def tearDown(self): self.bm.stop() # BlobFile will try to delete itself in _close_writer # thus when calling rmtree we may get a FileNotFoundError # for the blob file shutil.rmtree(self.blob_dir, ignore_errors=True) shutil.rmtree(self.db_dir) @defer.inlineCallbacks def _create_and_add_blob(self, should_announce=False): # create and add blob to blob manager data_len = random.randint(1, 1000) data = ''.join(random.choice(string.lowercase) for data_len in range(data_len)) hashobj = get_lbry_hash_obj() hashobj.update(data) out = hashobj.hexdigest() blob_hash = out # create new blob yield self.bm.setup() blob = yield self.bm.get_blob(blob_hash, len(data)) writer, finished_d = yield blob.open_for_writing(self.peer) yield writer.write(data) yield self.bm.blob_completed(blob, should_announce) yield self.bm.add_blob_to_upload_history(blob_hash, 'test', len(data)) # check to see if blob is there self.assertTrue(os.path.isfile(os.path.join(self.blob_dir, blob_hash))) blobs = yield self.bm.get_all_verified_blobs() self.assertTrue(blob_hash in blobs) defer.returnValue(blob_hash) @defer.inlineCallbacks def test_create_blob(self): blob_hashes = [] # create a bunch of blobs for i in range(0, 10): blob_hash = yield self._create_and_add_blob() blob_hashes.append(blob_hash) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(10, len(blobs)) @defer.inlineCallbacks def test_delete_blob(self): # create blob blob_hash = yield self._create_and_add_blob() blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 1) # delete blob yield self.bm.delete_blobs([blob_hash]) self.assertFalse(os.path.isfile(os.path.join(self.blob_dir, blob_hash))) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 0) blobs = yield self.bm._get_all_blob_hashes() self.assertEqual(len(blobs), 0) self.assertFalse(blob_hash in self.bm.blobs) # delete blob that was already deleted once out = yield self.bm.delete_blobs([blob_hash]) # delete blob that does not exist, nothing will # happen blob_hash = random_lbry_hash() out = yield self.bm.delete_blobs([blob_hash]) @defer.inlineCallbacks def test_delete_open_blob(self): # Test that a blob that is opened for writing will not be deleted # create blobs blob_hashes = [] for i in range(0, 10): blob_hash = yield self._create_and_add_blob() blob_hashes.append(blob_hash) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 10) # open the last blob blob = yield self.bm.get_blob(blob_hashes[-1]) writer, finished_d = yield blob.open_for_writing(self.peer) # delete the last blob and check if it still exists out = yield self.bm.delete_blobs([blob_hash]) blobs = yield self.bm.get_all_verified_blobs() self.assertEqual(len(blobs), 10) self.assertTrue(blob_hashes[-1] in blobs) self.assertTrue(os.path.isfile(os.path.join(self.blob_dir, blob_hashes[-1]))) @defer.inlineCallbacks def test_should_announce(self): # create blob with should announce blob_hash = yield self._create_and_add_blob(should_announce=True) out = yield self.bm.get_should_announce(blob_hash) self.assertTrue(out) count = yield self.bm.count_should_announce_blobs() self.assertEqual(1, count) # set should annouce to False out = yield self.bm.set_should_announce(blob_hash, should_announce=False) out = yield self.bm.get_should_announce(blob_hash) self.assertFalse(out) count = yield self.bm.count_should_announce_blobs() self.assertEqual(0, count)