def setUp(self):
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.blob_manager = MagicMock()
self.client = MagicMock()
self.blob_hash = ('d17272b17a1ad61c4316ac13a651c2b0952063214a81333e'
'838364b01b2f07edbd165bb7ec60d2fb2f337a2c02923852')
self.blob = BlobFile(self.blob_dir, self.blob_hash)
self.blob_manager.get_blob.side_effect = lambda _: self.blob
self.response = MagicMock(code=200, length=400)
self.client.get.side_effect = lambda uri: defer.succeed(self.response)
self.downloader = HTTPBlobDownloader(self.blob_manager,
[self.blob_hash], ['server1'],
self.client,
retry=False)
self.downloader.interval = 0
def __init__(self,
rowid,
stream_hash,
peer_finder,
rate_limiter,
blob_manager,
storage,
lbry_file_manager,
payment_rate_manager,
wallet,
download_directory,
file_name,
stream_name,
sd_hash,
key,
suggested_file_name,
download_mirrors=None):
super().__init__(stream_hash, peer_finder, rate_limiter, blob_manager,
storage, payment_rate_manager, wallet,
download_directory, key, stream_name, file_name)
self.sd_hash = sd_hash
self.rowid = rowid
self.suggested_file_name = unhexlify(suggested_file_name).decode()
self.lbry_file_manager = lbry_file_manager
self._saving_status = False
self.claim_id = None
self.outpoint = None
self.claim_name = None
self.txid = None
self.nout = None
self.channel_claim_id = None
self.channel_name = None
self.metadata = None
self.mirror = None
if download_mirrors or conf.settings['download_mirrors']:
self.mirror = HTTPBlobDownloader(
self.blob_manager,
servers=download_mirrors or conf.settings['download_mirrors'])
def download_sd_blob(blob_hash,
blob_manager,
peer_finder,
rate_limiter,
payment_rate_manager,
wallet,
timeout=None,
download_mirrors=None):
"""
Downloads a single blob from the network
@param session:
@param blob_hash:
@param payment_rate_manager:
@return: An object of type HashBlob
"""
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
rate_limiter, payment_rate_manager,
wallet, timeout)
mirror = HTTPBlobDownloader(blob_manager, [blob_hash],
download_mirrors or [],
sd_hashes=[blob_hash],
retry=False)
mirror.start()
sd_blob = yield downloader.download()
mirror.stop()
sd_reader = BlobStreamDescriptorReader(sd_blob)
sd_info = yield sd_reader.get_info()
try:
validate_descriptor(sd_info)
except InvalidStreamDescriptorError as err:
yield blob_manager.delete_blobs([blob_hash])
raise err
raw_sd = yield sd_reader._get_raw_data()
yield blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
defer.returnValue(sd_blob)
class ManagedEncryptedFileDownloader(EncryptedFileSaver):
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self,
rowid,
stream_hash,
peer_finder,
rate_limiter,
blob_manager,
storage,
lbry_file_manager,
payment_rate_manager,
wallet,
download_directory,
file_name,
stream_name,
sd_hash,
key,
suggested_file_name,
download_mirrors=None):
super().__init__(stream_hash, peer_finder, rate_limiter, blob_manager,
storage, payment_rate_manager, wallet,
download_directory, key, stream_name, file_name)
self.sd_hash = sd_hash
self.rowid = rowid
self.suggested_file_name = unhexlify(suggested_file_name).decode()
self.lbry_file_manager = lbry_file_manager
self._saving_status = False
self.claim_id = None
self.outpoint = None
self.claim_name = None
self.txid = None
self.nout = None
self.channel_claim_id = None
self.channel_name = None
self.metadata = None
self.mirror = None
if download_mirrors or conf.settings['download_mirrors']:
self.mirror = HTTPBlobDownloader(
self.blob_manager,
servers=download_mirrors or conf.settings['download_mirrors'])
def set_claim_info(self, claim_info):
self.claim_id = claim_info['claim_id']
self.txid = claim_info['txid']
self.nout = claim_info['nout']
self.channel_claim_id = claim_info['channel_claim_id']
self.outpoint = "%s:%i" % (self.txid, self.nout)
self.claim_name = claim_info['name']
self.channel_name = claim_info['channel_name']
self.metadata = claim_info['value']['stream']['metadata']
@defer.inlineCallbacks
def get_claim_info(self, include_supports=True):
claim_info = yield self.storage.get_content_claim(
self.stream_hash, include_supports)
if claim_info:
self.set_claim_info(claim_info)
defer.returnValue(claim_info)
@property
def saving_status(self):
return self._saving_status
def restore(self, status):
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
# start returns self.finished_deferred
# which fires when we've finished downloading the file
# and we don't want to wait for the entire download
self.start()
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
pass
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
self.completed = True
else:
raise Exception(
f"Unknown status for stream {self.stream_hash}: {status}")
@defer.inlineCallbacks
def stop(self, err=None, change_status=True):
log.debug('Stopping download for stream %s',
short_hash(self.stream_hash))
if self.mirror:
self.mirror.stop()
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
yield EncryptedFileDownloader.stop(self, err=err)
if change_status is True:
status = yield self._save_status()
defer.returnValue(status)
@defer.inlineCallbacks
def status(self):
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
num_blobs_completed = len(completed_blobs)
num_blobs_known = len(blob_hashes)
if self.completed:
status = "completed"
elif self.stopped:
status = "stopped"
else:
status = "running"
defer.returnValue(
EncryptedFileStatusReport(self.file_name, num_blobs_completed,
num_blobs_known, status))
@defer.inlineCallbacks
def _start(self):
yield EncryptedFileSaver._start(self)
status = yield self._save_status()
log_status(self.sd_hash, status)
if self.mirror:
self.mirror.download_stream(self.stream_hash, self.sd_hash)
defer.returnValue(status)
def _get_finished_deferred_callback_value(self):
if self.completed is True:
return "Download successful"
else:
return "Download stopped"
@defer.inlineCallbacks
def _save_status(self):
self._saving_status = True
if self.completed is True:
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
elif self.stopped is True:
status = ManagedEncryptedFileDownloader.STATUS_STOPPED
else:
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
status = yield self.lbry_file_manager.change_lbry_file_status(
self, status)
self._saving_status = False
defer.returnValue(status)
def save_status(self):
return self._save_status()
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager, download_manager)
class HTTPBlobDownloaderTest(unittest.TestCase):
def setUp(self):
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.blob_manager = MagicMock()
self.client = MagicMock()
self.blob_hash = ('d17272b17a1ad61c4316ac13a651c2b0952063214a81333e'
'838364b01b2f07edbd165bb7ec60d2fb2f337a2c02923852')
self.blob = BlobFile(self.blob_dir, self.blob_hash)
self.blob_manager.get_blob.side_effect = lambda _: defer.succeed(self.
blob)
self.response = MagicMock(code=200, length=400)
self.client.get.side_effect = lambda uri: defer.succeed(self.response)
self.downloader = HTTPBlobDownloader(self.blob_manager,
[self.blob_hash], ['server1'],
self.client,
retry=False)
self.downloader.interval = 0
def tearDown(self):
self.downloader.stop()
rm_db_and_blob_dir(self.db_dir, self.blob_dir)
@defer.inlineCallbacks
def test_download_successful(self):
self.client.collect.side_effect = collect
yield self.downloader.start()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format(
'server1', self.blob_hash))
self.client.collect.assert_called()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertTrue(self.blob.get_is_verified())
self.assertEqual(self.blob.writers, {})
@defer.inlineCallbacks
def test_download_invalid_content(self):
self.client.collect.side_effect = bad_collect
yield self.downloader.start()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertFalse(self.blob.get_is_verified())
self.assertEqual(self.blob.writers, {})
@defer.inlineCallbacks
def test_peer_finished_first_causing_a_write_on_closed_handle(self):
self.client.collect.side_effect = lambda response, write: defer.fail(
IOError('I/O operation on closed file'))
yield self.downloader.start()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format(
'server1', self.blob_hash))
self.client.collect.assert_called()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertEqual(self.blob.writers, {})
@defer.inlineCallbacks
def test_download_transfer_failed(self):
self.client.collect.side_effect = lambda response, write: defer.fail(
Exception())
yield self.downloader.start()
self.assertEqual(len(self.client.collect.mock_calls),
self.downloader.max_failures)
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertFalse(self.blob.get_is_verified())
self.assertEqual(self.blob.writers, {})
@defer.inlineCallbacks
def test_blob_not_found(self):
self.response.code = 404
yield self.downloader.start()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format(
'server1', self.blob_hash))
self.client.collect.assert_not_called()
self.assertFalse(self.blob.get_is_verified())
self.assertEqual(self.blob.writers, {})
def test_stop(self):
self.client.collect.side_effect = lambda response, write: defer.Deferred(
)
self.downloader.start(
) # hangs if yielded, as intended, to simulate a long ongoing write while we call stop
self.downloader.stop()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format(
'server1', self.blob_hash))
self.client.collect.assert_called()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertFalse(self.blob.get_is_verified())
self.assertEqual(self.blob.writers, {})