def download_stream(sd_blob_hash):
prm = self.session.payment_rate_manager
d = download_sd_blob(self.session, sd_blob_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(create_downloader, prm)
d.addCallback(start_lbry_file)
return d
def download_stream(sd_blob_hash):
prm = self.session.payment_rate_manager
d = download_sd_blob(self.session, sd_blob_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(create_downloader, prm)
d.addCallback(start_lbry_file)
return d
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager,
self.peer_finder, self.rate_limiter,
self.payment_rate_manager,
self.wallet, self.timeout,
conf.settings['download_mirrors'])
defer.returnValue(sd_blob)
def download_file(sd_hash):
prm = self.session.payment_rate_manager
d = download_sd_blob(self.session, sd_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(make_downloader, prm)
d.addCallback(lambda downloader: downloader.start())
return d
def download(self, stream_info, name):
if self._running:
raise Exception("Already running")
self._running = True
self.set_status(INITIALIZING_CODE, name)
self.sd_hash = stream_info.source_hash
if stream_info.has_fee:
try:
fee = yield threads.deferToThread(self.check_fee_and_convert,
stream_info.source_fee)
except Exception as err:
self._running = False
self.finished_deferred.errback(err)
raise err
else:
fee = None
self.set_status(DOWNLOAD_METADATA_CODE, name)
sd_blob = yield download_sd_blob(self.session, self.sd_hash,
self.payment_rate_manager)
stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(
sd_blob)
factory = self.get_downloader_factory(stream_metadata.factories)
self.downloader = yield self.get_downloader(factory, stream_metadata)
self.set_status(DOWNLOAD_RUNNING_CODE, name)
if fee:
yield self.pay_key_fee(fee, name)
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
self.download_path)
self.finished_deferred = self.downloader.start()
self.finished_deferred.addCallback(self.finish, name)
def download_file(sd_hash):
prm = PaymentRateManager(self.session.base_payment_rate_manager)
d = download_sd_blob(self.session, sd_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(make_downloader, prm)
d.addCallback(lambda downloader: downloader.start())
return d
def download_stream(sd_blob_hash):
logging.debug("Downloaded the sd blob. Reading it now")
prm = PaymentRateManager(self.session.base_payment_rate_manager)
d = download_sd_blob(self.session, sd_blob_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(create_downloader, prm)
d.addCallback(start_lbry_file)
return d
def download_file(sd_hash):
prm = self.session.payment_rate_manager
sd_blob = yield download_sd_blob(self.session, sd_hash, prm)
metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob)
downloader = yield make_downloader(metadata, prm)
downloaders.append(downloader)
finished_value = yield downloader.start()
defer.returnValue(finished_value)
def download_file(sd_hash):
prm = self.session.payment_rate_manager
sd_blob = yield download_sd_blob(self.session, sd_hash, prm)
metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob)
downloader = yield make_downloader(metadata, prm)
downloaders.append(downloader)
finished_value = yield downloader.start()
defer.returnValue(finished_value)
def start(self, stream_info, name):
def _cause_timeout(err):
log.info('Cancelling download')
self.timeout_counter = self.timeout * 2
def _set_status(x, status):
log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status))
self.code = next(s for s in STREAM_STAGES if s[0] == status)
return x
def get_downloader_factory(metadata):
for factory in metadata.factories:
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
return factory, metadata
raise Exception('No suitable factory was found in {}'.format(metadata.factories))
def make_downloader(args):
factory, metadata = args
return factory.make_downloader(metadata,
[self.data_rate, True],
self.payment_rate_manager,
download_directory=self.download_directory,
file_name=self.file_name)
self.resolved_name = name
self.stream_info = deepcopy(stream_info)
self.description = self.stream_info['description']
self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
if 'fee' in self.stream_info:
self.fee = FeeValidator(self.stream_info['fee'])
max_key_fee = self._convert_max_fee()
converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount
if converted_fee > self.wallet.wallet_balance:
log.warning("Insufficient funds to download lbry://%s", self.resolved_name)
return defer.fail(InsufficientFundsError())
if converted_fee > max_key_fee:
log.warning(
"Key fee %f above limit of %f didn't download lbry://%s",
converted_fee, max_key_fee, self.resolved_name)
return defer.fail(KeyFeeAboveMaxAllowed())
log.info(
"Key fee %f below limit of %f, downloading lbry://%s",
converted_fee, max_key_fee, self.resolved_name)
self.checker.start(1)
self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
self.d.addCallback(lambda _: download_sd_blob(
self.session, self.stream_hash, self.payment_rate_manager))
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
self.d.addCallback(get_downloader_factory)
self.d.addCallback(make_downloader)
self.d.addCallbacks(self._start_download, _cause_timeout)
self.d.callback(None)
return self.finished
def start(self, stream_info, name):
def _cause_timeout(err):
log.error(err)
log.debug('Forcing a timeout')
self.timeout_counter = self.timeout * 2
def _set_status(x, status):
log.info("Download lbry://%s status changed to %s" %
(self.resolved_name, status))
self.code = next(s for s in STREAM_STAGES if s[0] == status)
return x
def get_downloader_factory(metadata):
for factory in metadata.factories:
if isinstance(factory, ManagedLBRYFileDownloaderFactory):
return factory, metadata
raise Exception('No suitable factory was found in {}'.format(
metadata.factories))
def make_downloader(args):
factory, metadata = args
return factory.make_downloader(
metadata, [self.data_rate, True],
self.payment_rate_manager,
download_directory=self.download_directory,
file_name=self.file_name)
self.resolved_name = name
self.stream_info = deepcopy(stream_info)
self.description = self.stream_info['description']
self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
if 'fee' in self.stream_info:
self.fee = LBRYFeeValidator(self.stream_info['fee'])
max_key_fee = self._convert_max_fee()
if self.exchange_rate_manager.to_lbc(
self.fee).amount > max_key_fee:
log.info(
"Key fee %f above limit of %f didn't download lbry://%s" %
(self.fee.amount, self.max_key_fee, self.resolved_name))
return defer.fail(KeyFeeAboveMaxAllowed())
log.info("Key fee %s below limit of %f, downloading lbry://%s" %
(json.dumps(self.fee), max_key_fee, self.resolved_name))
self.checker.start(1)
self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
self.d.addCallback(lambda _: download_sd_blob(
self.session, self.stream_hash, self.payment_rate_manager))
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
self.d.addCallback(get_downloader_factory)
self.d.addCallback(make_downloader)
self.d.addCallbacks(self._start_download, _cause_timeout)
self.d.callback(None)
return self.finished
def start(self, stream_info):
self.stream_info = stream_info
if 'stream_hash' in self.stream_info.keys():
self.description = self.stream_info['description']
if 'key_fee' in self.stream_info.keys():
self.key_fee = float(self.stream_info['key_fee'])
if 'key_fee_address' in self.stream_info.keys():
self.key_fee_address = self.stream_info['key_fee_address']
else:
self.key_fee_address = None
else:
self.key_fee = None
self.key_fee_address = None
self.stream_hash = self.stream_info['stream_hash']
elif 'stream_hash' in json.loads(self.stream_info['value']):
self.resolved_name = self.stream_info.get('name', None)
self.description = json.loads(self.stream_info['value']).get('description', None)
try:
if 'key_fee' in json.loads(self.stream_info['value']):
self.key_fee = float(json.loads(self.stream_info['value'])['key_fee'])
except ValueError:
self.key_fee = None
self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None)
self.stream_hash = json.loads(self.stream_info['value'])['stream_hash']
else:
print 'InvalidStreamInfoError'
raise InvalidStreamInfoError(self.stream_info)
if self.key_fee > self.max_key_fee:
if self.pay_key:
print "Key fee (" + str(self.key_fee) + ") above limit of " + str(
self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)
return self.finished_deferred.callback(None)
else:
pass
def _get_downloader_for_return():
return defer.succeed(self.downloader)
self.loading_metadata_deferred = defer.Deferred(None)
self.loading_metadata_deferred.addCallback(
lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.loading_metadata_deferred.addCallback(self._handle_metadata)
self.loading_metadata_deferred.addErrback(self._handle_load_canceled)
self.loading_metadata_deferred.addErrback(self._handle_load_failed)
if self.pay_key:
self.loading_metadata_deferred.addCallback(lambda _: self._pay_key_fee())
self.loading_metadata_deferred.addCallback(lambda _: self._make_downloader())
self.loading_metadata_deferred.addCallback(lambda _: self.downloader.start())
self.loading_metadata_deferred.addErrback(self._handle_download_error)
self.loading_metadata_deferred.addCallback(lambda _: _get_downloader_for_return())
self.loading_metadata_deferred.callback(None)
return defer.succeed(None)
def start(self, stream_info, name):
self.resolved_name = name
self.stream_info = stream_info
if 'stream_hash' in self.stream_info.keys():
self.stream_hash = self.stream_info['stream_hash']
elif 'sources' in self.stream_info.keys():
self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
else:
raise InvalidStreamInfoError(self.stream_info)
if 'description' in self.stream_info.keys():
self.description = self.stream_info['description']
if 'key_fee' in self.stream_info.keys():
self.key_fee = float(self.stream_info['key_fee'])
if 'key_fee_address' in self.stream_info.keys():
self.key_fee_address = self.stream_info['key_fee_address']
else:
self.key_fee_address = None
else:
self.key_fee = None
self.key_fee_address = None
if self.key_fee > self.max_key_fee:
if self.pay_key:
log.info("Key fee (" + str(self.key_fee) + ") above limit of " + str(
self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name))
return defer.fail(None)
else:
pass
def _cause_timeout():
self.timeout_counter = self.timeout * 2
def _set_status(x, status):
self.code = next(s for s in STREAM_STAGES if s[0] == status)
return x
self.checker.start(1)
self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
self.d.addCallback(lambda metadata: (
next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)),
metadata))
self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata,
[self.data_rate, True],
self.payment_rate_manager,
download_directory=self.download_directory,
file_name=self.file_name))
self.d.addCallbacks(self._start_download, lambda _: _cause_timeout())
self.d.callback(None)
return self.finished
def download(self, name, key_fee):
# download sd blob, and start downloader
self.set_status(DOWNLOAD_METADATA_CODE, name)
sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager)
self.downloader = yield self._create_downloader(sd_blob)
self.set_status(DOWNLOAD_RUNNING_CODE, name)
if key_fee:
yield self.pay_key_fee(key_fee, name)
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
self.finished_deferred = self.downloader.start()
self.finished_deferred.addCallback(self.finish, name)
def test_lbry_transfer(self):
sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager,
self.peer_finder, self.rate_limiter,
self.prm, self.wallet)
metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
downloader = yield metadata.factories[0].make_downloader(
metadata,
self.prm.min_blob_data_payment_rate,
self.prm,
self.db_dir,
download_mirrors=None)
yield downloader.start()
with open(os.path.join(self.db_dir, 'test_file'), 'rb') as f:
hashsum = md5()
hashsum.update(f.read())
self.assertEqual(hashsum.hexdigest(),
"4ca2aafb4101c1e42235aad24fbb83be")
def download(self, stream_info, name):
self.set_status(INITIALIZING_CODE, name)
self.sd_hash = stream_info['sources']['lbry_sd_hash']
if 'fee' in stream_info:
fee = self.check_fee(stream_info['fee'])
else:
fee = None
self.set_status(DOWNLOAD_METADATA_CODE, name)
sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager)
stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
factory = self.get_downloader_factory(stream_metadata.factories)
self.downloader = yield self.get_downloader(factory, stream_metadata)
self.set_status(DOWNLOAD_RUNNING_CODE, name)
if fee:
yield self.pay_key_fee(fee, name)
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
self.finished_deferred = self.downloader.start()
self.finished_deferred.addCallback(self.finish, name)
def start(self, stream_info):
self.stream_info = stream_info
if 'stream_hash' in self.stream_info.keys():
self.description = self.stream_info['description']
if 'key_fee' in self.stream_info.keys():
self.key_fee = float(self.stream_info['key_fee'])
if 'key_fee_address' in self.stream_info.keys():
self.key_fee_address = self.stream_info['key_fee_address']
else:
self.key_fee_address = None
else:
self.key_fee = None
self.key_fee_address = None
self.stream_hash = self.stream_info['stream_hash']
else:
print 'InvalidStreamInfoError'
raise InvalidStreamInfoError(self.stream_info)
if self.key_fee > self.max_key_fee:
if self.pay_key:
print "Key fee (" + str(self.key_fee) + ") above limit of " + str(
self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)
return defer.fail(None)
else:
pass
d = defer.Deferred(None)
d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
d.addCallback(lambda metadata:
metadata.factories[1].make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager))
d.addErrback(lambda err: err.trap(defer.CancelledError))
d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
d.addCallback(self._start_download)
d.callback(None)
return d
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(self.session, self.sd_hash,
self.payment_rate_manager,
self.timeout)
defer.returnValue(sd_blob)
def start(self, stream_info, name):
def _cancel(err):
# this callback sequence gets cancelled in check_status if
# it takes too long when that happens, we want the logic
# to live in check_status
if err.check(defer.CancelledError):
return
if self.checker:
self.checker.stop()
self.finished.errback(err)
def _set_status(x, status):
log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status))
self.code = next(s for s in STREAM_STAGES if s[0] == status)
return x
def get_downloader_factory(metadata):
for factory in metadata.factories:
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
return factory, metadata
raise Exception('No suitable factory was found in {}'.format(metadata.factories))
def make_downloader(args):
factory, metadata = args
return factory.make_downloader(metadata,
[self.data_rate, True],
self.payment_rate_manager,
download_directory=self.download_directory,
file_name=self.file_name)
self.resolved_name = name
self.stream_info = deepcopy(stream_info)
self.description = self.stream_info['description']
self.sd_hash = self.stream_info['sources']['lbry_sd_hash']
if 'fee' in self.stream_info:
self.fee = FeeValidator(self.stream_info['fee'])
max_key_fee = self._convert_max_fee()
converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount
if converted_fee > self.wallet.get_balance():
msg = "Insufficient funds to download lbry://{}. Need {:0.2f}, have {:0.2f}".format(
self.resolved_name, converted_fee, self.wallet.get_balance())
raise InsufficientFundsError(msg)
if converted_fee > max_key_fee:
msg = "Key fee {:0.2f} above limit of {:0.2f} didn't download lbry://{}".format(
converted_fee, max_key_fee, self.resolved_name)
raise KeyFeeAboveMaxAllowed(msg)
log.info(
"Key fee %f below limit of %f, downloading lbry://%s",
converted_fee, max_key_fee, self.resolved_name)
self.checker.start(1)
self._d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
self._d.addCallback(lambda _: download_sd_blob(
self.session, self.sd_hash, self.payment_rate_manager))
self._d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self._d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
self._d.addCallback(get_downloader_factory)
self._d.addCallback(make_downloader)
self._d.addCallbacks(self._start_download, _cancel)
self._d.callback(None)
return self.finished
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(self.session, self.sd_hash,
self.payment_rate_manager, self.timeout)
defer.returnValue(sd_blob)
