def download_stream(sd_blob_hash): prm = self.session.payment_rate_manager d = download_sd_blob(self.session, sd_blob_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(create_downloader, prm) d.addCallback(start_lbry_file) return d
def _download_sd_blob(self): sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager, self.wallet, self.timeout, conf.settings['download_mirrors']) defer.returnValue(sd_blob)
def download_file(sd_hash): prm = self.session.payment_rate_manager d = download_sd_blob(self.session, sd_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(make_downloader, prm) d.addCallback(lambda downloader: downloader.start()) return d
def download(self, stream_info, name): if self._running: raise Exception("Already running") self._running = True self.set_status(INITIALIZING_CODE, name) self.sd_hash = stream_info.source_hash if stream_info.has_fee: try: fee = yield threads.deferToThread(self.check_fee_and_convert, stream_info.source_fee) except Exception as err: self._running = False self.finished_deferred.errback(err) raise err else: fee = None self.set_status(DOWNLOAD_METADATA_CODE, name) sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager) stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob( sd_blob) factory = self.get_downloader_factory(stream_metadata.factories) self.downloader = yield self.get_downloader(factory, stream_metadata) self.set_status(DOWNLOAD_RUNNING_CODE, name) if fee: yield self.pay_key_fee(fee, name) log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path) self.finished_deferred = self.downloader.start() self.finished_deferred.addCallback(self.finish, name)
def download_file(sd_hash): prm = PaymentRateManager(self.session.base_payment_rate_manager) d = download_sd_blob(self.session, sd_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(make_downloader, prm) d.addCallback(lambda downloader: downloader.start()) return d
def download_stream(sd_blob_hash): logging.debug("Downloaded the sd blob. Reading it now") prm = PaymentRateManager(self.session.base_payment_rate_manager) d = download_sd_blob(self.session, sd_blob_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(create_downloader, prm) d.addCallback(start_lbry_file) return d
def download_file(sd_hash): prm = self.session.payment_rate_manager sd_blob = yield download_sd_blob(self.session, sd_hash, prm) metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob) downloader = yield make_downloader(metadata, prm) downloaders.append(downloader) finished_value = yield downloader.start() defer.returnValue(finished_value)
def start(self, stream_info, name): def _cause_timeout(err): log.info('Cancelling download') self.timeout_counter = self.timeout * 2 def _set_status(x, status): log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status)) self.code = next(s for s in STREAM_STAGES if s[0] == status) return x def get_downloader_factory(metadata): for factory in metadata.factories: if isinstance(factory, ManagedEncryptedFileDownloaderFactory): return factory, metadata raise Exception('No suitable factory was found in {}'.format(metadata.factories)) def make_downloader(args): factory, metadata = args return factory.make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager, download_directory=self.download_directory, file_name=self.file_name) self.resolved_name = name self.stream_info = deepcopy(stream_info) self.description = self.stream_info['description'] self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] if 'fee' in self.stream_info: self.fee = FeeValidator(self.stream_info['fee']) max_key_fee = self._convert_max_fee() converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount if converted_fee > self.wallet.wallet_balance: log.warning("Insufficient funds to download lbry://%s", self.resolved_name) return defer.fail(InsufficientFundsError()) if converted_fee > max_key_fee: log.warning( "Key fee %f above limit of %f didn't download lbry://%s", converted_fee, max_key_fee, self.resolved_name) return defer.fail(KeyFeeAboveMaxAllowed()) log.info( "Key fee %f below limit of %f, downloading lbry://%s", converted_fee, max_key_fee, self.resolved_name) self.checker.start(1) self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) self.d.addCallback(lambda _: download_sd_blob( self.session, self.stream_hash, self.payment_rate_manager)) self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE)) self.d.addCallback(get_downloader_factory) self.d.addCallback(make_downloader) self.d.addCallbacks(self._start_download, _cause_timeout) self.d.callback(None) return self.finished
def start(self, stream_info, name): def _cause_timeout(err): log.error(err) log.debug('Forcing a timeout') self.timeout_counter = self.timeout * 2 def _set_status(x, status): log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status)) self.code = next(s for s in STREAM_STAGES if s[0] == status) return x def get_downloader_factory(metadata): for factory in metadata.factories: if isinstance(factory, ManagedLBRYFileDownloaderFactory): return factory, metadata raise Exception('No suitable factory was found in {}'.format( metadata.factories)) def make_downloader(args): factory, metadata = args return factory.make_downloader( metadata, [self.data_rate, True], self.payment_rate_manager, download_directory=self.download_directory, file_name=self.file_name) self.resolved_name = name self.stream_info = deepcopy(stream_info) self.description = self.stream_info['description'] self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] if 'fee' in self.stream_info: self.fee = LBRYFeeValidator(self.stream_info['fee']) max_key_fee = self._convert_max_fee() if self.exchange_rate_manager.to_lbc( self.fee).amount > max_key_fee: log.info( "Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount, self.max_key_fee, self.resolved_name)) return defer.fail(KeyFeeAboveMaxAllowed()) log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee), max_key_fee, self.resolved_name)) self.checker.start(1) self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) self.d.addCallback(lambda _: download_sd_blob( self.session, self.stream_hash, self.payment_rate_manager)) self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE)) self.d.addCallback(get_downloader_factory) self.d.addCallback(make_downloader) self.d.addCallbacks(self._start_download, _cause_timeout) self.d.callback(None) return self.finished
def start(self, stream_info): self.stream_info = stream_info if 'stream_hash' in self.stream_info.keys(): self.description = self.stream_info['description'] if 'key_fee' in self.stream_info.keys(): self.key_fee = float(self.stream_info['key_fee']) if 'key_fee_address' in self.stream_info.keys(): self.key_fee_address = self.stream_info['key_fee_address'] else: self.key_fee_address = None else: self.key_fee = None self.key_fee_address = None self.stream_hash = self.stream_info['stream_hash'] elif 'stream_hash' in json.loads(self.stream_info['value']): self.resolved_name = self.stream_info.get('name', None) self.description = json.loads(self.stream_info['value']).get('description', None) try: if 'key_fee' in json.loads(self.stream_info['value']): self.key_fee = float(json.loads(self.stream_info['value'])['key_fee']) except ValueError: self.key_fee = None self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None) self.stream_hash = json.loads(self.stream_info['value'])['stream_hash'] else: print 'InvalidStreamInfoError' raise InvalidStreamInfoError(self.stream_info) if self.key_fee > self.max_key_fee: if self.pay_key: print "Key fee (" + str(self.key_fee) + ") above limit of " + str( self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name) return self.finished_deferred.callback(None) else: pass def _get_downloader_for_return(): return defer.succeed(self.downloader) self.loading_metadata_deferred = defer.Deferred(None) self.loading_metadata_deferred.addCallback( lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager)) self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob) self.loading_metadata_deferred.addCallback(self._handle_metadata) self.loading_metadata_deferred.addErrback(self._handle_load_canceled) self.loading_metadata_deferred.addErrback(self._handle_load_failed) if self.pay_key: self.loading_metadata_deferred.addCallback(lambda _: self._pay_key_fee()) self.loading_metadata_deferred.addCallback(lambda _: self._make_downloader()) self.loading_metadata_deferred.addCallback(lambda _: self.downloader.start()) self.loading_metadata_deferred.addErrback(self._handle_download_error) self.loading_metadata_deferred.addCallback(lambda _: _get_downloader_for_return()) self.loading_metadata_deferred.callback(None) return defer.succeed(None)
def start(self, stream_info, name): self.resolved_name = name self.stream_info = stream_info if 'stream_hash' in self.stream_info.keys(): self.stream_hash = self.stream_info['stream_hash'] elif 'sources' in self.stream_info.keys(): self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] else: raise InvalidStreamInfoError(self.stream_info) if 'description' in self.stream_info.keys(): self.description = self.stream_info['description'] if 'key_fee' in self.stream_info.keys(): self.key_fee = float(self.stream_info['key_fee']) if 'key_fee_address' in self.stream_info.keys(): self.key_fee_address = self.stream_info['key_fee_address'] else: self.key_fee_address = None else: self.key_fee = None self.key_fee_address = None if self.key_fee > self.max_key_fee: if self.pay_key: log.info("Key fee (" + str(self.key_fee) + ") above limit of " + str( self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)) return defer.fail(None) else: pass def _cause_timeout(): self.timeout_counter = self.timeout * 2 def _set_status(x, status): self.code = next(s for s in STREAM_STAGES if s[0] == status) return x self.checker.start(1) self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager)) self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE)) self.d.addCallback(lambda metadata: ( next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), metadata)) self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager, download_directory=self.download_directory, file_name=self.file_name)) self.d.addCallbacks(self._start_download, lambda _: _cause_timeout()) self.d.callback(None) return self.finished
def download(self, name, key_fee): # download sd blob, and start downloader self.set_status(DOWNLOAD_METADATA_CODE, name) sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager) self.downloader = yield self._create_downloader(sd_blob) self.set_status(DOWNLOAD_RUNNING_CODE, name) if key_fee: yield self.pay_key_fee(key_fee, name) log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path) self.finished_deferred = self.downloader.start() self.finished_deferred.addCallback(self.finish, name)
def test_lbry_transfer(self): sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.prm, self.wallet) metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob) downloader = yield metadata.factories[0].make_downloader( metadata, self.prm.min_blob_data_payment_rate, self.prm, self.db_dir, download_mirrors=None) yield downloader.start() with open(os.path.join(self.db_dir, 'test_file'), 'rb') as f: hashsum = md5() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "4ca2aafb4101c1e42235aad24fbb83be")
def download(self, stream_info, name): self.set_status(INITIALIZING_CODE, name) self.sd_hash = stream_info['sources']['lbry_sd_hash'] if 'fee' in stream_info: fee = self.check_fee(stream_info['fee']) else: fee = None self.set_status(DOWNLOAD_METADATA_CODE, name) sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager) stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob) factory = self.get_downloader_factory(stream_metadata.factories) self.downloader = yield self.get_downloader(factory, stream_metadata) self.set_status(DOWNLOAD_RUNNING_CODE, name) if fee: yield self.pay_key_fee(fee, name) log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path) self.finished_deferred = self.downloader.start() self.finished_deferred.addCallback(self.finish, name)
def start(self, stream_info): self.stream_info = stream_info if 'stream_hash' in self.stream_info.keys(): self.description = self.stream_info['description'] if 'key_fee' in self.stream_info.keys(): self.key_fee = float(self.stream_info['key_fee']) if 'key_fee_address' in self.stream_info.keys(): self.key_fee_address = self.stream_info['key_fee_address'] else: self.key_fee_address = None else: self.key_fee = None self.key_fee_address = None self.stream_hash = self.stream_info['stream_hash'] else: print 'InvalidStreamInfoError' raise InvalidStreamInfoError(self.stream_info) if self.key_fee > self.max_key_fee: if self.pay_key: print "Key fee (" + str(self.key_fee) + ") above limit of " + str( self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name) return defer.fail(None) else: pass d = defer.Deferred(None) d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager)) d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) d.addCallback(lambda metadata: metadata.factories[1].make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager)) d.addErrback(lambda err: err.trap(defer.CancelledError)) d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback())) d.addCallback(self._start_download) d.callback(None) return d
def _download_sd_blob(self): sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager, self.timeout) defer.returnValue(sd_blob)
def start(self, stream_info, name): def _cancel(err): # this callback sequence gets cancelled in check_status if # it takes too long when that happens, we want the logic # to live in check_status if err.check(defer.CancelledError): return if self.checker: self.checker.stop() self.finished.errback(err) def _set_status(x, status): log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status)) self.code = next(s for s in STREAM_STAGES if s[0] == status) return x def get_downloader_factory(metadata): for factory in metadata.factories: if isinstance(factory, ManagedEncryptedFileDownloaderFactory): return factory, metadata raise Exception('No suitable factory was found in {}'.format(metadata.factories)) def make_downloader(args): factory, metadata = args return factory.make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager, download_directory=self.download_directory, file_name=self.file_name) self.resolved_name = name self.stream_info = deepcopy(stream_info) self.description = self.stream_info['description'] self.sd_hash = self.stream_info['sources']['lbry_sd_hash'] if 'fee' in self.stream_info: self.fee = FeeValidator(self.stream_info['fee']) max_key_fee = self._convert_max_fee() converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount if converted_fee > self.wallet.get_balance(): msg = "Insufficient funds to download lbry://{}. Need {:0.2f}, have {:0.2f}".format( self.resolved_name, converted_fee, self.wallet.get_balance()) raise InsufficientFundsError(msg) if converted_fee > max_key_fee: msg = "Key fee {:0.2f} above limit of {:0.2f} didn't download lbry://{}".format( converted_fee, max_key_fee, self.resolved_name) raise KeyFeeAboveMaxAllowed(msg) log.info( "Key fee %f below limit of %f, downloading lbry://%s", converted_fee, max_key_fee, self.resolved_name) self.checker.start(1) self._d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) self._d.addCallback(lambda _: download_sd_blob( self.session, self.sd_hash, self.payment_rate_manager)) self._d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) self._d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE)) self._d.addCallback(get_downloader_factory) self._d.addCallback(make_downloader) self._d.addCallbacks(self._start_download, _cancel) self._d.callback(None) return self.finished