Пример #1
0
 def download_stream(sd_blob_hash):
     prm = self.session.payment_rate_manager
     d = download_sd_blob(self.session, sd_blob_hash, prm)
     d.addCallback(sd_identifier.get_metadata_for_sd_blob)
     d.addCallback(create_downloader, prm)
     d.addCallback(start_lbry_file)
     return d
Пример #2
0
 def download_stream(sd_blob_hash):
     prm = self.session.payment_rate_manager
     d = download_sd_blob(self.session, sd_blob_hash, prm)
     d.addCallback(sd_identifier.get_metadata_for_sd_blob)
     d.addCallback(create_downloader, prm)
     d.addCallback(start_lbry_file)
     return d
Пример #3
0
 def _download_sd_blob(self):
     sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager,
                                      self.peer_finder, self.rate_limiter,
                                      self.payment_rate_manager,
                                      self.wallet, self.timeout,
                                      conf.settings['download_mirrors'])
     defer.returnValue(sd_blob)
Пример #4
0
 def download_file(sd_hash):
     prm = self.session.payment_rate_manager
     d = download_sd_blob(self.session, sd_hash, prm)
     d.addCallback(sd_identifier.get_metadata_for_sd_blob)
     d.addCallback(make_downloader, prm)
     d.addCallback(lambda downloader: downloader.start())
     return d
Пример #5
0
    def download(self, stream_info, name):
        if self._running:
            raise Exception("Already running")
        self._running = True

        self.set_status(INITIALIZING_CODE, name)
        self.sd_hash = stream_info.source_hash

        if stream_info.has_fee:
            try:
                fee = yield threads.deferToThread(self.check_fee_and_convert,
                                                  stream_info.source_fee)
            except Exception as err:
                self._running = False
                self.finished_deferred.errback(err)
                raise err
        else:
            fee = None

        self.set_status(DOWNLOAD_METADATA_CODE, name)
        sd_blob = yield download_sd_blob(self.session, self.sd_hash,
                                         self.payment_rate_manager)
        stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(
            sd_blob)
        factory = self.get_downloader_factory(stream_metadata.factories)
        self.downloader = yield self.get_downloader(factory, stream_metadata)

        self.set_status(DOWNLOAD_RUNNING_CODE, name)
        if fee:
            yield self.pay_key_fee(fee, name)
        log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
                 self.download_path)
        self.finished_deferred = self.downloader.start()
        self.finished_deferred.addCallback(self.finish, name)
Пример #6
0
 def download_file(sd_hash):
     prm = PaymentRateManager(self.session.base_payment_rate_manager)
     d = download_sd_blob(self.session, sd_hash, prm)
     d.addCallback(sd_identifier.get_metadata_for_sd_blob)
     d.addCallback(make_downloader, prm)
     d.addCallback(lambda downloader: downloader.start())
     return d
Пример #7
0
 def download_stream(sd_blob_hash):
     logging.debug("Downloaded the sd blob. Reading it now")
     prm = PaymentRateManager(self.session.base_payment_rate_manager)
     d = download_sd_blob(self.session, sd_blob_hash, prm)
     d.addCallback(sd_identifier.get_metadata_for_sd_blob)
     d.addCallback(create_downloader, prm)
     d.addCallback(start_lbry_file)
     return d
Пример #8
0
 def download_file(sd_hash):
     prm = self.session.payment_rate_manager
     sd_blob = yield download_sd_blob(self.session, sd_hash, prm)
     metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob)
     downloader = yield make_downloader(metadata, prm)
     downloaders.append(downloader)
     finished_value = yield downloader.start()
     defer.returnValue(finished_value)
Пример #9
0
 def download_file(sd_hash):
     prm = self.session.payment_rate_manager
     sd_blob = yield download_sd_blob(self.session, sd_hash, prm)
     metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob)
     downloader = yield make_downloader(metadata, prm)
     downloaders.append(downloader)
     finished_value = yield downloader.start()
     defer.returnValue(finished_value)
Пример #10
0
    def start(self, stream_info, name):
        def _cause_timeout(err):
            log.info('Cancelling download')
            self.timeout_counter = self.timeout * 2

        def _set_status(x, status):
            log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status))
            self.code = next(s for s in STREAM_STAGES if s[0] == status)
            return x

        def get_downloader_factory(metadata):
            for factory in metadata.factories:
                if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
                    return factory, metadata
            raise Exception('No suitable factory was found in {}'.format(metadata.factories))

        def make_downloader(args):
            factory, metadata = args
            return factory.make_downloader(metadata,
                                           [self.data_rate, True],
                                           self.payment_rate_manager,
                                           download_directory=self.download_directory,
                                           file_name=self.file_name)

        self.resolved_name = name
        self.stream_info = deepcopy(stream_info)
        self.description = self.stream_info['description']
        self.stream_hash = self.stream_info['sources']['lbry_sd_hash']

        if 'fee' in self.stream_info:
            self.fee = FeeValidator(self.stream_info['fee'])
            max_key_fee = self._convert_max_fee()
            converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount
            if converted_fee > self.wallet.wallet_balance:
                log.warning("Insufficient funds to download lbry://%s", self.resolved_name)
                return defer.fail(InsufficientFundsError())
            if converted_fee > max_key_fee:
                log.warning(
                    "Key fee %f above limit of %f didn't download lbry://%s",
                    converted_fee, max_key_fee, self.resolved_name)
                return defer.fail(KeyFeeAboveMaxAllowed())
            log.info(
                "Key fee %f below limit of %f, downloading lbry://%s",
                converted_fee, max_key_fee, self.resolved_name)

        self.checker.start(1)

        self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
        self.d.addCallback(lambda _: download_sd_blob(
            self.session, self.stream_hash, self.payment_rate_manager))
        self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
        self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
        self.d.addCallback(get_downloader_factory)
        self.d.addCallback(make_downloader)
        self.d.addCallbacks(self._start_download, _cause_timeout)
        self.d.callback(None)

        return self.finished
Пример #11
0
    def start(self, stream_info, name):
        def _cause_timeout(err):
            log.error(err)
            log.debug('Forcing a timeout')
            self.timeout_counter = self.timeout * 2

        def _set_status(x, status):
            log.info("Download lbry://%s status changed to %s" %
                     (self.resolved_name, status))
            self.code = next(s for s in STREAM_STAGES if s[0] == status)
            return x

        def get_downloader_factory(metadata):
            for factory in metadata.factories:
                if isinstance(factory, ManagedLBRYFileDownloaderFactory):
                    return factory, metadata
            raise Exception('No suitable factory was found in {}'.format(
                metadata.factories))

        def make_downloader(args):
            factory, metadata = args
            return factory.make_downloader(
                metadata, [self.data_rate, True],
                self.payment_rate_manager,
                download_directory=self.download_directory,
                file_name=self.file_name)

        self.resolved_name = name
        self.stream_info = deepcopy(stream_info)
        self.description = self.stream_info['description']
        self.stream_hash = self.stream_info['sources']['lbry_sd_hash']

        if 'fee' in self.stream_info:
            self.fee = LBRYFeeValidator(self.stream_info['fee'])
            max_key_fee = self._convert_max_fee()
            if self.exchange_rate_manager.to_lbc(
                    self.fee).amount > max_key_fee:
                log.info(
                    "Key fee %f above limit of %f didn't download lbry://%s" %
                    (self.fee.amount, self.max_key_fee, self.resolved_name))
                return defer.fail(KeyFeeAboveMaxAllowed())
            log.info("Key fee %s below limit of %f, downloading lbry://%s" %
                     (json.dumps(self.fee), max_key_fee, self.resolved_name))

        self.checker.start(1)

        self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
        self.d.addCallback(lambda _: download_sd_blob(
            self.session, self.stream_hash, self.payment_rate_manager))
        self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
        self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
        self.d.addCallback(get_downloader_factory)
        self.d.addCallback(make_downloader)
        self.d.addCallbacks(self._start_download, _cause_timeout)
        self.d.callback(None)

        return self.finished
Пример #12
0
    def start(self, stream_info):
        self.stream_info = stream_info
        if 'stream_hash' in self.stream_info.keys():
            self.description = self.stream_info['description']
            if 'key_fee' in self.stream_info.keys():
                self.key_fee = float(self.stream_info['key_fee'])
                if 'key_fee_address' in self.stream_info.keys():
                    self.key_fee_address = self.stream_info['key_fee_address']
                else:
                    self.key_fee_address = None
            else:
                self.key_fee = None
                self.key_fee_address = None
            self.stream_hash = self.stream_info['stream_hash']
        elif 'stream_hash' in json.loads(self.stream_info['value']):
            self.resolved_name = self.stream_info.get('name', None)
            self.description = json.loads(self.stream_info['value']).get('description', None)

            try:
                if 'key_fee' in json.loads(self.stream_info['value']):
                    self.key_fee = float(json.loads(self.stream_info['value'])['key_fee'])
            except ValueError:
                self.key_fee = None
            self.key_fee_address = json.loads(self.stream_info['value']).get('key_fee_address', None)
            self.stream_hash = json.loads(self.stream_info['value'])['stream_hash']
        else:
            print 'InvalidStreamInfoError'
            raise InvalidStreamInfoError(self.stream_info)

        if self.key_fee > self.max_key_fee:
            if self.pay_key:
                print "Key fee (" + str(self.key_fee) + ") above limit of " + str(
                    self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)
                return self.finished_deferred.callback(None)
        else:
            pass

        def _get_downloader_for_return():
            return defer.succeed(self.downloader)

        self.loading_metadata_deferred = defer.Deferred(None)
        self.loading_metadata_deferred.addCallback(
            lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
        self.loading_metadata_deferred.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
        self.loading_metadata_deferred.addCallback(self._handle_metadata)
        self.loading_metadata_deferred.addErrback(self._handle_load_canceled)
        self.loading_metadata_deferred.addErrback(self._handle_load_failed)
        if self.pay_key:
            self.loading_metadata_deferred.addCallback(lambda _: self._pay_key_fee())
        self.loading_metadata_deferred.addCallback(lambda _: self._make_downloader())
        self.loading_metadata_deferred.addCallback(lambda _: self.downloader.start())
        self.loading_metadata_deferred.addErrback(self._handle_download_error)
        self.loading_metadata_deferred.addCallback(lambda _: _get_downloader_for_return())
        self.loading_metadata_deferred.callback(None)

        return defer.succeed(None)
Пример #13
0
    def start(self, stream_info, name):
        self.resolved_name = name
        self.stream_info = stream_info
        if 'stream_hash' in self.stream_info.keys():
            self.stream_hash = self.stream_info['stream_hash']
        elif 'sources' in self.stream_info.keys():
            self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
        else:
            raise InvalidStreamInfoError(self.stream_info)
        if 'description' in self.stream_info.keys():
            self.description = self.stream_info['description']
        if 'key_fee' in self.stream_info.keys():
            self.key_fee = float(self.stream_info['key_fee'])
            if 'key_fee_address' in self.stream_info.keys():
                self.key_fee_address = self.stream_info['key_fee_address']
            else:
                self.key_fee_address = None
        else:
            self.key_fee = None
            self.key_fee_address = None
        if self.key_fee > self.max_key_fee:
            if self.pay_key:
                log.info("Key fee (" + str(self.key_fee) + ") above limit of " + str(
                    self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name))
                return defer.fail(None)
        else:
            pass

        def _cause_timeout():
            self.timeout_counter = self.timeout * 2

        def _set_status(x, status):
            self.code = next(s for s in STREAM_STAGES if s[0] == status)
            return x

        self.checker.start(1)

        self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
        self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
        self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
        self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
        self.d.addCallback(lambda metadata: (
        next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)),
        metadata))
        self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata,
                                                                               [self.data_rate, True],
                                                                               self.payment_rate_manager,
                                                                               download_directory=self.download_directory,
                                                                               file_name=self.file_name))
        self.d.addCallbacks(self._start_download, lambda _: _cause_timeout())
        self.d.callback(None)

        return self.finished
Пример #14
0
    def download(self, name, key_fee):
        # download sd blob, and start downloader
        self.set_status(DOWNLOAD_METADATA_CODE, name)
        sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager)
        self.downloader = yield self._create_downloader(sd_blob)

        self.set_status(DOWNLOAD_RUNNING_CODE, name)
        if key_fee:
            yield self.pay_key_fee(key_fee, name)

        log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
        self.finished_deferred = self.downloader.start()
        self.finished_deferred.addCallback(self.finish, name)
Пример #15
0
 def test_lbry_transfer(self):
     sd_blob = yield download_sd_blob(self.sd_hash, self.blob_manager,
                                      self.peer_finder, self.rate_limiter,
                                      self.prm, self.wallet)
     metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
     downloader = yield metadata.factories[0].make_downloader(
         metadata,
         self.prm.min_blob_data_payment_rate,
         self.prm,
         self.db_dir,
         download_mirrors=None)
     yield downloader.start()
     with open(os.path.join(self.db_dir, 'test_file'), 'rb') as f:
         hashsum = md5()
         hashsum.update(f.read())
     self.assertEqual(hashsum.hexdigest(),
                      "4ca2aafb4101c1e42235aad24fbb83be")
Пример #16
0
    def download(self, stream_info, name):
        self.set_status(INITIALIZING_CODE, name)
        self.sd_hash = stream_info['sources']['lbry_sd_hash']
        if 'fee' in stream_info:
            fee = self.check_fee(stream_info['fee'])
        else:
            fee = None

        self.set_status(DOWNLOAD_METADATA_CODE, name)
        sd_blob = yield download_sd_blob(self.session, self.sd_hash, self.payment_rate_manager)
        stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
        factory = self.get_downloader_factory(stream_metadata.factories)
        self.downloader = yield self.get_downloader(factory, stream_metadata)

        self.set_status(DOWNLOAD_RUNNING_CODE, name)
        if fee:
            yield self.pay_key_fee(fee, name)
        log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
        self.finished_deferred = self.downloader.start()
        self.finished_deferred.addCallback(self.finish, name)
Пример #17
0
    def start(self, stream_info):
        self.stream_info = stream_info
        if 'stream_hash' in self.stream_info.keys():
            self.description = self.stream_info['description']
            if 'key_fee' in self.stream_info.keys():
                self.key_fee = float(self.stream_info['key_fee'])
                if 'key_fee_address' in self.stream_info.keys():
                    self.key_fee_address = self.stream_info['key_fee_address']
                else:
                    self.key_fee_address = None
            else:
                self.key_fee = None
                self.key_fee_address = None

            self.stream_hash = self.stream_info['stream_hash']

        else:
            print 'InvalidStreamInfoError'
            raise InvalidStreamInfoError(self.stream_info)

        if self.key_fee > self.max_key_fee:
            if self.pay_key:
                print "Key fee (" + str(self.key_fee) + ") above limit of " + str(
                    self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name)
                return defer.fail(None)
        else:
            pass

        d = defer.Deferred(None)
        d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
        d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
        d.addCallback(lambda metadata:
                      metadata.factories[1].make_downloader(metadata, [self.data_rate, True], self.payment_rate_manager))
        d.addErrback(lambda err: err.trap(defer.CancelledError))
        d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
        d.addCallback(self._start_download)
        d.callback(None)

        return d
Пример #18
0
 def _download_sd_blob(self):
     sd_blob = yield download_sd_blob(self.session, self.sd_hash,
                                      self.payment_rate_manager,
                                      self.timeout)
     defer.returnValue(sd_blob)
Пример #19
0
    def start(self, stream_info, name):
        def _cancel(err):
            # this callback sequence gets cancelled in check_status if
            # it takes too long when that happens, we want the logic
            # to live in check_status
            if err.check(defer.CancelledError):
                return
            if self.checker:
                self.checker.stop()
            self.finished.errback(err)

        def _set_status(x, status):
            log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status))
            self.code = next(s for s in STREAM_STAGES if s[0] == status)
            return x

        def get_downloader_factory(metadata):
            for factory in metadata.factories:
                if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
                    return factory, metadata
            raise Exception('No suitable factory was found in {}'.format(metadata.factories))

        def make_downloader(args):
            factory, metadata = args
            return factory.make_downloader(metadata,
                                           [self.data_rate, True],
                                           self.payment_rate_manager,
                                           download_directory=self.download_directory,
                                           file_name=self.file_name)

        self.resolved_name = name
        self.stream_info = deepcopy(stream_info)
        self.description = self.stream_info['description']
        self.sd_hash = self.stream_info['sources']['lbry_sd_hash']

        if 'fee' in self.stream_info:
            self.fee = FeeValidator(self.stream_info['fee'])
            max_key_fee = self._convert_max_fee()
            converted_fee = self.exchange_rate_manager.to_lbc(self.fee).amount
            if converted_fee > self.wallet.get_balance():
                msg = "Insufficient funds to download lbry://{}. Need {:0.2f}, have {:0.2f}".format(
                    self.resolved_name, converted_fee, self.wallet.get_balance())
                raise InsufficientFundsError(msg)
            if converted_fee > max_key_fee:
                msg = "Key fee {:0.2f} above limit of {:0.2f} didn't download lbry://{}".format(
                    converted_fee, max_key_fee, self.resolved_name)
                raise KeyFeeAboveMaxAllowed(msg)
            log.info(
                "Key fee %f below limit of %f, downloading lbry://%s",
                converted_fee, max_key_fee, self.resolved_name)

        self.checker.start(1)

        self._d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
        self._d.addCallback(lambda _: download_sd_blob(
            self.session, self.sd_hash, self.payment_rate_manager))
        self._d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
        self._d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
        self._d.addCallback(get_downloader_factory)
        self._d.addCallback(make_downloader)
        self._d.addCallbacks(self._start_download, _cancel)
        self._d.callback(None)

        return self.finished
Пример #20
0
 def _download_sd_blob(self):
     sd_blob = yield download_sd_blob(self.session, self.sd_hash,
                                      self.payment_rate_manager, self.timeout)
     defer.returnValue(sd_blob)