def test_fee_converts_to_lbc(self):
fee = Claim().stream.fee
fee.usd = Decimal(10.0)
fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
manager = get_fake_exchange_rate_manager()
result = manager.convert_currency(fee.currency, "LBC", fee.amount)
self.assertEqual(60.0, result)
def test_missing_feed(self):
fee = Claim().stream.fee
fee.usd = Decimal(1.0)
fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
manager = FakeExchangeRateManager([LBRYFeed()], {'BTCLBC': 1.0})
with self.assertRaises(CurrencyConversionError):
manager.convert_currency(fee.currency, "LBC", fee.amount)
async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
claim = Claim()
if fee:
if fee['currency'] == 'LBC':
claim.stream.fee.lbc = Decimal(fee['amount'])
elif fee['currency'] == 'USD':
claim.stream.fee.usd = Decimal(fee['amount'])
claim.stream.title = "33rpm"
claim.stream.languages.append("en")
claim.stream.source.sd_hash = sd_hash
claim.stream.source.media_type = "image/png"
tx = get_claim_transaction("33rpm", claim.to_bytes())
tx.height = 514081
txo = tx.outputs[0]
txo.meta.update({
"permanent_url": "33rpm#c49566d631226492317d06ad7fdbe1ed32925124",
})
class FakeHeaders:
def estimated_timestamp(self, height):
return 1984
def __init__(self, height):
self.height = height
def __getitem__(self, item):
return {'timestamp': 1984}
wallet = Wallet()
ledger = Ledger({
'db': Database(':memory:'),
'headers': FakeHeaders(514082)
})
await ledger.db.open()
wallet.generate_account(ledger)
manager = WalletManager()
manager.config = Config()
manager.wallets.append(wallet)
manager.ledgers[Ledger] = ledger
manager.ledger.network.client = ClientSession(
network=manager.ledger.network, server=('fakespv.lbry.com', 50001)
)
async def mock_resolve(*args, **kwargs):
result = {txo.meta['permanent_url']: txo}
claims = [
StreamManager._convert_to_old_resolve_output(manager, result)[txo.meta['permanent_url']]
]
await storage.save_claims(claims)
return result
manager.ledger.resolve = mock_resolve
async def get_balance(*_):
return balance
manager.get_balance = get_balance
return manager, txo.meta['permanent_url']
async def test_session_bloat_from_socket_timeout(self):
await self.account.ensure_address_gap()
address1, address2 = await self.account.receiving.get_addresses(
limit=2, only_usable=True)
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
await self.blockchain.generate(1)
await asyncio.wait([
self.on_transaction_id(sendtxid1),
self.on_transaction_id(sendtxid2)
])
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
channel = Claim()
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel,
self.account.ledger.address_to_hash160(address1))
channel_txo.generate_channel_private_key()
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo],
[self.account], self.account)
stream = Claim()
stream.stream.description = "0" * 8000
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream,
self.account.ledger.address_to_hash160(address1))
stream_tx = await Transaction.create([], [stream_txo], [self.account],
self.account)
stream_txo.sign(channel_txo)
await stream_tx.sign([self.account])
self.paused_session.clear()
self.resumed_session.clear()
await self.broadcast(channel_tx)
await self.broadcast(stream_tx)
await asyncio.wait_for(self.paused_session.wait(), 2)
self.assertEqual(1, len(self.session_manager.sessions))
real_sock = self.client_session.transport._extra.pop('socket')
mock_sock = Mock(spec=socket.socket)
for attr in dir(real_sock):
if not attr.startswith('__'):
setattr(mock_sock, attr, getattr(real_sock, attr))
def recv(*a, **kw):
raise TimeoutError("[Errno 110] Connection timed out")
mock_sock.recv = recv
self.client_session.transport._sock = mock_sock
self.client_session.transport._extra['socket'] = mock_sock
self.assertFalse(self.resumed_session.is_set())
self.assertFalse(self.session_manager.session_event.is_set())
await self.session_manager.session_event.wait()
self.assertEqual(0, len(self.session_manager.sessions))
def _checksig(self, value, address):
try:
claim_dict = Claim.from_bytes(value)
cert_id = claim_dict.signing_channel_hash
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = Claim.from_bytes(cert_claim.value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception:
pass
def __init__(self,
stream_hash: str,
outpoint: opt_str = None,
claim_id: opt_str = None,
name: opt_str = None,
amount: opt_int = None,
height: opt_int = None,
serialized: opt_str = None,
channel_claim_id: opt_str = None,
address: opt_str = None,
claim_sequence: opt_int = None,
channel_name: opt_str = None):
self.stream_hash = stream_hash
self.claim_id = claim_id
self.outpoint = outpoint
self.claim_name = name
self.amount = amount
self.height = height
self.claim: typing.Optional[
Claim] = None if not serialized else Claim.from_bytes(
binascii.unhexlify(serialized))
self.claim_address = address
self.claim_sequence = claim_sequence
self.channel_claim_id = channel_claim_id
self.channel_name = channel_name
def test_missing_feed(self):
# test when a feed is missing for conversion
fee = Claim().stream.fee
fee.usd = Decimal(1.0)
fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
rates = {
'BTCLBC': {
'spot': 1.0,
'ts': test_utils.DEFAULT_ISO_TIME + 1
},
}
market_feeds = [BTCLBCFeed()]
manager = DummyExchangeRateManager(market_feeds, rates)
with self.assertRaises(Exception):
manager.convert_currency(fee.currency, "LBC", fee.amount)
def get_channel(self, title, amount, name='@foo', key=b'a'):
claim = Claim()
claim.channel.title = title
channel = Output.pay_claim_name_pubkey_hash(amount, name, claim,
b'abc')
self._set_channel_key(channel, key)
return self._make_tx(channel)
def get_repost(self, claim_id, amount, channel):
claim = Claim()
claim.repost.reference.claim_id = claim_id
result = self._make_tx(
Output.pay_claim_name_pubkey_hash(amount, 'repost', claim, b'abc'))
result[0].outputs[0].sign(channel)
result[0]._reset()
return result
def claim(self) -> Claim:
if self.is_claim:
if not isinstance(self.script.values['claim'], Claim):
self.script.values['claim'] = Claim.from_bytes(
self.script.values['claim'])
return self.script.values['claim']
raise ValueError(
'Only claim name and claim update have the claim payload.')
def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
claim = {
"address": "bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF",
"amount": "0.1",
"claim_id": "c49566d631226492317d06ad7fdbe1ed32925124",
"claim_sequence": 1,
"decoded_claim": True,
"confirmations": 1057,
"effective_amount": "0.1",
"has_signature": False,
"height": 514081,
"hex": "",
"name": "33rpm",
"nout": 0,
"permanent_url": "33rpm#c49566d631226492317d06ad7fdbe1ed32925124",
"supports": [],
"txid": "81ac52662af926fdf639d56920069e0f63449d4cde074c61717cb99ddde40e3c",
}
claim_obj = Claim()
if fee:
if fee['currency'] == 'LBC':
claim_obj.stream.fee.lbc = Decimal(fee['amount'])
elif fee['currency'] == 'USD':
claim_obj.stream.fee.usd = Decimal(fee['amount'])
claim_obj.stream.title = "33rpm"
claim_obj.stream.languages.append("en")
claim_obj.stream.source.sd_hash = sd_hash
claim_obj.stream.source.media_type = "image/png"
claim['value'] = claim_obj
claim['protobuf'] = binascii.hexlify(claim_obj.to_bytes()).decode()
async def mock_resolve(*args):
await storage.save_claims([claim])
return {
claim['permanent_url']: claim
}
mock_wallet = mock.Mock(spec=LbryWalletManager)
mock_wallet.ledger.resolve = mock_resolve
mock_wallet.ledger.network.client.server = ('fakespv.lbry.com', 50001)
async def get_balance(*_):
return balance
mock_wallet.default_account.get_balance = get_balance
return mock_wallet, claim['permanent_url']
def get_stream(self, title, amount, name='foo', channel=None):
claim = Claim()
claim.stream.title = title
result = self._make_tx(Output.pay_claim_name_pubkey_hash(amount, name, claim, b'abc'))
if channel:
result[0].outputs[0].sign(channel)
result[0]._reset()
return result
def test_stream_claim(self):
stream = Stream()
claim = stream.claim
self.assertEqual(claim.claim_type, Claim.STREAM)
claim = Claim.from_bytes(claim.to_bytes())
self.assertEqual(claim.claim_type, Claim.STREAM)
self.assertIsNotNone(claim.stream)
with self.assertRaisesRegex(ValueError, 'Claim is not a channel.'):
print(claim.channel)
async def get_channel(claim_name='@foo'):
seed = Mnemonic.mnemonic_to_seed(Mnemonic().make_seed(), '')
key = PrivateKey.from_seed(Ledger, seed)
channel_key = key.child(KeyPath.CHANNEL).child(0)
channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(),
b'abc')
channel_txo.set_channel_private_key(channel_key)
get_tx().add_outputs([channel_txo])
return channel_txo
def test_fee_converts_to_lbc(self):
fee = Claim().stream.fee
fee.usd = Decimal(10.0)
fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
rates = {
'BTCLBC': {
'spot': 3.0,
'ts': test_utils.DEFAULT_ISO_TIME + 1
},
'USDBTC': {
'spot': 2.0,
'ts': test_utils.DEFAULT_ISO_TIME + 2
}
}
market_feeds = [BTCLBCFeed(), USDBTCFeed()]
manager = DummyExchangeRateManager(market_feeds, rates)
result = manager.convert_currency(fee.currency, "LBC", fee.amount)
self.assertEqual(60.0, result)
async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False):
account = (daemon or self.daemon).wallet_manager.default_account
claim_address = await account.receiving.get_or_create_usable_address()
claim = Claim()
claim.channel.public_key_bytes = pubkey_bytes
tx = await Transaction.claim_create(
name, claim, lbc_to_dewies(price),
claim_address, [self.account], self.account
)
await tx.sign([self.account])
await (daemon or self.daemon).broadcast_or_release(tx, blocking)
return self.sout(tx)
def test_normalize_tags(self):
claim = Claim()
claim.channel.update(tags=['Anime', 'anime', ' aNiMe', 'maNGA '])
self.assertCountEqual(claim.channel.tags, ['anime', 'manga'])
claim.channel.update(tags=['Juri', 'juRi'])
self.assertCountEqual(claim.channel.tags, ['anime', 'manga', 'juri'])
claim.channel.update(tags='Anime')
self.assertCountEqual(claim.channel.tags, ['anime', 'manga', 'juri'])
claim.channel.update(clear_tags=True)
self.assertEqual(len(claim.channel.tags), 0)
claim.channel.update(tags='Anime')
self.assertEqual(claim.channel.tags, ['anime'])
def _save_content_claim(transaction,
claim_outpoint,
stream_hash=None,
bt_infohash=None):
assert stream_hash or bt_infohash
# get the claim id and serialized metadata
claim_info = transaction.execute(
"select claim_id, serialized_metadata from claim where claim_outpoint=?",
(claim_outpoint, )).fetchone()
if not claim_info:
raise Exception("claim not found")
new_claim_id, claim = claim_info[0], Claim.from_bytes(
binascii.unhexlify(claim_info[1]))
# certificate claims should not be in the content_claim table
if not claim.is_stream:
raise Exception("claim does not contain a stream")
# get the known sd hash for this stream
known_sd_hash = transaction.execute(
"select sd_hash from stream where stream_hash=?",
(stream_hash, )).fetchone()
if not known_sd_hash:
raise Exception("stream not found")
# check the claim contains the same sd hash
if known_sd_hash[0] != claim.stream.source.sd_hash:
raise Exception("stream mismatch")
# if there is a current claim associated to the file, check that the new claim is an update to it
current_associated_content = transaction.execute(
"select claim_outpoint from content_claim where stream_hash=?",
(stream_hash, )).fetchone()
if current_associated_content:
current_associated_claim_id = transaction.execute(
"select claim_id from claim where claim_outpoint=?",
current_associated_content).fetchone()[0]
if current_associated_claim_id != new_claim_id:
raise Exception(
f"mismatching claim ids when updating stream {current_associated_claim_id} vs {new_claim_id}"
)
# update the claim associated to the file
transaction.execute("delete from content_claim where stream_hash=?",
(stream_hash, )).fetchall()
transaction.execute("insert into content_claim values (?, NULL, ?)",
(stream_hash, claim_outpoint)).fetchall()
def _decode_claim_result(claim):
if 'decoded_claim' in claim:
return claim
if 'value' not in claim:
log.warning('Got an invalid claim while parsing, please report: %s',
claim)
claim['protobuf'] = None
claim['value'] = None
backend_message = ' SDK message: ' + claim.get('error', '')
claim['error'] = "Failed to parse: missing value." + backend_message
return claim
try:
if not isinstance(claim['value'], Claim):
claim['value'] = Claim.from_bytes(unhexlify(claim['value']))
claim['protobuf'] = hexlify(claim['value'].to_bytes())
claim['decoded_claim'] = True
except DecodeError:
claim['decoded_claim'] = False
claim['protobuf'] = claim['value']
claim['value'] = None
return claim
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: Optional[float] = None,
file_name: Optional[str] = None,
download_directory: Optional[str] = None,
save_file: Optional[bool] = None,
resolve_timeout: float = 3.0,
wallet: Optional['Wallet'] = None) -> ManagedStream:
manager = self.wallet_manager
wallet = wallet or manager.default_wallet
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
txo: Optional[Output] = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
payment = None
try:
# resolve the claim
if not URL.parse(uri).has_stream:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
response = await asyncio.wait_for(
manager.ledger.resolve(wallet.accounts, [uri]),
resolve_timeout
)
resolved_result = self._convert_to_old_resolve_output(manager, response)
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values() if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
txo = response[uri]
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed: download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory, node=self.node
)
return updated_stream
if not to_replace and txo.has_price and not txo.purchase_receipt:
payment = await manager.create_purchase_transaction(
wallet.accounts, txo, exchange_rate_manager
)
stream = ManagedStream(
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(self.node, timeout)
stream.set_claim(resolved, claim)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
if payment is not None:
await manager.broadcast_or_release(payment)
payment = None # to avoid releasing in `finally` later
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
if save_file:
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError)
if isinstance(err, expected):
log.warning("Failed to download %s: %s", uri, str(err))
elif isinstance(err, asyncio.CancelledError):
pass
else:
log.exception("Unexpected error downloading stream:")
error = err
raise
finally:
if payment is not None:
# payment is set to None after broadcasting, if we're here an exception probably happened
await manager.ledger.release_tx(payment)
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
server = self.wallet_manager.ledger.network.client.server
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__,
None if not error else str(error),
None if not server else f"{server[0]}:{server[1]}"
)
)
async def test_creating_updating_and_abandoning_claim_with_channel(self):
await self.account.ensure_address_gap()
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
notifications = asyncio.create_task(asyncio.wait(
[asyncio.ensure_future(self.on_address_update(address1)),
asyncio.ensure_future(self.on_address_update(address2))]
))
await self.send_to_address_and_wait(address1, 5)
await self.send_to_address_and_wait(address2, 5, 1)
await notifications
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
channel = Claim()
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
)
channel_txo.set_channel_private_key(
await self.account.generate_channel_private_key()
)
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
stream = Claim()
stream.stream.source.media_type = "video/mp4"
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
)
stream_tx = await Transaction.create([], [stream_txo], [self.account], self.account)
stream_txo.sign(channel_txo)
await stream_tx.sign([self.account])
notifications = asyncio.create_task(asyncio.wait(
[asyncio.ensure_future(self.ledger.wait(channel_tx)), asyncio.ensure_future(self.ledger.wait(stream_tx))]
))
await self.broadcast(channel_tx)
await self.broadcast(stream_tx)
await notifications
notifications = asyncio.create_task(asyncio.wait(
[asyncio.ensure_future(self.ledger.wait(channel_tx)), asyncio.ensure_future(self.ledger.wait(stream_tx))]
))
await self.generate(1)
await notifications
self.assertEqual(d2l(await self.account.get_balance()), '7.985786')
self.assertEqual(d2l(await self.account.get_balance(include_claims=True)), '9.985786')
response = await self.ledger.resolve([], ['lbry://@bar/foo'])
self.assertEqual(response['lbry://@bar/foo'].claim.claim_type, 'stream')
abandon_tx = await Transaction.create([Input.spend(stream_tx.outputs[0])], [], [self.account], self.account)
notify = asyncio.create_task(self.ledger.wait(abandon_tx))
await self.broadcast(abandon_tx)
await notify
notify = asyncio.create_task(self.ledger.wait(abandon_tx))
await self.generate(1)
await notify
response = await self.ledger.resolve([], ['lbry://@bar/foo'])
self.assertIn('error', response['lbry://@bar/foo'])
# checks for expected format in inexistent URIs
response = await self.ledger.resolve([], ['lbry://404', 'lbry://@404', 'lbry://@404/404'])
self.assertEqual('Could not find claim at "lbry://404".', response['lbry://404']['error']['text'])
self.assertEqual('Could not find channel in "lbry://@404".', response['lbry://@404']['error']['text'])
self.assertEqual('Could not find channel in "lbry://@404/404".', response['lbry://@404/404']['error']['text'])
def get_stream(claim_name='foo'):
stream_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(),
b'abc')
get_tx().add_outputs([stream_txo])
return stream_txo
def get_channel(claim_name='@foo'):
channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(),
b'abc')
channel_txo.generate_channel_private_key()
get_tx().add_outputs([channel_txo])
return channel_txo
async def test_creating_updating_and_abandoning_claim_with_channel(self):
await self.account.ensure_address_gap()
address1, address2 = await self.account.receiving.get_addresses(
limit=2, only_usable=True)
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
await self.blockchain.generate(1)
await asyncio.wait([
self.on_transaction_id(sendtxid1),
self.on_transaction_id(sendtxid2)
])
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
channel = Claim()
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel,
self.account.ledger.address_to_hash160(address1))
channel_txo.generate_channel_private_key()
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo],
[self.account], self.account)
stream = Claim()
stream.stream.source.media_type = "video/mp4"
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream,
self.account.ledger.address_to_hash160(address1))
stream_tx = await Transaction.create([], [stream_txo], [self.account],
self.account)
stream_txo.sign(channel_txo)
await stream_tx.sign([self.account])
await self.broadcast(channel_tx)
await self.broadcast(stream_tx)
await asyncio.wait([ # mempool
self.ledger.wait(channel_tx),
self.ledger.wait(stream_tx)
])
await self.blockchain.generate(1)
await asyncio.wait([ # confirmed
self.ledger.wait(channel_tx),
self.ledger.wait(stream_tx)
])
self.assertEqual(d2l(await self.account.get_balance()), '7.985786')
self.assertEqual(
d2l(await self.account.get_balance(include_claims=True)),
'9.985786')
response = await self.ledger.resolve([], ['lbry://@bar/foo'])
self.assertEqual(response['lbry://@bar/foo'].claim.claim_type,
'stream')
abandon_tx = await Transaction.create(
[Input.spend(stream_tx.outputs[0])], [], [self.account],
self.account)
await self.broadcast(abandon_tx)
await self.ledger.wait(abandon_tx)
await self.blockchain.generate(1)
await self.ledger.wait(abandon_tx)
response = await self.ledger.resolve([], ['lbry://@bar/foo'])
self.assertIn('error', response['lbry://@bar/foo'])
# checks for expected format in inexistent URIs
response = await self.ledger.resolve([], ['lbry://404', 'lbry://@404'])
self.assertEqual('lbry://404 did not resolve to a claim',
response['lbry://404']['error'])
self.assertEqual('lbry://@404 did not resolve to a claim',
response['lbry://@404']['error'])
def _make_db(new_db):
# create the new tables
new_db.executescript(CREATE_TABLES_QUERY)
# first migrate the blobs
blobs = blobs_db_cursor.execute("select * from blobs").fetchall()
_populate_blobs(blobs) # pylint: disable=no-value-for-parameter
log.info("migrated %i blobs",
new_db.execute("select count(*) from blob").fetchone()[0])
# used to store the query arguments if we need to try re-importing the lbry file later
file_args = {} # <sd_hash>: args tuple
file_outpoints = {} # <outpoint tuple>: sd_hash
# get the file and stream queries ready
for (rowid, sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate, status) in \
lbryfile_db.execute(
"select distinct lbry_files.rowid, d.sd_blob_hash, lbry_files.*, o.blob_data_rate, o.status "
"from lbry_files "
"inner join lbry_file_descriptors d on lbry_files.stream_hash=d.stream_hash "
"inner join lbry_file_options o on lbry_files.stream_hash=o.stream_hash"):
# this is try to link the file to a content claim after we've imported all the files
if rowid in old_rowid_to_outpoint:
file_outpoints[old_rowid_to_outpoint[rowid]] = sd_hash
elif sd_hash in old_sd_hash_to_outpoint:
file_outpoints[old_sd_hash_to_outpoint[sd_hash]] = sd_hash
sd_hash_to_stream_hash[sd_hash] = stream_hash
if stream_hash in stream_hash_to_stream_blobs:
file_args[sd_hash] = (
sd_hash, stream_hash, key, stream_name,
suggested_file_name, data_rate or 0.0, status,
stream_hash_to_stream_blobs.pop(stream_hash))
# used to store the query arguments if we need to try re-importing the claim
claim_queries = {} # <sd_hash>: claim query tuple
# get the claim queries ready, only keep those with associated files
for outpoint, sd_hash in file_outpoints.items():
if outpoint in claim_outpoint_queries:
claim_queries[sd_hash] = claim_outpoint_queries[outpoint]
# insert the claims
new_db.executemany(
"insert or ignore into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
[
("%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]),
claim_arg_tup[2], claim_arg_tup[3], claim_arg_tup[7],
claim_arg_tup[6], claim_arg_tup[8],
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id,
claim_arg_tup[5], claim_arg_tup[4])
for sd_hash, claim_arg_tup in claim_queries.items()
if claim_arg_tup
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
)
log.info("migrated %i claims",
new_db.execute("select count(*) from claim").fetchone()[0])
damaged_stream_sds = []
# import the files and get sd hashes of streams to attempt recovering
for sd_hash, file_query in file_args.items():
failed_sd = _import_file(*file_query)
if failed_sd:
damaged_stream_sds.append(failed_sd)
# recover damaged streams
if damaged_stream_sds:
blob_dir = os.path.join(conf.data_dir, "blobfiles")
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list(
{p
for p in os.listdir(blob_dir) if p in damaged_stream_sds})
for damaged_sd in damaged_sds_on_disk:
try:
decoded, sd_length = verify_sd_blob(damaged_sd, blob_dir)
blobs = decoded['blobs']
_add_recovered_blobs(blobs, damaged_sd, sd_length) # pylint: disable=no-value-for-parameter
_import_file(*file_args[damaged_sd])
damaged_stream_sds.remove(damaged_sd)
except (OSError, ValueError, TypeError, IOError,
AssertionError, sqlite3.IntegrityError):
continue
log.info("migrated %i files",
new_db.execute("select count(*) from file").fetchone()[0])
# associate the content claims to their respective files
for claim_arg_tup in claim_queries.values():
if claim_arg_tup and (claim_arg_tup[0], claim_arg_tup[1]) in file_outpoints \
and file_outpoints[(claim_arg_tup[0], claim_arg_tup[1])] in sd_hash_to_stream_hash:
try:
new_db.execute(
"insert or ignore into content_claim values (?, ?)",
(sd_hash_to_stream_hash.get(
file_outpoints.get(
(claim_arg_tup[0], claim_arg_tup[1]))),
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1])))
except sqlite3.IntegrityError:
continue
log.info(
"migrated %i content claims",
new_db.execute("select count(*) from content_claim").fetchone()[0])
async def download_stream_from_uri(
self,
uri,
exchange_rate_manager: 'ExchangeRateManager',
timeout: typing.Optional[float] = None,
file_name: typing.Optional[str] = None,
download_directory: typing.Optional[str] = None,
save_file: typing.Optional[bool] = None,
resolve_timeout: float = 3.0) -> ManagedStream:
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
try:
# resolve the claim
if not URL.parse(uri).has_stream:
raise ResolveError(
"cannot download a channel claim, specify a /path")
try:
resolved_result = self._convert_to_old_resolve_output(
await asyncio.wait_for(self.wallet.ledger.resolve([uri]),
resolve_timeout))
except asyncio.TimeoutError:
raise ResolveTimeout(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise
raise ResolveError(
f"Unexpected error resolving stream: {str(err)}")
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values()
if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get(
'claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(
f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(
outpoint, resolved['claim_id'], claim)
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
await updated_stream.start(node=self.node,
timeout=timeout,
save_now=save_file)
if not updated_stream.output_file_exists and (
save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name,
download_directory=download_directory,
node=self.node)
return updated_stream
content_fee = None
fee_amount, fee_address = None, None
# check that the fee is payable
if not to_replace and claim.stream.has_fee and claim.stream.fee.amount:
fee_amount = round(
exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC",
claim.stream.fee.amount), 5)
max_fee_amount = round(
exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC",
Decimal(self.config.max_key_fee['amount'])), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address or resolved['address']
stream = ManagedStream(self.loop,
self.config,
self.blob_manager,
claim.stream.source.sd_hash,
download_directory,
file_name,
ManagedStream.STATUS_RUNNING,
content_fee=content_fee,
analytics_manager=self.analytics_manager)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(self.node, timeout)
stream.set_claim(resolved, claim)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
elif fee_address:
stream.content_fee = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)),
fee_address.encode('latin1'))
log.info("paid fee of %s for %s", fee_amount, uri)
await self.storage.save_content_fee(stream.stream_hash,
stream.content_fee)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[
stream.stream_hash] = lambda: self._update_content_claim(stream
)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
if save_file:
await asyncio.wait_for(stream.save_file(node=self.node),
timeout -
(self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeout(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, dont delete stream
error = err
raise
finally:
if self.analytics_manager and (
error or (stream and
(stream.downloader.time_to_descriptor
or stream.downloader.time_to_first_bytes))):
server = self.wallet.ledger.network.client.server
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time,
self.loop.time() - start_time,
None if not stream else stream.download_id, uri,
outpoint, None if not stream else len(
stream.downloader.blob_downloader.
active_connections), None if not stream else len(
stream.downloader.blob_downloader.scores),
None if not stream else len(
stream.downloader.blob_downloader.
connection_failures), False
if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay
if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash, None if
not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].length, None if not stream
else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__,
None if not error else str(error),
None if not server else f"{server[0]}:{server[1]}"))
