def _checksig(self, name, value, address):
try:
parse_lbry_uri(name.decode()) # skip invalid names
cert_id = Claim.FromString(value).publisherSignature.certificateId[::-1] or None
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = smart_decode(cert_claim.value)
claim_dict = smart_decode(value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception as e:
pass
def _checksig(self, name, value, address):
try:
parse_lbry_uri(name.decode()) # skip invalid names
claim_dict = smart_decode(value)
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = smart_decode(cert_claim.value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception as e:
pass
def _save_claims(transaction):
content_claims_to_update = []
for claim_info in claim_infos:
outpoint = "%s:%i" % (claim_info['txid'], claim_info['nout'])
claim_id = claim_info['claim_id']
name = claim_info['name']
amount = lbc_to_dewies(claim_info['amount'])
height = claim_info['height']
address = claim_info['address']
sequence = claim_info['claim_sequence']
try:
certificate_id = claim_info['value'].get(
'publisherSignature', {}).get('certificateId')
except AttributeError:
certificate_id = None
try:
if claim_info['value'].get('stream', {}).get(
'source', {}).get('sourceType') == "lbry_sd_hash":
source_hash = claim_info['value'].get(
'stream', {}).get('source', {}).get('source')
else:
source_hash = None
except AttributeError:
source_hash = None
serialized = claim_info.get('hex') or binascii.hexlify(
smart_decode(claim_info['value']).serialized).decode()
transaction.execute(
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(outpoint, claim_id, name, amount, height, serialized,
certificate_id, address, sequence))
# if this response doesn't have support info don't overwrite the existing
# support info
if 'supports' in claim_info:
support_callbacks.append(
(claim_id, claim_info['supports']))
if not source_hash:
continue
stream_hash = transaction.execute(
"select file.stream_hash from stream "
"inner join file on file.stream_hash=stream.stream_hash where sd_hash=?",
(source_hash, )).fetchone()
if not stream_hash:
continue
stream_hash = stream_hash[0]
known_outpoint = transaction.execute(
"select claim_outpoint from content_claim where stream_hash=?",
(stream_hash, ))
known_claim_id = transaction.execute(
"select claim_id from claim "
"inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint "
"where c3.stream_hash=?", (stream_hash, ))
if not known_claim_id:
content_claims_to_update.append((stream_hash, outpoint))
elif known_outpoint != outpoint:
content_claims_to_update.append((stream_hash, outpoint))
for stream_hash, outpoint in content_claims_to_update:
self._save_content_claim(transaction, outpoint, stream_hash)
if stream_hash in self.content_claim_callbacks:
update_file_callbacks.append(
self.content_claim_callbacks[stream_hash]())
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
claim = smart_decode(claim_info['value'])
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
download_directory, file_name)
try:
downloader.download(node)
await downloader.got_descriptor.wait()
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
except (asyncio.TimeoutError, asyncio.CancelledError):
log.info("stream timeout")
downloader.stop()
log.info("stopped stream")
raise DownloadSDTimeout(downloader.sd_hash)
rowid = await self._store_stream(downloader)
await self.storage.save_content_claim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
)
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, download_directory,
file_name, downloader, ManagedStream.STATUS_RUNNING)
stream.set_claim(claim_info, claim)
self.streams.add(stream)
try:
await stream.downloader.wrote_bytes_event.wait()
self.wait_for_stream_finished(stream)
return stream
except asyncio.CancelledError:
downloader.stop()
log.debug("stopped stream")
await self.stop_stream(stream)
raise DownloadDataTimeout(downloader.sd_hash)
async def get_certificate_and_validate_result(self, claim_result):
if not claim_result or 'value' not in claim_result:
return claim_result
certificate = None
certificate_id = smart_decode(claim_result['value']).certificate_id
if certificate_id:
certificate = await self.network.get_claims_by_ids(certificate_id)
certificate = certificate.pop(certificate_id) if certificate else None
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
claim_address, name, decoded_certificate=None):
if not certificate_claim:
return False, None
if 'value' not in certificate_claim:
log.warning('Got an invalid claim while parsing certificates, please report: %s', certificate_claim)
return False, None
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
if not isinstance(certificate, ClaimDict):
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
if _validate_signed_claim(claim, claim_address, name, certificate):
return True, certificate_claim['name']
return False, None
def test_smart_decode_raises(self):
with self.assertRaises(TypeError):
smart_decode(1)
with self.assertRaises(DecodeError):
smart_decode("aaab")
with self.assertRaises(DecodeError):
smart_decode("{'bogus_dict':1}")
def __init__(self, stream_hash: str, outpoint: opt_str = None, claim_id: opt_str = None, name: opt_str = None,
amount: opt_int = None, height: opt_int = None, serialized: opt_str = None,
channel_claim_id: opt_str = None, address: opt_str = None, claim_sequence: opt_int = None,
channel_name: opt_str = None):
self.stream_hash = stream_hash
self.claim_id = claim_id
self.outpoint = outpoint
self.claim_name = name
self.amount = amount
self.height = height
self.claim: typing.Optional[ClaimDict] = None if not serialized else smart_decode(serialized)
self.claim_address = address
self.claim_sequence = claim_sequence
self.channel_claim_id = channel_claim_id
self.channel_name = channel_name
def get_test_daemon(data_rate=None, generous=True, with_fee=False):
if data_rate is None:
data_rate = conf.ADJUSTABLE_SETTINGS['data_rate'][1]
rates = {
'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1},
'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2}
}
component_manager = ComponentManager(
skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT,
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
HEADERS_COMPONENT, RATE_LIMITER_COMPONENT],
file_manager=FakeFileManager
)
daemon = LBRYDaemon(component_manager=component_manager)
daemon.payment_rate_manager = OnlyFreePaymentsManager()
daemon.wallet_manager = mock.Mock(spec=LbryWalletManager)
daemon.wallet_manager.wallet = mock.Mock(spec=Wallet)
daemon.wallet_manager.wallet.use_encryption = False
daemon.wallet_manager.network = FakeNetwork()
daemon.storage = mock.Mock(spec=SQLiteStorage)
market_feeds = [BTCLBCFeed(), USDBTCFeed()]
daemon.exchange_rate_manager = DummyExchangeRateManager(market_feeds, rates)
daemon.file_manager = component_manager.get_component(FILE_MANAGER_COMPONENT)
metadata = {
"author": "fake author",
"language": "en",
"content_type": "fake/format",
"description": "fake description",
"license": "fake license",
"license_url": "fake license url",
"nsfw": False,
"sources": {
"lbry_sd_hash": 'd2b8b6e907dde95245fe6d144d16c2fdd60c4e0c6463ec98'
'b85642d06d8e9414e8fcfdcb7cb13532ec5454fb8fe7f280'
},
"thumbnail": "fake thumbnail",
"title": "fake title",
"ver": "0.0.3"
}
if with_fee:
metadata.update(
{"fee": {"USD": {"address": "bQ6BGboPV2SpTMEP7wLNiAcnsZiH8ye6eA", "amount": 0.75}}})
migrated = smart_decode(json.dumps(metadata))
daemon._resolve = daemon.wallet_manager.resolve = lambda *_: defer.succeed(
{"test": {'claim': {'value': migrated.claim_dict}}})
return daemon
async def parse_and_validate_claim_result(self,
claim_result,
certificate=None,
raw=False):
if not claim_result or 'value' not in claim_result:
return claim_result
claim_result['decoded_claim'] = False
decoded = None
if not raw:
claim_value = claim_result['value']
try:
decoded = smart_decode(claim_value)
claim_result['value'] = decoded.claim_dict
claim_result['decoded_claim'] = True
except DecodeError:
pass
if decoded:
claim_result['has_signature'] = False
if decoded.has_signature:
if certificate is None:
log.info("fetching certificate to check claim signature")
certificate = await self.network.get_claims_by_ids(
decoded.certificate_id)
if not certificate:
log.warning('Certificate %s not found',
decoded.certificate_id)
claim_result['has_signature'] = True
claim_result['signature_is_valid'] = False
validated, channel_name = validate_claim_signature_and_get_channel_name(
decoded, certificate, claim_result['address'],
claim_result['name'])
claim_result['channel_name'] = channel_name
if validated:
claim_result['signature_is_valid'] = True
if 'height' in claim_result and claim_result['height'] is None:
claim_result['height'] = -1
if 'amount' in claim_result:
claim_result = format_amount_value(claim_result)
claim_result['permanent_url'] = _get_permanent_url(
claim_result, decoded.certificate_id if decoded else None)
return claim_result
def _decode_claim_result(claim):
if 'has_signature' in claim and claim['has_signature']:
if not claim['signature_is_valid']:
log.warning("lbry://%s#%s has an invalid signature",
claim['name'], claim['claim_id'])
if 'value' not in claim:
log.warning('Got an invalid claim while parsing, please report: %s', claim)
claim['hex'] = None
claim['value'] = None
claim['error'] = "Failed to parse: missing value"
return claim
try:
decoded = smart_decode(claim['value'])
claim_dict = decoded.claim_dict
claim['value'] = claim_dict
claim['hex'] = hexlify(decoded.serialized)
except DecodeError:
claim['hex'] = claim['value']
claim['value'] = None
claim['error'] = "Failed to decode value"
return claim
def _make_db(new_db):
# create the new tables
new_db.executescript(CREATE_TABLES_QUERY)
# first migrate the blobs
blobs = blobs_db_cursor.execute("select * from blobs").fetchall()
_populate_blobs(blobs) # pylint: disable=no-value-for-parameter
log.info("migrated %i blobs", new_db.execute("select count(*) from blob").fetchone()[0])
# used to store the query arguments if we need to try re-importing the lbry file later
file_args = {} # <sd_hash>: args tuple
file_outpoints = {} # <outpoint tuple>: sd_hash
# get the file and stream queries ready
for (rowid, sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate, status) in \
lbryfile_db.execute(
"select distinct lbry_files.rowid, d.sd_blob_hash, lbry_files.*, o.blob_data_rate, o.status "
"from lbry_files "
"inner join lbry_file_descriptors d on lbry_files.stream_hash=d.stream_hash "
"inner join lbry_file_options o on lbry_files.stream_hash=o.stream_hash"):
# this is try to link the file to a content claim after we've imported all the files
if rowid in old_rowid_to_outpoint:
file_outpoints[old_rowid_to_outpoint[rowid]] = sd_hash
elif sd_hash in old_sd_hash_to_outpoint:
file_outpoints[old_sd_hash_to_outpoint[sd_hash]] = sd_hash
sd_hash_to_stream_hash[sd_hash] = stream_hash
if stream_hash in stream_hash_to_stream_blobs:
file_args[sd_hash] = (
sd_hash, stream_hash, key, stream_name,
suggested_file_name, data_rate or 0.0,
status, stream_hash_to_stream_blobs.pop(stream_hash)
)
# used to store the query arguments if we need to try re-importing the claim
claim_queries = {} # <sd_hash>: claim query tuple
# get the claim queries ready, only keep those with associated files
for outpoint, sd_hash in file_outpoints.items():
if outpoint in claim_outpoint_queries:
claim_queries[sd_hash] = claim_outpoint_queries[outpoint]
# insert the claims
new_db.executemany(
"insert or ignore into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
[
(
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
)
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
)
log.info("migrated %i claims", new_db.execute("select count(*) from claim").fetchone()[0])
damaged_stream_sds = []
# import the files and get sd hashes of streams to attempt recovering
for sd_hash, file_query in file_args.items():
failed_sd = _import_file(*file_query)
if failed_sd:
damaged_stream_sds.append(failed_sd)
# recover damaged streams
if damaged_stream_sds:
blob_dir = os.path.join(db_dir, "blobfiles")
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
if p in damaged_stream_sds})
for damaged_sd in damaged_sds_on_disk:
try:
decoded, sd_length = verify_sd_blob(damaged_sd, blob_dir)
blobs = decoded['blobs']
_add_recovered_blobs(blobs, damaged_sd, sd_length) # pylint: disable=no-value-for-parameter
_import_file(*file_args[damaged_sd])
damaged_stream_sds.remove(damaged_sd)
except (OSError, ValueError, TypeError, IOError, AssertionError, sqlite3.IntegrityError):
continue
log.info("migrated %i files", new_db.execute("select count(*) from file").fetchone()[0])
# associate the content claims to their respective files
for claim_arg_tup in claim_queries.values():
if claim_arg_tup and (claim_arg_tup[0], claim_arg_tup[1]) in file_outpoints \
and file_outpoints[(claim_arg_tup[0], claim_arg_tup[1])] in sd_hash_to_stream_hash:
try:
new_db.execute(
"insert or ignore into content_claim values (?, ?)",
(
sd_hash_to_stream_hash.get(file_outpoints.get((claim_arg_tup[0], claim_arg_tup[1]))),
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1])
)
)
except sqlite3.IntegrityError:
continue
log.info("migrated %i content claims", new_db.execute("select count(*) from content_claim").fetchone()[0])
def claim(self) -> ClaimDict:
if self.script.is_claim_name or self.script.is_update_claim:
return smart_decode(self.script.values['claim'])
raise ValueError(
'Only claim name and claim update have the claim payload.')
def test_binary_decode(self):
self.assertEqual(expected_binary_claim_decoded, smart_decode(binary_claim).claim_dict)
def test_hex_decode(self):
self.assertEqual(decoded_hex_encoded_003, smart_decode(hex_encoded_003).claim_dict)
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
stream.set_claim(claim_info, smart_decode(claim_info['value']))
