async def resolve(self, page, page_size, *uris):
for uri in uris:
try:
parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.args[0]}
resolutions = await self.network.get_values_for_uris(
self.headers.hash().decode(), *uris)
return await self.resolver._handle_resolutions(resolutions, uris, page,
page_size)
def resolve(self, page, page_size, *uris):
for uri in uris:
try:
parse_lbry_uri(uri)
except URIParseError as err:
defer.returnValue({'error': err.message})
resolutions = yield self.network.get_values_for_uris(
self.headers.hash().decode(), *uris)
return (yield
self.resolver._handle_resolutions(resolutions, uris, page,
page_size))
def _checksig(self, name, value, address):
try:
parse_lbry_uri(name.decode()) # skip invalid names
cert_id = Claim.FromString(
value).publisherSignature.certificateId[::-1] or None
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.get_claim_info(cert_id)
if cert_claim:
certificate = smart_decode(cert_claim.value)
claim_dict = smart_decode(value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception as e:
pass
def update_db(app_db, names_db, height_db, expiring_height):
outpoint_db = plyvel.DB('db/claim_outpoint/')
def get_txid_for_claim_id(claim_id):
txid_nout = outpoint_db.get(claim_id)
txid = txid_nout[0:64]
return txid
expired_names, known_types, txids, expired_channels = {}, set(), {}, {}
[app_db.delete(x[0]) for x in app_db]
with app_db.write_batch() as writer:
for (claim_id, height) in height_db:
key = struct.pack('>I40s', int(height), claim_id)
try:
name = names_db.get(claim_id).decode('utf8')
parsed = parse_lbry_uri(name)
decoded = smart_decode(values_db.get(claim_id))
known_types.add(decoded.get('claimType', 'unknown'))
if decoded.get('claimType') == 'certificateType' or parsed.is_channel:
expired_channels[name] = (height, claim_id)
if int(height) < expiring_height:
expired_names[name] = (int(height), claim_id)
txids[name] = get_txid_for_claim_id(claim_id)
writer.put(key, name.encode('utf8'))
except (DecodeError, UnicodeDecodeError, URIParseError):
continue
return expired_names, known_types, txids, expired_channels
def resolve(self, page, page_size, *uris):
for uri in uris:
try:
parse_lbry_uri(uri)
except URIParseError as err:
defer.returnValue({'error': err.message})
resolutions = yield self.network.get_values_for_uris(
self.headers.hash().decode(), *uris)
resolver = Resolver(
self.headers.claim_trie_root,
self.headers.height,
self.transaction_class,
hash160_to_address=lambda x: self.hash160_to_address(x),
network=self.network)
defer.returnValue(
(yield resolver._handle_resolutions(resolutions, uris, page,
page_size)))
def claim_new_channel(self, channel_name, amount):
parsed_channel_name = parse_lbry_uri(channel_name)
if not parsed_channel_name.is_channel:
raise Exception("Invalid channel name")
elif (parsed_channel_name.path or parsed_channel_name.claim_id
or parsed_channel_name.bid_position
or parsed_channel_name.claim_sequence):
raise Exception(
"New channel claim should have no fields other than name")
log.info("Preparing to make certificate claim for %s", channel_name)
return self._claim_certificate(parsed_channel_name.name, amount)
def import_signed_claim_transaction(self, claim, claim_id, undo_info):
""" handle the import of claims/updates signed """
try:
decoded_claim = smart_decode(claim.value)
parsed_uri = parse_lbry_uri(claim.name)
if decoded_claim.has_signature:
cert_id = decoded_claim.certificate_id
else:
cert_id = None
except Exception as e:
logger.warn("decode error for lbry://{}#{}".format(
claim.name, claim_id))
decoded_claim = None
cert_id = None
if type(claim) == deserialize.NameClaim:
undo_info = self.import_signed_claim(claim, cert_id, claim_id,
undo_info)
elif type(claim) == deserialize.ClaimUpdate:
undo_info = self.import_signed_update(claim, cert_id, claim_id,
undo_info)
return undo_info
def _handle_resolve_uri_response(self,
uri,
resolution,
page=0,
page_size=10,
raw=False):
result = {}
claim_trie_root = self.claim_trie_root
parsed_uri = parse_lbry_uri(uri)
certificate = None
# parse an included certificate
if 'certificate' in resolution:
certificate_response = resolution['certificate']['result']
certificate_resolution_type = resolution['certificate'][
'resolution_type']
if certificate_resolution_type == "winning" and certificate_response:
if 'height' in certificate_response:
height = certificate_response['height']
depth = self.height - height
certificate_result = _verify_proof(
parsed_uri.name,
claim_trie_root,
certificate_response,
height,
depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result[
'certificate'] = self.parse_and_validate_claim_result(
certificate_result, raw=raw)
elif certificate_resolution_type == "claim_id":
result['certificate'] = self.parse_and_validate_claim_result(
certificate_response, raw=raw)
elif certificate_resolution_type == "sequence":
result['certificate'] = self.parse_and_validate_claim_result(
certificate_response, raw=raw)
else:
log.error("unknown response type: %s",
certificate_resolution_type)
if 'certificate' in result:
certificate = result['certificate']
if 'unverified_claims_in_channel' in resolution:
max_results = len(
resolution['unverified_claims_in_channel'])
result['claims_in_channel'] = max_results
else:
result['claims_in_channel'] = 0
else:
result['error'] = "claim not found"
result['success'] = False
result['uri'] = str(parsed_uri)
else:
certificate = None
# if this was a resolution for a name, parse the result
if 'claim' in resolution:
claim_response = resolution['claim']['result']
claim_resolution_type = resolution['claim']['resolution_type']
if claim_resolution_type == "winning" and claim_response:
if 'height' in claim_response:
height = claim_response['height']
depth = self.height - height
claim_result = _verify_proof(
parsed_uri.name,
claim_trie_root,
claim_response,
height,
depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result['claim'] = self.parse_and_validate_claim_result(
claim_result, certificate, raw)
elif claim_resolution_type == "claim_id":
result['claim'] = self.parse_and_validate_claim_result(
claim_response, certificate, raw)
elif claim_resolution_type == "sequence":
result['claim'] = self.parse_and_validate_claim_result(
claim_response, certificate, raw)
else:
log.error("unknown response type: %s", claim_resolution_type)
# if this was a resolution for a name in a channel make sure there is only one valid
# match
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
unverified_claims_for_name = resolution[
'unverified_claims_for_name']
channel_info = yield self.get_channel_claims_page(
unverified_claims_for_name, result['certificate'], page=1)
claims_in_channel, upper_bound = channel_info
if len(claims_in_channel) > 1:
log.error("Multiple signed claims for the same name")
elif not claims_in_channel:
log.error("No valid claims for this name for this channel")
else:
result['claim'] = claims_in_channel[0]
# parse and validate claims in a channel iteratively into pages of results
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
ids_to_check = resolution['unverified_claims_in_channel']
channel_info = yield self.get_channel_claims_page(
ids_to_check,
result['certificate'],
page=page,
page_size=page_size)
claims_in_channel, upper_bound = channel_info
if claims_in_channel:
result['claims_in_channel'] = claims_in_channel
elif 'error' not in result:
result['error'] = "claim not found"
result['success'] = False
result['uri'] = str(parsed_uri)
defer.returnValue(result)
def cmd_claimtrie_get_value_for_uri(self, block_hash, uri):
uri = str(uri)
block_hash = str(block_hash)
cache_key = block_hash + uri
if cache_key in self.short_term_cache:
return self.short_term_cache.get(cache_key)
try:
parsed_uri = parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.message}
result = {}
if parsed_uri.is_channel:
certificate = None
if parsed_uri.claim_id:
certificate_info = self.get_claim_info(parsed_uri.claim_id)
if certificate_info and certificate_info['name'] == parsed_uri.name:
certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
elif parsed_uri.claim_sequence:
claim_id = self.storage.get_claimid_for_nth_claim_to_name(str(parsed_uri.name),
parsed_uri.claim_sequence)
certificate_info = self.get_claim_info(str(claim_id))
if certificate_info:
certificate = {'resolution_type': SEQUENCE, 'result': certificate_info}
else:
certificate_info = self.cmd_claimtrie_getvalue(parsed_uri.name, block_hash)
if certificate_info:
certificate = {'resolution_type': WINNING, 'result': certificate_info}
if certificate and not parsed_uri.path:
result['certificate'] = certificate
channel_id = certificate['result'].get('claim_id') or certificate['result'].get('claimId')
channel_id = str(channel_id)
claim_ids_in_channel = self.storage.get_claims_signed_by(channel_id)
claims_in_channel = {cid: (self.storage.get_claim_name(cid),
self.storage.get_claim_height(cid))
for cid in claim_ids_in_channel}
result['unverified_claims_in_channel'] = claims_in_channel
elif certificate:
result['certificate'] = certificate
channel_id = certificate['result'].get('claim_id') or certificate['result'].get('claimId')
channel_id = str(channel_id)
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path)
claims_in_channel = {cid: (self.storage.get_claim_name(cid),
self.storage.get_claim_height(cid))
for cid in claim_ids_matching_name}
result['unverified_claims_for_name'] = claims_in_channel
else:
claim = None
if parsed_uri.claim_id:
claim_info = self.get_claim_info(parsed_uri.claim_id)
if claim_info and claim_info['name'] == parsed_uri.name:
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
elif parsed_uri.claim_sequence:
claim_id = self.storage.get_claimid_for_nth_claim_to_name(str(parsed_uri.name),
parsed_uri.claim_sequence)
claim_info = self.get_claim_info(str(claim_id))
if claim_info:
claim = {'resolution_type': SEQUENCE, 'result': claim_info}
else:
claim_info = self.cmd_claimtrie_getvalue(parsed_uri.name, block_hash)
if claim_info:
claim = {'resolution_type': WINNING, 'result': claim_info}
if (claim and
# is not an unclaimed winning name
(claim['resolution_type'] != WINNING or lbrycrd_proof_has_winning_claim(claim['result']['proof']))):
try:
claim_val = self.get_claim_info(claim['result']['claim_id'])
decoded = smart_decode(claim_val['value'])
if decoded.certificate_id:
certificate_info = self.get_claim_info(decoded.certificate_id)
if certificate_info:
certificate = {'resolution_type': CLAIM_ID,
'result': certificate_info}
result['certificate'] = certificate
except DecodeError:
pass
result['claim'] = claim
self.short_term_cache.put(cache_key, result)
return result
def cmd_claimtrie_get_value_for_uri(self, block_hash, uri):
uri = str(uri)
block_hash = str(block_hash)
try:
parsed_uri = parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.message}
result = {}
if parsed_uri.is_channel:
certificate = None
if parsed_uri.claim_id:
certificate_info = self.get_claim_info(parsed_uri.claim_id)
if certificate_info:
certificate = {
'resolution_type': CLAIM_ID,
'result': certificate_info
}
elif parsed_uri.claim_sequence:
claim_id = self.storage.get_claimid_for_nth_claim_to_name(
str(parsed_uri.name), parsed_uri.claim_sequence)
certificate_info = self.get_claim_info(str(claim_id))
if certificate_info:
certificate = {
'resolution_type': SEQUENCE,
'result': certificate_info
}
else:
certificate_info = self.cmd_claimtrie_getvalue(
parsed_uri.name, block_hash)
if certificate_info:
certificate = {
'resolution_type': WINNING,
'result': certificate_info
}
if certificate and not parsed_uri.path:
result['certificate'] = certificate
channel_id = certificate['result'].get(
'claim_id') or certificate['result'].get('claimId')
channel_id = str(channel_id)
claim_ids_in_channel = self.storage.get_claims_signed_by(
channel_id)
claims_in_channel = {
cid: (self.storage.get_claim_name(cid),
self.storage.get_claim_height(cid))
for cid in claim_ids_in_channel
}
result['unverified_claims_in_channel'] = claims_in_channel
elif certificate:
result['certificate'] = certificate
channel_id = certificate['result'].get(
'claim_id') or certificate['result'].get('claimId')
channel_id = str(channel_id)
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(
channel_id, parsed_uri.path)
claims_in_channel = {
cid: (self.storage.get_claim_name(cid),
self.storage.get_claim_height(cid))
for cid in claim_ids_matching_name
}
result['unverified_claims_for_name'] = claims_in_channel
else:
claim = None
if parsed_uri.claim_id:
claim_info = self.get_claim_info(parsed_uri.claim_id)
if claim_info:
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
elif parsed_uri.claim_sequence:
claim_id = self.storage.get_claimid_for_nth_claim_to_name(
str(parsed_uri.name), parsed_uri.claim_sequence)
claim_info = self.get_claim_info(str(claim_id))
if claim_info:
claim = {'resolution_type': SEQUENCE, 'result': claim_info}
else:
claim_info = self.cmd_claimtrie_getvalue(
parsed_uri.name, block_hash)
if claim_info:
claim = {'resolution_type': WINNING, 'result': claim_info}
if (claim and
# is not an unclaimed winning name
(claim['resolution_type'] != WINNING or
lbrycrd_proof_has_winning_claim(claim['result']['proof']))):
try:
claim_val = self.get_claim_info(
claim['result']['claim_id'])
decoded = smart_decode(claim_val['value'])
if decoded.certificate_id:
certificate_info = self.get_claim_info(
decoded.certificate_id)
if certificate_info:
certificate = {
'resolution_type': CLAIM_ID,
'result': certificate_info
}
result['certificate'] = certificate
except DecodeError:
pass
result['claim'] = claim
return result
async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None):
# TODO: this thing is huge, refactor
CLAIM_ID = "claim_id"
WINNING = "winning"
SEQUENCE = "sequence"
uri = uri
block_hash = block_hash
try:
parsed_uri = parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.message}
result = {}
if parsed_uri.is_channel:
certificate = None
# TODO: this is also done on the else, refactor
if parsed_uri.claim_id:
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if certificate_info and certificate_info['name'] == parsed_uri.name:
certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
elif parsed_uri.claim_sequence:
certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
if certificate_info:
certificate = {'resolution_type': SEQUENCE, 'result': certificate_info}
else:
certificate_info = await self.claimtrie_getvalue(parsed_uri.name, block_hash)
if certificate_info:
certificate = {'resolution_type': WINNING, 'result': certificate_info}
if certificate and 'claim_id' not in certificate['result']:
return result
if certificate and not parsed_uri.path:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id)
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel if claim}
elif certificate:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path)
claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name)
claims_in_channel = {claim['claim_id']: (claim['name'], claim['height'])
for claim in claims}
result['unverified_claims_for_name'] = claims_in_channel
else:
claim = None
if parsed_uri.claim_id:
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if claim_info and claim_info['name'] == parsed_uri.name:
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
elif parsed_uri.claim_sequence:
claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
if claim_info:
claim = {'resolution_type': SEQUENCE, 'result': claim_info}
else:
claim_info = await self.claimtrie_getvalue(parsed_uri.name, block_hash)
if claim_info:
claim = {'resolution_type': WINNING, 'result': claim_info}
if (claim and
# is not an unclaimed winning name
(claim['resolution_type'] != WINNING or proof_has_winning_claim(claim['result']['proof']))):
raw_claim_id = unhexlify(claim['result']['claim_id'])[::-1]
raw_certificate_id = self.bp.get_claim_info(raw_claim_id).cert_id
if raw_certificate_id:
certificate_id = hash_to_hex_str(raw_certificate_id)
certificate = await self.claimtrie_getclaimbyid(certificate_id)
if certificate:
certificate = {'resolution_type': CLAIM_ID,
'result': certificate}
result['certificate'] = certificate
result['claim'] = claim
return result
async def get_names(app_db, names_db, height_db):
current_height = await rpc("getblockcount")
expiring_height = current_height - 262974
expired_names, types, expired_txids_by_name, expired_chan = update_db(app_db, names_db, height_db, expiring_height)
trie = await rpc("getclaimsintrie")
expiring_names = {}
valid_expiring_names = {}
renewed_names = {}
expiring_channels = {}
signed_expiring_claims = {}
print(len(expired_names.keys()))
for name, claims in [(r['name'], r['claims']) for r in trie]:
max_height = max(int(c['height']) for c in claims)
current_value = [c['value'] for c in claims if int(c['height']) == max_height][0]
claim_id = [c['claimId'] for c in claims if int(c['height']) == max_height][0]
if max_height < (expiring_height + 576*90): # ~90 days ahead
expiring_names[name] = (max_height, claim_id)
try:
parsed = parse_lbry_uri(name)
decoded = smart_decode(current_value.encode('ISO-8859-1'))
claim_type = decoded.get('claimType', 'unknown')
if claim_type == 'certificateType' or parsed.is_channel:
expiring_channels[name] = (max_height, claim_id)
if decoded.signature:
signed_expiring_claims[name] = (max_height, claim_id)
types.add(claim_type)
valid_expiring_names[name] = expiring_names[name]
except DecodeError:
print("Could not decode %s - %s" % (name, current_value.encode('ISO-8859-1')))
pass
except (UnicodeDecodeError, URIParseError, AssertionError) as e:
print("Could not decode %s - %s" % (name, e))
pass
if name in expired_names:
renewed_names[name] = expired_names.pop(name)
print("removing %s as it is also claimed at %s" % (name, max_height))
# verify for spent txs in expired set
async with aiohttp.ClientSession(json_serialize=ujson.dumps) as session:
for name in list(expired_names.keys()):
txid = expired_txids_by_name[name]
claims = await rpc("getclaimsfortx", [txid], session)
if not claims:
del expired_names[name]
print("Spent expired: %s - %s" % (name, txid))
[expired_chan.pop(name) for name in list(expired_chan.keys()) if name not in expired_names]
for (key, name) in list(app_db.iterator()):
if name.decode('utf8') not in expired_names:
app_db.delete(key)
print(len(expired_names.keys()))
print(types)
print("Renewed names: %s" % len(renewed_names))
print("Expired names: %s" % len(expired_names))
print("Expired channels: %s" % len(expired_chan))
print("Expiring names: %s" % len(expiring_names))
print("Expiring channels: %s" % len(expiring_channels))
print("Signed expiring names: %s" % len(signed_expiring_claims))
print("Valid expiring names: %s" % len(valid_expiring_names))
expired_stats = extract_stats(expired_names, 'expired')
expired_chan_stats = extract_stats(expired_chan, 'expired channels')
expiring_stats = extract_stats(expiring_names, 'expiring')
expiring_channels_stats = extract_stats(expiring_channels, 'expiring channels')
signed_expiring_stats = extract_stats(signed_expiring_claims, 'signed expiring channels')
valid_expiring_stats = extract_stats(valid_expiring_names, 'valid and expiring')
app_db.put(b'stats', ujson.dumps([expired_stats, expired_chan_stats, expiring_stats, valid_expiring_stats,
expiring_channels_stats, signed_expiring_stats]).encode('utf8'))
working_data = {'expired': sorted_values(expired_names), 'expiring': sorted_values(expiring_names),
'valid_expiring': sorted_values(valid_expiring_names),
'expired_channels': sorted_values(expired_chan),
'expiring_channels': sorted_values(expiring_channels),
'known_types': list(types), 'last_run_height': current_height}
app_db.put(b'working_data', ujson.dumps(working_data).encode('utf8'))