async def resolve(self, page, page_size, *uris): for uri in uris: try: parse_lbry_uri(uri) except URIParseError as err: return {'error': err.args[0]} resolutions = await self.network.get_values_for_uris( self.headers.hash().decode(), *uris) return await self.resolver._handle_resolutions(resolutions, uris, page, page_size)
def resolve(self, page, page_size, *uris): for uri in uris: try: parse_lbry_uri(uri) except URIParseError as err: defer.returnValue({'error': err.message}) resolutions = yield self.network.get_values_for_uris( self.headers.hash().decode(), *uris) return (yield self.resolver._handle_resolutions(resolutions, uris, page, page_size))
def _checksig(self, name, value, address): try: parse_lbry_uri(name.decode()) # skip invalid names cert_id = Claim.FromString( value).publisherSignature.certificateId[::-1] or None if not self.should_validate_signatures: return cert_id if cert_id: cert_claim = self.get_claim_info(cert_id) if cert_claim: certificate = smart_decode(cert_claim.value) claim_dict = smart_decode(value) claim_dict.validate_signature(address, certificate) return cert_id except Exception as e: pass
def update_db(app_db, names_db, height_db, expiring_height): outpoint_db = plyvel.DB('db/claim_outpoint/') def get_txid_for_claim_id(claim_id): txid_nout = outpoint_db.get(claim_id) txid = txid_nout[0:64] return txid expired_names, known_types, txids, expired_channels = {}, set(), {}, {} [app_db.delete(x[0]) for x in app_db] with app_db.write_batch() as writer: for (claim_id, height) in height_db: key = struct.pack('>I40s', int(height), claim_id) try: name = names_db.get(claim_id).decode('utf8') parsed = parse_lbry_uri(name) decoded = smart_decode(values_db.get(claim_id)) known_types.add(decoded.get('claimType', 'unknown')) if decoded.get('claimType') == 'certificateType' or parsed.is_channel: expired_channels[name] = (height, claim_id) if int(height) < expiring_height: expired_names[name] = (int(height), claim_id) txids[name] = get_txid_for_claim_id(claim_id) writer.put(key, name.encode('utf8')) except (DecodeError, UnicodeDecodeError, URIParseError): continue return expired_names, known_types, txids, expired_channels
def resolve(self, page, page_size, *uris): for uri in uris: try: parse_lbry_uri(uri) except URIParseError as err: defer.returnValue({'error': err.message}) resolutions = yield self.network.get_values_for_uris( self.headers.hash().decode(), *uris) resolver = Resolver( self.headers.claim_trie_root, self.headers.height, self.transaction_class, hash160_to_address=lambda x: self.hash160_to_address(x), network=self.network) defer.returnValue( (yield resolver._handle_resolutions(resolutions, uris, page, page_size)))
def claim_new_channel(self, channel_name, amount): parsed_channel_name = parse_lbry_uri(channel_name) if not parsed_channel_name.is_channel: raise Exception("Invalid channel name") elif (parsed_channel_name.path or parsed_channel_name.claim_id or parsed_channel_name.bid_position or parsed_channel_name.claim_sequence): raise Exception( "New channel claim should have no fields other than name") log.info("Preparing to make certificate claim for %s", channel_name) return self._claim_certificate(parsed_channel_name.name, amount)
def import_signed_claim_transaction(self, claim, claim_id, undo_info): """ handle the import of claims/updates signed """ try: decoded_claim = smart_decode(claim.value) parsed_uri = parse_lbry_uri(claim.name) if decoded_claim.has_signature: cert_id = decoded_claim.certificate_id else: cert_id = None except Exception as e: logger.warn("decode error for lbry://{}#{}".format( claim.name, claim_id)) decoded_claim = None cert_id = None if type(claim) == deserialize.NameClaim: undo_info = self.import_signed_claim(claim, cert_id, claim_id, undo_info) elif type(claim) == deserialize.ClaimUpdate: undo_info = self.import_signed_update(claim, cert_id, claim_id, undo_info) return undo_info
def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False): result = {} claim_trie_root = self.claim_trie_root parsed_uri = parse_lbry_uri(uri) certificate = None # parse an included certificate if 'certificate' in resolution: certificate_response = resolution['certificate']['result'] certificate_resolution_type = resolution['certificate'][ 'resolution_type'] if certificate_resolution_type == "winning" and certificate_response: if 'height' in certificate_response: height = certificate_response['height'] depth = self.height - height certificate_result = _verify_proof( parsed_uri.name, claim_trie_root, certificate_response, height, depth, transaction_class=self.transaction_class, hash160_to_address=self.hash160_to_address) result[ 'certificate'] = self.parse_and_validate_claim_result( certificate_result, raw=raw) elif certificate_resolution_type == "claim_id": result['certificate'] = self.parse_and_validate_claim_result( certificate_response, raw=raw) elif certificate_resolution_type == "sequence": result['certificate'] = self.parse_and_validate_claim_result( certificate_response, raw=raw) else: log.error("unknown response type: %s", certificate_resolution_type) if 'certificate' in result: certificate = result['certificate'] if 'unverified_claims_in_channel' in resolution: max_results = len( resolution['unverified_claims_in_channel']) result['claims_in_channel'] = max_results else: result['claims_in_channel'] = 0 else: result['error'] = "claim not found" result['success'] = False result['uri'] = str(parsed_uri) else: certificate = None # if this was a resolution for a name, parse the result if 'claim' in resolution: claim_response = resolution['claim']['result'] claim_resolution_type = resolution['claim']['resolution_type'] if claim_resolution_type == "winning" and claim_response: if 'height' in claim_response: height = claim_response['height'] depth = self.height - height claim_result = _verify_proof( parsed_uri.name, claim_trie_root, claim_response, height, depth, transaction_class=self.transaction_class, hash160_to_address=self.hash160_to_address) result['claim'] = self.parse_and_validate_claim_result( claim_result, certificate, raw) elif claim_resolution_type == "claim_id": result['claim'] = self.parse_and_validate_claim_result( claim_response, certificate, raw) elif claim_resolution_type == "sequence": result['claim'] = self.parse_and_validate_claim_result( claim_response, certificate, raw) else: log.error("unknown response type: %s", claim_resolution_type) # if this was a resolution for a name in a channel make sure there is only one valid # match elif 'unverified_claims_for_name' in resolution and 'certificate' in result: unverified_claims_for_name = resolution[ 'unverified_claims_for_name'] channel_info = yield self.get_channel_claims_page( unverified_claims_for_name, result['certificate'], page=1) claims_in_channel, upper_bound = channel_info if len(claims_in_channel) > 1: log.error("Multiple signed claims for the same name") elif not claims_in_channel: log.error("No valid claims for this name for this channel") else: result['claim'] = claims_in_channel[0] # parse and validate claims in a channel iteratively into pages of results elif 'unverified_claims_in_channel' in resolution and 'certificate' in result: ids_to_check = resolution['unverified_claims_in_channel'] channel_info = yield self.get_channel_claims_page( ids_to_check, result['certificate'], page=page, page_size=page_size) claims_in_channel, upper_bound = channel_info if claims_in_channel: result['claims_in_channel'] = claims_in_channel elif 'error' not in result: result['error'] = "claim not found" result['success'] = False result['uri'] = str(parsed_uri) defer.returnValue(result)
def cmd_claimtrie_get_value_for_uri(self, block_hash, uri): uri = str(uri) block_hash = str(block_hash) cache_key = block_hash + uri if cache_key in self.short_term_cache: return self.short_term_cache.get(cache_key) try: parsed_uri = parse_lbry_uri(uri) except URIParseError as err: return {'error': err.message} result = {} if parsed_uri.is_channel: certificate = None if parsed_uri.claim_id: certificate_info = self.get_claim_info(parsed_uri.claim_id) if certificate_info and certificate_info['name'] == parsed_uri.name: certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info} elif parsed_uri.claim_sequence: claim_id = self.storage.get_claimid_for_nth_claim_to_name(str(parsed_uri.name), parsed_uri.claim_sequence) certificate_info = self.get_claim_info(str(claim_id)) if certificate_info: certificate = {'resolution_type': SEQUENCE, 'result': certificate_info} else: certificate_info = self.cmd_claimtrie_getvalue(parsed_uri.name, block_hash) if certificate_info: certificate = {'resolution_type': WINNING, 'result': certificate_info} if certificate and not parsed_uri.path: result['certificate'] = certificate channel_id = certificate['result'].get('claim_id') or certificate['result'].get('claimId') channel_id = str(channel_id) claim_ids_in_channel = self.storage.get_claims_signed_by(channel_id) claims_in_channel = {cid: (self.storage.get_claim_name(cid), self.storage.get_claim_height(cid)) for cid in claim_ids_in_channel} result['unverified_claims_in_channel'] = claims_in_channel elif certificate: result['certificate'] = certificate channel_id = certificate['result'].get('claim_id') or certificate['result'].get('claimId') channel_id = str(channel_id) claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path) claims_in_channel = {cid: (self.storage.get_claim_name(cid), self.storage.get_claim_height(cid)) for cid in claim_ids_matching_name} result['unverified_claims_for_name'] = claims_in_channel else: claim = None if parsed_uri.claim_id: claim_info = self.get_claim_info(parsed_uri.claim_id) if claim_info and claim_info['name'] == parsed_uri.name: claim = {'resolution_type': CLAIM_ID, 'result': claim_info} elif parsed_uri.claim_sequence: claim_id = self.storage.get_claimid_for_nth_claim_to_name(str(parsed_uri.name), parsed_uri.claim_sequence) claim_info = self.get_claim_info(str(claim_id)) if claim_info: claim = {'resolution_type': SEQUENCE, 'result': claim_info} else: claim_info = self.cmd_claimtrie_getvalue(parsed_uri.name, block_hash) if claim_info: claim = {'resolution_type': WINNING, 'result': claim_info} if (claim and # is not an unclaimed winning name (claim['resolution_type'] != WINNING or lbrycrd_proof_has_winning_claim(claim['result']['proof']))): try: claim_val = self.get_claim_info(claim['result']['claim_id']) decoded = smart_decode(claim_val['value']) if decoded.certificate_id: certificate_info = self.get_claim_info(decoded.certificate_id) if certificate_info: certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info} result['certificate'] = certificate except DecodeError: pass result['claim'] = claim self.short_term_cache.put(cache_key, result) return result
def cmd_claimtrie_get_value_for_uri(self, block_hash, uri): uri = str(uri) block_hash = str(block_hash) try: parsed_uri = parse_lbry_uri(uri) except URIParseError as err: return {'error': err.message} result = {} if parsed_uri.is_channel: certificate = None if parsed_uri.claim_id: certificate_info = self.get_claim_info(parsed_uri.claim_id) if certificate_info: certificate = { 'resolution_type': CLAIM_ID, 'result': certificate_info } elif parsed_uri.claim_sequence: claim_id = self.storage.get_claimid_for_nth_claim_to_name( str(parsed_uri.name), parsed_uri.claim_sequence) certificate_info = self.get_claim_info(str(claim_id)) if certificate_info: certificate = { 'resolution_type': SEQUENCE, 'result': certificate_info } else: certificate_info = self.cmd_claimtrie_getvalue( parsed_uri.name, block_hash) if certificate_info: certificate = { 'resolution_type': WINNING, 'result': certificate_info } if certificate and not parsed_uri.path: result['certificate'] = certificate channel_id = certificate['result'].get( 'claim_id') or certificate['result'].get('claimId') channel_id = str(channel_id) claim_ids_in_channel = self.storage.get_claims_signed_by( channel_id) claims_in_channel = { cid: (self.storage.get_claim_name(cid), self.storage.get_claim_height(cid)) for cid in claim_ids_in_channel } result['unverified_claims_in_channel'] = claims_in_channel elif certificate: result['certificate'] = certificate channel_id = certificate['result'].get( 'claim_id') or certificate['result'].get('claimId') channel_id = str(channel_id) claim_ids_matching_name = self.get_signed_claims_with_name_for_channel( channel_id, parsed_uri.path) claims_in_channel = { cid: (self.storage.get_claim_name(cid), self.storage.get_claim_height(cid)) for cid in claim_ids_matching_name } result['unverified_claims_for_name'] = claims_in_channel else: claim = None if parsed_uri.claim_id: claim_info = self.get_claim_info(parsed_uri.claim_id) if claim_info: claim = {'resolution_type': CLAIM_ID, 'result': claim_info} elif parsed_uri.claim_sequence: claim_id = self.storage.get_claimid_for_nth_claim_to_name( str(parsed_uri.name), parsed_uri.claim_sequence) claim_info = self.get_claim_info(str(claim_id)) if claim_info: claim = {'resolution_type': SEQUENCE, 'result': claim_info} else: claim_info = self.cmd_claimtrie_getvalue( parsed_uri.name, block_hash) if claim_info: claim = {'resolution_type': WINNING, 'result': claim_info} if (claim and # is not an unclaimed winning name (claim['resolution_type'] != WINNING or lbrycrd_proof_has_winning_claim(claim['result']['proof']))): try: claim_val = self.get_claim_info( claim['result']['claim_id']) decoded = smart_decode(claim_val['value']) if decoded.certificate_id: certificate_info = self.get_claim_info( decoded.certificate_id) if certificate_info: certificate = { 'resolution_type': CLAIM_ID, 'result': certificate_info } result['certificate'] = certificate except DecodeError: pass result['claim'] = claim return result
async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None): # TODO: this thing is huge, refactor CLAIM_ID = "claim_id" WINNING = "winning" SEQUENCE = "sequence" uri = uri block_hash = block_hash try: parsed_uri = parse_lbry_uri(uri) except URIParseError as err: return {'error': err.message} result = {} if parsed_uri.is_channel: certificate = None # TODO: this is also done on the else, refactor if parsed_uri.claim_id: certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id) if certificate_info and certificate_info['name'] == parsed_uri.name: certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info} elif parsed_uri.claim_sequence: certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence) if certificate_info: certificate = {'resolution_type': SEQUENCE, 'result': certificate_info} else: certificate_info = await self.claimtrie_getvalue(parsed_uri.name, block_hash) if certificate_info: certificate = {'resolution_type': WINNING, 'result': certificate_info} if certificate and 'claim_id' not in certificate['result']: return result if certificate and not parsed_uri.path: result['certificate'] = certificate channel_id = certificate['result']['claim_id'] claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id) result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height']) for claim in claims_in_channel if claim} elif certificate: result['certificate'] = certificate channel_id = certificate['result']['claim_id'] claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path) claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name) claims_in_channel = {claim['claim_id']: (claim['name'], claim['height']) for claim in claims} result['unverified_claims_for_name'] = claims_in_channel else: claim = None if parsed_uri.claim_id: claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id) if claim_info and claim_info['name'] == parsed_uri.name: claim = {'resolution_type': CLAIM_ID, 'result': claim_info} elif parsed_uri.claim_sequence: claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence) if claim_info: claim = {'resolution_type': SEQUENCE, 'result': claim_info} else: claim_info = await self.claimtrie_getvalue(parsed_uri.name, block_hash) if claim_info: claim = {'resolution_type': WINNING, 'result': claim_info} if (claim and # is not an unclaimed winning name (claim['resolution_type'] != WINNING or proof_has_winning_claim(claim['result']['proof']))): raw_claim_id = unhexlify(claim['result']['claim_id'])[::-1] raw_certificate_id = self.bp.get_claim_info(raw_claim_id).cert_id if raw_certificate_id: certificate_id = hash_to_hex_str(raw_certificate_id) certificate = await self.claimtrie_getclaimbyid(certificate_id) if certificate: certificate = {'resolution_type': CLAIM_ID, 'result': certificate} result['certificate'] = certificate result['claim'] = claim return result
async def get_names(app_db, names_db, height_db): current_height = await rpc("getblockcount") expiring_height = current_height - 262974 expired_names, types, expired_txids_by_name, expired_chan = update_db(app_db, names_db, height_db, expiring_height) trie = await rpc("getclaimsintrie") expiring_names = {} valid_expiring_names = {} renewed_names = {} expiring_channels = {} signed_expiring_claims = {} print(len(expired_names.keys())) for name, claims in [(r['name'], r['claims']) for r in trie]: max_height = max(int(c['height']) for c in claims) current_value = [c['value'] for c in claims if int(c['height']) == max_height][0] claim_id = [c['claimId'] for c in claims if int(c['height']) == max_height][0] if max_height < (expiring_height + 576*90): # ~90 days ahead expiring_names[name] = (max_height, claim_id) try: parsed = parse_lbry_uri(name) decoded = smart_decode(current_value.encode('ISO-8859-1')) claim_type = decoded.get('claimType', 'unknown') if claim_type == 'certificateType' or parsed.is_channel: expiring_channels[name] = (max_height, claim_id) if decoded.signature: signed_expiring_claims[name] = (max_height, claim_id) types.add(claim_type) valid_expiring_names[name] = expiring_names[name] except DecodeError: print("Could not decode %s - %s" % (name, current_value.encode('ISO-8859-1'))) pass except (UnicodeDecodeError, URIParseError, AssertionError) as e: print("Could not decode %s - %s" % (name, e)) pass if name in expired_names: renewed_names[name] = expired_names.pop(name) print("removing %s as it is also claimed at %s" % (name, max_height)) # verify for spent txs in expired set async with aiohttp.ClientSession(json_serialize=ujson.dumps) as session: for name in list(expired_names.keys()): txid = expired_txids_by_name[name] claims = await rpc("getclaimsfortx", [txid], session) if not claims: del expired_names[name] print("Spent expired: %s - %s" % (name, txid)) [expired_chan.pop(name) for name in list(expired_chan.keys()) if name not in expired_names] for (key, name) in list(app_db.iterator()): if name.decode('utf8') not in expired_names: app_db.delete(key) print(len(expired_names.keys())) print(types) print("Renewed names: %s" % len(renewed_names)) print("Expired names: %s" % len(expired_names)) print("Expired channels: %s" % len(expired_chan)) print("Expiring names: %s" % len(expiring_names)) print("Expiring channels: %s" % len(expiring_channels)) print("Signed expiring names: %s" % len(signed_expiring_claims)) print("Valid expiring names: %s" % len(valid_expiring_names)) expired_stats = extract_stats(expired_names, 'expired') expired_chan_stats = extract_stats(expired_chan, 'expired channels') expiring_stats = extract_stats(expiring_names, 'expiring') expiring_channels_stats = extract_stats(expiring_channels, 'expiring channels') signed_expiring_stats = extract_stats(signed_expiring_claims, 'signed expiring channels') valid_expiring_stats = extract_stats(valid_expiring_names, 'valid and expiring') app_db.put(b'stats', ujson.dumps([expired_stats, expired_chan_stats, expiring_stats, valid_expiring_stats, expiring_channels_stats, signed_expiring_stats]).encode('utf8')) working_data = {'expired': sorted_values(expired_names), 'expiring': sorted_values(expiring_names), 'valid_expiring': sorted_values(valid_expiring_names), 'expired_channels': sorted_values(expired_chan), 'expiring_channels': sorted_values(expiring_channels), 'known_types': list(types), 'last_run_height': current_height} app_db.put(b'working_data', ujson.dumps(working_data).encode('utf8'))