def get_block_distance(atxid, btxid): if type(atxid) == bytes: atxid = bytes_to_hex(atxid) if type(btxid) == bytes: btxid = bytes_to_hex(btxid) block_a = get_blockchain_connection().getblock(atxid) block_b = get_blockchain_connection().getblock(btxid) height_a = int(block_a["height"]) height_b = int(block_b["height"]) return abs(height_a - height_b)
def generate_signed_ticket(cls, ticket): signed_ticket = Signature( dictionary={ "signature": get_blockchain_connection().pastelid_sign( ticket.serialize_base64()), "pastelid": get_blockchain_connection().pastelid }) # make sure we validate correctly signed_ticket.validate(ticket) return signed_ticket
def generate_signed_ticket(self, ticket): # we sign cnode_container of Regticket. as anything passed to `pastelid_sign` is converted to base64 # we just send bytes(json(cnode_container)) there. signature = get_blockchain_connection().pastelid_sign( ticket.serialize_base64()) signed_ticket = Signature(dictionary={ "signature": signature, "pastelid": get_blockchain_connection().pastelid, }) # make sure we validate correctly signed_ticket.validate(ticket) return signed_ticket
def __init__(self, data: list, receiver_pastel_id: str, version: int = MAX_SUPPORTED_VERSION, container: Optional[dict] = None): """ Must be created without `container` argument on client side and with one on server side. """ self.data = data self.receiver_pastel_id = receiver_pastel_id self.version = version if container and (container['data'] != self.data or container['receiver_id'] != receiver_pastel_id): raise ValueError( 'Data in container and data/receiver_pastel_id does must match!' ) self.container = container if container is not None else { "version": self.version, "sender_id": get_blockchain_connection().pastelid, "receiver_id": self.receiver_pastel_id, "data": self.data, "nonce": nacl.utils.random(NONCE_LENGTH), "timestamp": time.time(), "signature": '', } self.sender_id = self.container['sender_id']
def masternode_image_upload_request_mn0(data, *args, **kwargs): # parse inputs upload_code = data['upload_code'] image_data = data['image_data'] mn_ticket_logger.info( 'Masternode image upload received, upload_code: {}'.format( upload_code)) sender_id = kwargs.get('sender_id') MASTERNODE_DB.connect(reuse_if_open=True) try: regticket_db = Regticket.get(upload_code=upload_code) regticket = RegistrationTicket(serialized=regticket_db.regticket) if regticket.author != sender_id: raise Exception( 'Given upload code was created by other public key') mn_ticket_logger.info( 'Given upload code exists with required public key') except DoesNotExist: mn_ticket_logger.exception( 'Given upload code DOES NOT exists with required public key') raise result = get_blockchain_connection().getlocalfee() fee = result['localfee'] regticket_db.image_data = image_data regticket_db.localfee = fee regticket_db.save() return fee
def __send_to_address(self, address, amount, comment=""): try: result = get_blockchain_connection().sendtoaddress( address, amount, public_comment=comment) except JSONRPCException as exc: return str(exc) else: return result
async def register_sell_ticket(self, txid, price): act_txid = txid result = get_blockchain_connection().register_sell_ticket( act_txid, price) SellticketDB.create(pastelid=self.pastelid, price=price, act_ticket_txid=act_txid) return result
def generate_regticket(cls, image_data: bytes, regticket_data: dict): image = ImageData( dictionary={ "image": image_data, "lubychunks": ImageData.generate_luby_chunks(image_data), "thumbnail": ImageData.generate_thumbnail(image_data), }) image.validate() blocknum = get_blockchain_connection().getblockcount() return RegistrationTicket( dictionary={ "artist_name": regticket_data.get('artist_name', ''), "artist_website": regticket_data.get('artist_website', ''), "artist_written_statement": regticket_data.get('artist_written_statement', ''), "artwork_title": regticket_data.get('artwork_title', ''), "artwork_series_name": regticket_data.get('artwork_series_name', ''), "artwork_creation_video_youtube_url": regticket_data.get('artwork_creation_video_youtube_url', ''), "artwork_keyword_set": regticket_data.get('artwork_keyword_set', ''), "total_copies": int(regticket_data.get('total_copies', 0)), # "copy_price": copy_price, "fingerprints": image.generate_fingerprints(), "lubyhashes": image.get_luby_hashes(), "lubyseeds": image.get_luby_seeds(), "thumbnailhash": image.get_thumbnail_hash(), "author": get_blockchain_connection().pastelid, "order_block_txid": get_blockchain_connection().getbestblockhash(), "blocknum": blocknum, "imagedata_hash": image.get_artwork_hash(), })
def get_owned_chunks(): """ Return list of database chunk records we should store. """ current_mn_id = Masternode.get( pastel_id=get_blockchain_connection().pastelid).id chunks_ranked_qs = ChunkMnRanked.select().where( ChunkMnRanked.masternode_id == current_mn_id) return [c.chunk for c in chunks_ranked_qs]
async def is_burn_10_tx_amount_valid(regticket, txid): networkfee_result = get_blockchain_connection().getnetworkfee() networkfee = networkfee_result['networkfee'] tx_amounts = [] sleep_counter = 0 while True: try: raw_tx_data = get_blockchain_connection().getrawtransaction( txid, verbose=1) break except Exception as ex: await asyncio.sleep(5) if sleep_counter > 10: raise ex for vout in raw_tx_data['vout']: tx_amounts.append(vout['value']) if regticket.localfee is not None: # we're main masternode (MN0) valid = False for tx_amount in tx_amounts: if regticket.localfee * Decimal( '0.099') <= tx_amount <= regticket.localfee * Decimal( '0.101'): valid = True break if not valid: return False, 'Wrong fee amount' regticket.is_valid_mn0 = True regticket.save() return True, None else: # we're MN1 or MN2 # we don't know exact MN0 fee, but it should be almost equal to the networkfee valid = False for tx_amount in tx_amounts: if networkfee * 0.09 <= tx_amount <= networkfee * 0.11: valid = True break if not valid: return False, 'Payment amount differs with 10% of fee size.' else: return True, None
def sign(self) -> None: """ Adds signature to the container if one not added yet. """ if self.container['signature']: return ensure_types_for_v1(self.container) container_serialized = msgpack.packb(self.container, default=default, use_bin_type=True) self.container['signature'] = get_blockchain_connection( ).pastelid_sign(get_pynode_digest_bytes_base64(container_serialized))
def reconstruct(serialized: bytes) -> 'RPCMessage': if len(serialized) > Settings.RPC_MSG_SIZELIMIT: raise ValueError("Message is too large: %s > %s" % (len(serialized), Settings.RPC_MSG_SIZELIMIT)) container = msgpack.unpackb(serialized, ext_hook=ext_hook, raw=False) if not validate_container_format(container): raise ValueError('Invalid container format') # validate receiver id is us if container['receiver_id'] != get_blockchain_connection().pastelid: raise ValueError("receiver_id is not us (%s != %s)" % (container['receiver_id'], get_blockchain_connection().pastelid)) require_true(container['timestamp'] > time.time() - 60) require_true(container['timestamp'] < time.time() + 60) return RPCMessage(container['data'], container['receiver_id'], container=container)
def write_to_blockchain(self): ticket = RegistrationTicket(serialized=self.regticket) current_block = get_blockchain_connection().getblockcount() # verify if confirmation receive for 5 blocks or less from regticket creation. if current_block - ticket.blocknum > Settings.MAX_CONFIRMATION_DISTANCE_IN_BLOCKS: self.status = REGTICKET_STATUS_ERROR error_msg = 'Second confirmation received too late - current block {}, regticket block: {}'. \ format(current_block, ticket.blocknum) self.error = error_msg raise Exception(error_msg) artist_signature = Signature(serialized=self.artists_signature_ticket) mn2_signature = Signature(serialized=self.mn1_serialized_signature) mn3_signature = Signature(serialized=self.mn2_serialized_signature) signatures_dict = { "artist": { artist_signature.pastelid: artist_signature.signature }, "mn2": { mn2_signature.pastelid: mn2_signature.signature }, "mn3": { mn3_signature.pastelid: mn3_signature.signature } } # write final ticket into blockchain art_ticket_data = { 'cnode_package': ticket.serialize_base64(), 'signatures_dict': signatures_dict, 'key1': ticket.base64_imagedatahash, # artist_signature.pastelid, 'key2': ticket.base64_imagedatahash, 'fee': int(self.localfee) } bc_response = get_blockchain_connection().register_art_ticket( **art_ticket_data) return bc_response
def masternodes_by_distance_from_image(image_hash): # FIXME: hardcoded list for testing only # TODO: how should it behave: # - get masternodelist (with pastelid) # calculate some hash for each node based on pastelid # calculated distance between node hash and image_hash # return masternodes sorted by this hash masternodes = get_blockchain_connection().masternode_list().values() mn_clients = [] for mn in masternodes: mn_clients.append( RPCClient(mn['extKey'], mn['extAddress'].split(':')[0], mn['extAddress'].split(':')[1])) return mn_clients
async def get_status(self, request): self.__logger.info('Status request received') # filter = await request.content.read() active_mns = list(Masternode.get_active_nodes()) # mn_status = {} result = { "status": "alive", "pastel_id": get_blockchain_connection().pastelid, "masternodes": { "count": len(active_mns) } } return web.json_response(result)
def verify(self) -> bool: """ Verify sender signature """ # remove signature from container # msgpack it, get digest, verify container = copy(self.container) signature = container['signature'] container['signature'] = '' container_serialized = msgpack.packb(container, default=default, use_bin_type=True) return get_blockchain_connection().pastelid_verify( get_pynode_digest_bytes_base64(container_serialized), signature, container['sender_id'])
def get_missing_chunk_ids(pastel_id=None): """ :param pastel_id: str :return: list of str chunkd ids (big integer numbers wrapper to string as they're stored in DB """ if not pastel_id: pastel_id = get_blockchain_connection().pastelid # return chunks that we're owner of but don't have it in the storage try: current_mn_id = Masternode.get(pastel_id=pastel_id).id except DoesNotExist: return [] chunks_ranked_qs = ChunkMnRanked.select().join(Chunk).where( (ChunkMnRanked.masternode_id == current_mn_id) & (Chunk.stored == False) & (Chunk.attempts_to_load < 1000)) return [c.chunk.chunk_id for c in chunks_ranked_qs]
async def is_burn_10_tx_height_valid(regticket, txid): regticket = RegistrationTicket(serialized=regticket.regticket) sleep_counter = 0 while True: try: raw_tx_data = get_blockchain_connection().getrawtransaction( txid, verbose=1) break except Exception as ex: await asyncio.sleep(5) if sleep_counter > 10: raise ex if not raw_tx_data: return False, 'Burn 10% txid is invalid' if raw_tx_data['expiryheight'] < regticket.blocknum: return False, 'Fee transaction is older then regticket.' return True, None
async def fetch_single_chunk_via_rpc(chunkid): for masternode in get_chunk_owners(chunkid): if masternode.pastel_id == get_blockchain_connection().pastelid: # don't attempt to connect ourselves continue mn = masternode.get_rpc_client() try: data = await mn.send_rpc_fetchchunk(chunkid) except RPCException as exc: tasks_logger.exception( "FETCHCHUNK RPC FAILED for node %s with exception %s" % (mn.server_ip, exc)) continue except (ClientConnectorError, ServerTimeoutError) as clien_ex: tasks_logger.error('{} for {}'.format(str(clien_ex), mn.server_ip)) continue except Exception as ex: tasks_logger.exception( "FETCHCHUNK RPC FAILED for node %s with exception %s" % (mn.server_ip, ex)) continue if data is None: tasks_logger.debug("MN %s returned None for fetchchunk %s" % (mn.server_ip, chunkid)) # chunk was not found continue # if chunk is received: # verify that digest matches digest = get_pynode_digest_int(data) if chunkid != str(digest): tasks_logger.info( "MN %s returned bad chunk for fetchchunk %s, mismatched digest: %s" % (mn.server_ip, chunkid, digest)) continue return data # nobody has this chunk tasks_logger.error("Unable to fetch chunk %s" % chunkid_to_hex(int(chunkid)))
def refresh_masternode_list(): masternode_list = get_blockchain_connection().masternode_list() fresh_mn_list = {} for k in masternode_list: node = masternode_list[k] # generate dict of {pastelid: <ip:port>} if len(node['extKey']) > 20 and len(node['extAddress']) > 4: fresh_mn_list[node['extKey']] = node['extAddress'] existing_mn_pastelids = set( [mn.pastel_id for mn in Masternode.get_active_nodes()]) fresh_mn_pastelids = set(fresh_mn_list.keys()) added_pastelids = fresh_mn_pastelids - existing_mn_pastelids if len(added_pastelids): data_for_insert = [{ 'pastel_id': pastelid, 'ext_address': fresh_mn_list[pastelid] } for pastelid in added_pastelids] Masternode.insert(data_for_insert).execute()
def get_masternode_ordering(blocknum=None): """ Fetch `masternode workers` from cNode, create RPCClient for each one, return list of RPCClients. """ mn_rpc_clients = [] workers = get_blockchain_connection().masternode_top(blocknum) index = 0 while len(mn_rpc_clients) < 3: node = workers[index] index += 1 if index >= len(workers): raise ValueError( 'There are less then 3 valid masternodes in `masternode top` output. ' 'Cannot select a quorum of 3 MNs.') remote_pastelid = node['extKey'] ip, py_rpc_port = node['extAddress'].split(':') if not node['extKey'] or not ip or not py_rpc_port: continue rpc_client = RPCClient(remote_pastelid, ip, py_rpc_port) mn_rpc_clients.append(rpc_client) return mn_rpc_clients
def validate(self): from core_modules.blackbox_modules.dupe_detection_utils import measure_similarity, \ assemble_fingerprints_for_pandas # we have no way to check these but will do so on activation: # o fingerprints # o lubyhashes # o thumbnailhash # # after these checks are done we know that fingerprints are not dupes and there is no race # validate that lubyhashes and lubychunks are the same length require_true(len(self.lubyhashes) == len(self.lubyseeds)) # validate that order txid is not too old block_distance = get_block_distance( get_blockchain_connection().getbestblockhash(), self.order_block_txid) if block_distance > Settings.MAX_REGISTRATION_BLOCK_DISTANCE: raise ValueError( "Block distance between order_block_height and current block is too large!" ) # validate that art hash doesn't exist: # TODO: move this artwork index logic into chainwrapper fingerprint_db = {} # validate that fingerprints are not dupes if len(fingerprint_db) > 0: # TODO: check for fingerprint dupes if Settings.DUPE_DETECTION_ENABLED: pandas_table = assemble_fingerprints_for_pandas([ (k, v) for k, v in fingerprint_db.items() ]) is_duplicate, params_df = measure_similarity( self.fingerprints, pandas_table) if is_duplicate: raise ValueError("Image failed fingerprint check!")
def update_masternode_list(): """ Fetch current masternode list from cNode (by calling `masternode list extra`) and update database Masternode table accordingly. Return 2 sets of MN pastelIDs - added and removed, """ masternode_list = get_blockchain_connection().masternode_list() # parse new list fresh_mn_list = {} for k in masternode_list: node = masternode_list[k] # generate dict of {pastelid: <ip:port>} if len(node['extKey']) > 20 and len(node['extAddress']) > 4: fresh_mn_list[node['extKey']] = node['extAddress'] existing_mn_pastelids = set( [mn.pastel_id for mn in Masternode.get_active_nodes()]) fresh_mn_pastelids = set(fresh_mn_list.keys()) added_pastelids = fresh_mn_pastelids - existing_mn_pastelids removed_pastelids = existing_mn_pastelids - fresh_mn_pastelids # FIXME: uncomment this if cNode will not return empty keys. # cNode returns empty extKey for random masternode, but it does not mean that this MNs should be deleted.. # maybe need to delete MN only if it has not responses several times for fetch chunk request # if len(removed_pastelids): # Masternode.delete().where(Masternode.pastel_id.in_(removed_pastelids)).execute() if len(added_pastelids): tasks_logger.warn('Got new Masternodes. Adding to the list') data_for_insert = [{ 'pastel_id': pastelid, 'ext_address': fresh_mn_list[pastelid] } for pastelid in added_pastelids] Masternode.insert(data_for_insert).execute() return added_pastelids, removed_pastelids
async def get_artworks_data(self): reg_tickets_txids = get_blockchain_connection().list_tickets( 'act') # list txid_list = set( map(lambda x: x['ticket']['reg_txid'], reg_tickets_txids)) db_txid_list = set([a.reg_ticket_txid for a in Artwork.select()]) # get act ticket txids which are in blockchain and not in db_txid_list reg_ticket_txid_to_fetch = txid_list - db_txid_list if len(reg_ticket_txid_to_fetch): client = Masternode.select()[0].get_rpc_client() # fetch missing data from the blockchain and write to DB for txid in reg_ticket_txid_to_fetch: try: ticket = get_blockchain_connection().get_ticket( txid) # it's registration ticket here except JSONRPCException as e: self.__logger.exception( 'Error obtain registration ticket txid: {}'.format( txid)) # to avoid processing invalid txid multiple times - write in to the DB with height=-1 Artwork.create(reg_ticket_txid=txid, blocknum=-1) continue try: act_ticket = get_blockchain_connection().find_ticket( 'act', txid) except JSONRPCException as e: self.__logger.exception( 'Error obtain act ticket by key: {}'.format(txid)) # to avoid processing invalid txid multiple times - write in to the DB with height=-1 Artwork.create(reg_ticket_txid=txid, blocknum=-1) continue regticket = RegistrationTicket( serialized_base64=ticket['ticket']['art_ticket']) artist_pastelid = list( ticket['ticket']['signatures']['artist'].keys())[0] # get thumbnail response = await client.rpc_download_thumbnail( regticket.thumbnailhash) thumbnail_data = b'' if response['status'] == "SUCCESS": thumbnail_data = response['image_data'] elif response['status'] == "ERROR": if 'masternodes' in response: # try to fetch thumbnail from recommended masternodes for pastelid in response['masternodes']: try: rpc_client = Masternode.select().get( Masternode.pastel_id == pastelid).get_rpc_client() response = await rpc_client.rpc_download_thumbnail( regticket.thumbnailhash) if response['status'] == "SUCCESS": thumbnail_data = response['image_data'] break elif response['status'] == "ERROR": continue except Exception: continue thumbnail_filename = '{}.png'.format(txid) thumbnail_path = os.path.join(get_thumbnail_dir(), thumbnail_filename) with open(thumbnail_path, 'wb') as f: f.write(thumbnail_data) # store artwork data to DB Artwork.create( reg_ticket_txid=txid, act_ticket_txid=act_ticket['txid'], artist_pastelid=artist_pastelid, artwork_title=regticket.artwork_title, total_copies=regticket.total_copies, artist_name=regticket.artist_name, artist_website=regticket.artist_website, artist_written_statement=regticket. artist_written_statement, artwork_series_name=regticket.artwork_series_name, artwork_creation_video_youtube_url=regticket. artwork_creation_video_youtube_url, artwork_keyword_set=regticket.artwork_keyword_set, imagedata_hash=regticket.imagedata_hash, blocknum=regticket.blocknum, order_block_txid=regticket.order_block_txid) result = [] for artwork in Artwork.select().where(Artwork.blocknum > 0): sale_data = {'forSale': False, 'price': -1, 'sell_txid': None} response = get_blockchain_connection().find_ticket( 'sell', artwork.act_ticket_txid) if response == 'Key is not found': sell_ticket = SellticketDB.get_or_none( SellticketDB.act_ticket_txid == artwork.act_ticket_txid) if sell_ticket: sale_data = { 'forSale': True, 'price': sell_ticket.price, 'sell_txid': None } elif type(response) == list: resp_json = response[0] sale_data = { 'forSale': True, 'price': resp_json['ticket']['asked_price'], 'sell_txid': resp_json['txid'] } elif type(response) == str: resp_json = json.loads(response) sale_data = { 'forSale': True, 'price': resp_json['ticket']['asked_price'], 'sell_txid': response['txid'] } result.append({ 'artistPastelId': artwork.artist_pastelid, 'name': artwork.artwork_title, 'numOfCopies': artwork.total_copies, 'copyPrice': -1, 'thumbnailPath': artwork.get_thumbnail_path(), 'artistName': artwork.artist_name, 'artistWebsite': artwork.artist_website, 'artistWrittenStatement': artwork.artist_written_statement, 'artworkSeriesName': artwork.artwork_series_name, 'artworkCreationVideoYoutubeUrl': artwork.artwork_creation_video_youtube_url, 'artworkKeywordSet': artwork.artwork_keyword_set, 'imageHash': artwork.get_image_hash_digest(), 'blocknum': artwork.blocknum, 'orderBlockTxid': artwork.order_block_txid, 'actTicketTxid': artwork.act_ticket_txid, 'saleData': sale_data }) return result
async def register_buy_ticket(self, sell_txid, price): return get_blockchain_connection().register_buy_ticket( sell_txid, price)
def validate(self, ticket): # as we sign cNode package - we should verify cNode package if not get_blockchain_connection().pastelid_verify( ticket.serialize_base64(), self.signature, self.pastelid): raise ValueError("Invalid signature")
import os from core_modules.settings import Settings os.environ.setdefault('PASTEL_ID', 'fakepastelid') os.environ.setdefault('PASSPHRASE', 'fakepassphrase') from cnode_connection import get_blockchain_connection get_blockchain_connection().pastelid_newkey(Settings.PASTEL_ID_PASSPHRASE)
from cnode_connection import get_blockchain_connection from core_modules.settings import Settings pastelid_list = get_blockchain_connection().pastelid_list() if not len(pastelid_list): raise Exception( 'There is no pastel IDs on this node. Please register one first.') else: pastelid = pastelid_list[0]['PastelID'] response = get_blockchain_connection().mnid_register( pastelid, Settings.PASTEL_ID_PASSPHRASE)
def get_and_proccess_new_activation_tickets(): """ As as input we have list of new activation ticket. Outputs of this task are: - Store appropriate registration tickets in local DB, marking them confirmed - Store chunks from these registration tickets in local DB, marking them confirmed. (which will be used by another chunk-processing tasks). """ # FIXME: use `height` param when it will be implemented on cNode act_tickets = get_blockchain_connection().list_tickets( 'act') # list if dicts with actticket data act_tickets_txids = [ticket['txid'] for ticket in act_tickets] if act_tickets_txids is None: return for txid in filter(lambda x: len(x) == TXID_LENGTH, act_tickets_txids): if ActivationTicket.select().where( ActivationTicket.txid == txid).count() != 0: continue tasks_logger.info('New activation ticket found: {}'.format(txid)) try: act_ticket = get_blockchain_connection().get_ticket(txid) except JSONRPCException as e: tasks_logger.exception( 'Exception while fetching actticket: {}'.format(str(e))) # to avoid processing invalid txid multiple times - write in to the DB with height=-1 ActivationTicket.create(txid=txid, height=-1) continue # fetch regticket from activation ticket # store regticket in local DB if not exist # get list of chunk ids, add to local DB (Chunk table) regticket_data = get_blockchain_connection().get_ticket( act_ticket['ticket']['reg_txid']) regticket = get_registration_ticket_object_from_data(regticket_data) chunk_hashes = regticket.lubyhashes # this is list of bytes objects # add thumbnail chunk try: tasks_logger.info( 'Creating Chunk record for thumbnail, hash {}'.format( regticket.thumbnailhash)) chunk = Chunk.create_from_hash( chunkhash=regticket.thumbnailhash, artwork_hash=regticket.thumbnailhash) except IntegrityError: # if Chunk with such chunkhash already exist tasks_logger.error('Error: thumbnail chunk already exists') chunk = Chunk.get_by_hash(chunkhash=regticket.thumbnailhash) chunk.confirmed = True chunk.save() chunks_created, chunks_updated = 0, 0 tasks_logger.info('Creating chunks record for artwork chunks...') for chunkhash in chunk_hashes: # if chunk exists - mark it as confirmed # else - create it. And mark as confirmed. More frequently we'll have no such chunk. try: chunk = Chunk.create_from_hash( chunkhash=chunkhash, artwork_hash=regticket.imagedata_hash) chunks_created += 1 except IntegrityError: # if Chunk with such chunkhash already exist chunk = Chunk.get_by_hash(chunkhash=chunkhash) chunks_updated += 1 chunk.confirmed = True chunk.save() tasks_logger.info('...Complete! Created {}, updated {} chunks'.format( chunks_created, chunks_updated)) # write processed act ticket to DB tasks_logger.info( 'Activation ticket processed, writing to the DB. Height: {}'. format(regticket_data['height'])) ActivationTicket.create(txid=txid, height=act_ticket['height'])