def encode_topic(topic): name, types = _process_topic(topic) for arg in types: try: process_type(arg) except AssertionError: raise ValueError("Invalid argument type") return hex(event_id(name, types)), "{}({})".format(name, ",".join(types))
def decode_single(typ, data): base, sub, _ = abi.process_type(typ) if sub != '256': raise NotImplementedError('havent gotten to this, {0}'.format((base, sub, _))) if base == 'address': raise NotImplementedError('havent gotten to this') # return encode_hex(data[12:]) elif base == 'string' or base == 'bytes' or base == 'hash': raise NotImplementedError('havent gotten to this') # return data[:int(sub)] if len(sub) else data elif base == 'uint': return int(data, 16) elif base == 'int': o = int(data, 16) return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o elif base == 'ureal': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return big_endian_to_int(data) * 1.0 / 2 ** low elif base == 'real': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high elif base == 'bool': raise NotImplementedError('havent gotten to this') # return bool(int(data.encode('hex'), 16)) else: raise ValueError("Unknown base: `{0}`".format(base))
def test_execution_of_call_with_single_bytes32(deploy_client, deployed_contracts, deploy_coinbase, deploy_future_block_call): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call(client_contract.setBytes32) value = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f' signature = call.registerData.encoded_abi_signature data = abi.encode_single(abi.process_type('bytes32'), value) txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data))) data_txn_hash = deploy_client.send_transaction( to=call._meta.address, data=txn_data, ) data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash) assert client_contract.v_bytes32() is None call_txn_hash = call.execute() deploy_client.wait_for_transaction(call_txn_hash) assert client_contract.v_bytes32() == value
def validate_argument(arg_meta, value): base, sub, arrlist = process_type(arg_meta['type']) if base == 'int': if not isinstance(value, (int, long)): return False exp = int(sub) lower_bound = (-1 * 2 ** (exp - 1)) + 1 upper_bound = (2 ** (exp - 1)) - 1 return lower_bound <= value <= upper_bound elif base == 'uint': if not isinstance(value, (int, long)): return False exp = int(sub) lower_bound = 0 upper_bound = (2 ** exp) - 1 return lower_bound <= value <= upper_bound elif base == 'address': if not isinstance(value, basestring): return False _value = value[2:] if value.startswith('0x') else value if set(_value).difference('1234567890abcdef'): return False return len(_value) == 40 elif base == 'bytes': if not isinstance(value, basestring): return False max_length = int(sub) return len(value) <= max_length else: raise ValueError("Unsupported base: '{0}'".format(base))
def decode_single(typ, data): base, sub, _ = abi.process_type(typ) if base == "address": return "0x" + data[-40:] elif base == "string" or base == "bytes" or base == "hash": if sub: bytes = ethereum_utils.int_to_32bytearray(int(data, 16)) while bytes and bytes[-1] == 0: bytes.pop() if bytes: return "".join(chr(b) for b in bytes) else: num_bytes = int(data[64 + 2 : 128 + 2], 16) bytes_as_hex = data[2 + 128 : 2 + 128 + (2 * num_bytes)] return ethereum_utils.decode_hex(bytes_as_hex) elif base == "uint": return int(data, 16) elif base == "int": o = int(data, 16) return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o elif base == "ureal": raise NotImplementedError("havent gotten to this") high, low = [int(x) for x in sub.split("x")] # return big_endian_to_int(data) * 1.0 / 2 ** low elif base == "real": raise NotImplementedError("havent gotten to this") high, low = [int(x) for x in sub.split("x")] # return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high elif base == "bool": return bool(int(data, 16)) else: raise ValueError("Unknown base: `{0}`".format(base))
def decode_single(typ, data): base, sub, _ = abi.process_type(typ) if base == 'address': return '0x' + data[-40:] elif base == 'string' or base == 'bytes' or base == 'hash': bytes = ethereum_utils.int_to_32bytearray(int(data, 16)) while bytes and bytes[-1] == 0: bytes.pop() if bytes: return ''.join(chr(b) for b in bytes) elif base == 'uint': return int(data, 16) elif base == 'int': o = int(data, 16) return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o elif base == 'ureal': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return big_endian_to_int(data) * 1.0 / 2 ** low elif base == 'real': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high elif base == 'bool': return bool(int(data, 16)) else: raise ValueError("Unknown base: `{0}`".format(base))
def test_execution_of_call_with_many_values(deploy_client, deployed_contracts, deploy_coinbase, deploy_future_block_call, CallLib): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call(client_contract.setMany) values = ( 1234567890, -1234567890, 987654321, '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13', 'd3cda913deb6f67967b99d67acdfa1712c293601', 'abcdefg', ) types = ( 'uint256', 'int256', 'uint256', 'bytes20', 'address', 'bytes', ) signature = call.registerData.encoded_abi_signature data = ''.join(( abi.encode_single(abi.process_type(t), v) for t, v in zip(types, values) )) txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data))) data_txn_hash = deploy_client.send_transaction( to=call._meta.address, data=txn_data, ) data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash) assert client_contract.vm_a() == 0 assert client_contract.vm_b() == 0 assert client_contract.vm_c() == 0 assert client_contract.vm_d() == None assert client_contract.vm_e() == '0x0000000000000000000000000000000000000000' assert client_contract.vm_f() == '' call_txn_hash = call.execute() txn_r = deploy_client.wait_for_transaction(call_txn_hash) call_logs = CallLib.CallExecuted.get_transaction_logs(call_txn_hash) call_data = [CallLib.CallExecuted.get_log_data(l) for l in call_logs] assert client_contract.vm_a() == values[0] assert client_contract.vm_b() == values[1] assert client_contract.vm_c() == values[2] assert client_contract.vm_d() == values[3] assert client_contract.vm_e() == '0xd3cda913deb6f67967b99d67acdfa1712c293601' assert client_contract.vm_f() == values[5]
def encode_sym_abi(types): proctypes = [abi.process_type(typ) for typ in types] sizes = [abi.get_size(typ) for typ in proctypes] headsize = 32 * len(types) myhead, mytail = b'', b'' for i, typ in enumerate(types): sym_arg_tag = 'symbolic_argument_' + str(i) if sizes[i] is None: myhead += abi.enc(abi.INT256, headsize + len(mytail)) mytail += abi.enc(proctypes[i], sym_arg_tag) else: myhead += abi.enc(proctypes[i], sym_arg_tag) return myhead + mytail
def test_execution_of_call_with_single_uint(deploy_client, deployed_contracts, deploy_future_block_call): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call( client_contract.setUInt, call_data=abi.encode_single(abi.process_type('uint256'), 1234567890), ) deploy_client.wait_for_block(call.targetBlock()) assert client_contract.v_uint() == 0 call_txn_hash = call.execute() call_txn_receipt = deploy_client.wait_for_transaction(call_txn_hash) assert client_contract.v_uint() == 1234567890
def test_execution_of_call_with_single_address(deploy_client, deployed_contracts, deploy_coinbase, deploy_future_block_call): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call( client_contract.setAddress, call_data=abi.encode_single(abi.process_type('address'), deploy_coinbase[2:]), ) deploy_client.wait_for_block(call.targetBlock()) assert client_contract.v_address() == '0x0000000000000000000000000000000000000000' call_txn_hash = call.execute() call_txn_receipt = deploy_client.wait_for_transaction(call_txn_hash) assert client_contract.v_address() == deploy_coinbase
def validate_argument(_type, value): base, sub, arr_list = abi.process_type(_type) if arr_list: arr_value, remainder = arr_list[-1], arr_list[:-1] if arr_value and len(value) != arr_value[0]: return False subtype = ''.join((base, sub, ''.join((str(v) for v in remainder)))) return all(validate_argument(subtype, v) for v in value) elif base == 'int': if not isinstance(value, (int, long)): return False exp = int(sub) lower_bound = -1 * 2 ** exp / 2 upper_bound = (2 ** exp) / 2 - 1 return lower_bound <= value <= upper_bound elif base == 'uint': if not isinstance(value, (int, long)): return False exp = int(sub) lower_bound = 0 upper_bound = (2 ** exp) - 1 return lower_bound <= value <= upper_bound elif base == 'address': if not isinstance(value, basestring): return False _value = value[2:] if value.startswith('0x') else value if set(_value).difference('1234567890abcdef'): return False return len(_value) == 40 elif base == 'bytes': if not isinstance(value, basestring): return False try: max_length = int(sub) except ValueError: if sub == '': return True raise return len(value) <= max_length elif base == 'string': return isinstance(value, basestring) else: raise ValueError("Unsupported base: '{0}'".format(base))
def test_execution_of_call_with_single_bytes32(deploy_client, deployed_contracts, deploy_coinbase, deploy_future_block_call): client_contract = deployed_contracts.TestCallExecution value = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f' call = deploy_future_block_call( client_contract.setBytes32, call_data=abi.encode_single(abi.process_type('bytes32'), value), ) deploy_client.wait_for_block(call.targetBlock()) assert client_contract.v_bytes32() is None call_txn_hash = call.execute() call_txn_receipt = deploy_client.wait_for_transaction(call_txn_hash) assert client_contract.v_bytes32() == value
def test_execution_of_call_with_single_uint(deploy_client, deployed_contracts, deploy_future_block_call): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call(client_contract.setUInt) signature = call.registerData.encoded_abi_signature data = abi.encode_single(abi.process_type('uint256'), 1234567890) txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data))) data_txn_hash = deploy_client.send_transaction( to=call._meta.address, data=txn_data, ) data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash) assert client_contract.v_uint() == 0 call_txn_hash = call.execute() deploy_client.wait_for_transaction(call_txn_hash) assert client_contract.v_uint() == 1234567890
def test_execution_of_call_with_single_bytes(deploy_client, deployed_contracts, deploy_coinbase, deploy_future_block_call, CallLib): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call(client_contract.setBytes) value = 'abcd' signature = call.registerData.encoded_abi_signature data = abi.encode_single(abi.process_type('bytes'), value) txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data))) data_txn_hash = deploy_client.send_transaction( to=call._meta.address, data=txn_data, ) data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash) assert client_contract.v_bytes() == '' assert call.callData() == data #call_txn_hash = call.execute() call_txn_hash = client_contract.setBytes(value) txn_r = deploy_client.wait_for_transaction(call_txn_hash) txn = deploy_client.get_transaction_by_hash(call_txn_hash) assert txn['input'] == txn_data call_logs = CallLib.CallExecuted.get_transaction_logs(call_txn_hash) call_data = [CallLib.CallExecuted.get_log_data(l) for l in call_logs] bytes_logs = client_contract.Bytes.get_transaction_logs(call_txn_hash) bytes_data = [client_contract.Bytes.get_log_data(l) for l in bytes_logs] assert client_contract.v_bytes() == value
def decode_single(typ, data): base, sub, _ = abi.process_type(typ) # ensure that we aren't trying to decode an empty response. assert len(data) > 2 if base == 'address': return '0x' + strip_0x_prefix(data[-40:]) elif base == 'string' or base == 'bytes' or base == 'hash': if sub: bytes = ethereum_utils.int_to_32bytearray(int(data, 16)) while bytes and bytes[-1] == 0: bytes.pop() if bytes: return ''.join(chr(b) for b in bytes) else: num_bytes = int(data[64 + 2:128 + 2], 16) bytes_as_hex = data[2 + 128:2 + 128 + (2 * num_bytes)] return ethereum_utils.decode_hex(bytes_as_hex) elif base == 'uint': return int(data, 16) elif base == 'int': o = int(data, 16) return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o elif base == 'ureal': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return big_endian_to_int(data) * 1.0 / 2 ** low elif base == 'real': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high elif base == 'bool': return bool(int(data, 16)) else: raise ValueError("Unknown base: `{0}`".format(base))
def decode_single(typ, data): base, sub, _ = abi.process_type(typ) # ensure that we aren't trying to decode an empty response. assert len(data) > 2 if base == 'address': return '0x' + strip_0x_prefix(data[-40:]) elif base == 'string' or base == 'bytes' or base == 'hash': if sub: bytes = ethereum_utils.int_to_32bytearray(int(data, 16)) while bytes and bytes[-1] == 0: bytes.pop() if bytes: return ''.join(chr(b) for b in bytes) else: num_bytes = int(data[64 + 2:128 + 2], 16) bytes_as_hex = data[2 + 128:2 + 128 + (2 * num_bytes)] return ethereum_utils.decode_hex(bytes_as_hex) elif base == 'uint': return int(data, 16) elif base == 'int': o = int(data, 16) return (o - 2**int(sub)) if o >= 2**(int(sub) - 1) else o elif base == 'ureal': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return big_endian_to_int(data) * 1.0 / 2 ** low elif base == 'real': raise NotImplementedError('havent gotten to this') high, low = [int(x) for x in sub.split('x')] # return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high elif base == 'bool': return bool(int(data, 16)) else: raise ValueError("Unknown base: `{0}`".format(base))
def test_execution_of_call_with_single_address(deploy_client, deployed_contracts, deploy_coinbase, deploy_future_block_call): client_contract = deployed_contracts.TestCallExecution call = deploy_future_block_call(client_contract.setAddress) signature = call.registerData.encoded_abi_signature data = abi.encode_single(abi.process_type('address'), deploy_coinbase[2:]) txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data))) data_txn_hash = deploy_client.send_transaction( to=call._meta.address, data=txn_data, ) data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash) assert client_contract.v_address() == '0x0000000000000000000000000000000000000000' call_txn_hash = call.execute() deploy_client.wait_for_transaction(call_txn_hash) assert client_contract.v_address() == deploy_coinbase
async def process_block_for_contract(self, collectible_address): if collectible_address in self._processing: log.warning("Already processing {}".format(collectible_address)) return self._processing[collectible_address] = True async with self.pool.acquire() as con: latest_block_number = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") collectible = await con.fetchrow( "SELECT * FROM collectibles WHERE contract_address = $1", collectible_address) if collectible is None: log.error( "Unable to find collectible with contract_address {}". format(collectible_address)) del self._processing[collectible_address] return if collectible['type'] == 1: events = await con.fetch( "SELECT * FROM collectible_transfer_events " "WHERE collectible_address = $1", collectible_address) elif collectible['type'] == 721: # use default erc721 event # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md events = [{ 'collectible_address': collectible_address, 'contract_address': collectible_address, 'name': 'Transfer', 'topic_hash': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', 'arguments': ['address', 'address', 'uint256'], 'indexed_arguments': [True, True, False], 'to_address_offset': 1, 'token_id_offset': 2 }] from_block_number = collectible['last_block'] + 1 if latest_block_number < from_block_number: del self._processing[collectible_address] log.info( "Aborting {} because latest block number < collectible's next block" .format(collectible_address)) return to_block_number = min(from_block_number + 1000, latest_block_number) updates = {} for event in events: contract_address = event['contract_address'] while True: try: logs = await self.eth.eth_getLogs( fromBlock=from_block_number, toBlock=to_block_number, topics=[[event['topic_hash']]], address=contract_address) break except: log.exception("error getting logs for block") continue if len(logs): for _log in logs: indexed_data = _log['topics'][1:] data_types = [ t for t, i in zip(event['arguments'], event['indexed_arguments']) if i is False ] try: data = decode_abi(data_types, data_decoder(_log['data'])) except: log.exception("Error decoding log data: {} {}".format( data_types, _log['data'])) del self._processing[collectible_address] return arguments = [] try: for t, i in zip(event['arguments'], event['indexed_arguments']): if i is True: arguments.append( decode_single( process_type(t), data_decoder(indexed_data.pop(0)))) else: arguments.append(data.pop(0)) except: log.exception("Error compiling event data") log.info("EVENT: {}".format(event)) log.info("LOG: {}".format(_log)) del self._processing[collectible_address] return to_address = arguments[event['to_address_offset']] token_id = parse_int(arguments[event['token_id_offset']]) log.debug("{} #{} -> {} -> {}".format( collectible['name'], token_id, event['name'], to_address)) updates[hex(token_id)] = (collectible_address, hex(token_id), to_address) if len(updates) > 0: new_tokens = [] for token_id in list(updates.keys()): async with self.pool.acquire() as con: token = await con.fetchrow( "SELECT * FROM collectible_tokens WHERE contract_address = $1 AND token_id = $2", collectible_address, token_id) if token is None: # get token details token_uri = None token_uri_data = await self.eth.eth_call( to_address=collectible_address, data="{}{:064x}".format(TOKEN_URI_CALL_DATA, int(token_id, 16))) if token_uri_data and token_uri_data != "0x": try: token_uri = decode_abi( ['string'], data_decoder(token_uri_data))[0].decode( 'utf-8', errors='replace') except: log.exception("Error decoding tokenURI data") token_image = None token_name = None token_description = None # if token_uri points to a valid url check if it points to json (for the erc721 metadata) parsed_uri = urlparse(token_uri) if token_uri and parsed_uri.netloc and parsed_uri.scheme in [ 'http', 'https' ]: try: resp = await AsyncHTTPClient( max_clients=100).fetch(parsed_uri.geturl()) metadata = json_decode(resp.body) if "properties" in metadata: metadata = metadata['properties'] if 'name' in metadata: if type( metadata['name'] ) == dict and 'description' in metadata['name']: token_name = metadata['name'][ 'description'] elif type(metadata['name']) == str: token_name = metadata['name'] if 'description' in metadata: if type( metadata['description'] ) == dict and 'description' in metadata[ 'description']: token_description = metadata[ 'description']['description'] elif type(metadata['description']) == str: token_description = metadata['description'] if 'image' in metadata: if type( metadata['image'] ) == dict and 'description' in metadata[ 'image']: token_image = metadata['image'][ 'description'] elif type(metadata['image']) == str: token_image = metadata['image'] except: log.exception( "Error getting token metadata for {}:{} from {}" .format(collectible_address, token_id, token_uri)) pass if not token_image: if collectible['image_url_format_string'] is not None: image_format_string = collectible[ 'image_url_format_string'] else: image_format_string = config['collectibles'][ 'image_format'] token_image = image_format_string.format( contract_address=collectible_address, token_id_hex=token_id, token_id_int=int(token_id, 16), token_uri=token_uri) new_token = updates.pop(token_id, ()) + ( token_uri, token_name, token_description, token_image) new_tokens.append(new_token) async with self.pool.acquire() as con: if len(new_tokens) > 0: await con.executemany( "INSERT INTO collectible_tokens (contract_address, token_id, owner_address, token_uri, name, description, image) " "VALUES ($1, $2, $3, $4, $5, $6, $7)", new_tokens) await con.executemany( "INSERT INTO collectible_tokens (contract_address, token_id, owner_address) " "VALUES ($1, $2, $3) " "ON CONFLICT (contract_address, token_id) DO UPDATE " "SET owner_address = EXCLUDED.owner_address", list(updates.values())) ready = collectible['ready'] or to_block_number == latest_block_number self.last_block = to_block_number async with self.pool.acquire() as con: await con.execute( "UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3", to_block_number, ready, collectible_address) del self._processing[collectible_address] if to_block_number < latest_block_number: asyncio.get_event_loop().create_task( self.process_block_for_contract(collectible_address))
async def process_block_for_asset_creation_contract(self, collectible_address): if collectible_address in self._processing and not self._processing[collectible_address].done(): log.debug("Already processing {}".format(collectible_address)) self._queue.add(collectible_address) return self._processing[collectible_address] = asyncio.Task.current_task() async with self.pool.acquire() as con: latest_block_number = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") collectible = await con.fetchrow("SELECT * FROM collectibles WHERE contract_address = $1", collectible_address) from_block_number = collectible['last_block'] + 1 if latest_block_number < from_block_number: del self._processing[collectible_address] return to_block_number = min(from_block_number + 1000, latest_block_number) topics = [[ASSET_CREATED_TOPIC]] log.debug("Getting logs for {} from blocks {}->{}".format(collectible_address, from_block_number, to_block_number)) req_start = time.time() while True: try: logs = await self.eth.eth_getLogs( fromBlock=from_block_number, toBlock=to_block_number, topics=topics, address=collectible['contract_address']) if time.time() - req_start > 10: log.warning("eth_getLogs(fromBlock={}, toBlock={}, topics={}, address={}) took {} seconds to complete".format( from_block_number, to_block_number, topics, collectible['contract_address'], time.time() - req_start)) break except JsonRPCError as e: if e.message != "Unknown block number": log.exception("unexpected error getting logs for fungible creation contract: {} (after {} seconds)".format(collectible_address, time.time() - req_start)) await asyncio.sleep(random.random()) continue except: log.exception("unexpected error getting logs for fungible creation contract: {} (after {} seconds)".format(collectible_address, time.time() - req_start)) await asyncio.sleep(random.random()) continue if len(logs): log.debug("Found {} logs for {} in blocks {}->{}".format(len(logs), collectible_address, from_block_number, to_block_number)) for i, _log in enumerate(logs): log_block_number = int(_log['blockNumber'], 16) if log_block_number < from_block_number or log_block_number > to_block_number: log.error("go unexpected block number in logs: {} (fromBlock={}, toBlock={}, collectible_address={})".format( log_block_number, from_block_number, to_block_number, collectible['contract_address'])) del self._processing[collectible_address] return topic = _log['topics'][0] if topic != ASSET_CREATED_TOPIC: continue asset_contract_address = decode_single( process_type('address'), data_decoder(_log['topics'][1])) try: token_uri_data = await self.eth.eth_call(to_address=asset_contract_address, data=TOKEN_URI_CALL_DATA) except: log.exception("Error getting token uri for fungible collectible asset {}".format(asset_contract_address)) continue asset_token_uri = decode_abi(['string'], data_decoder(token_uri_data)) try: asset_token_uri = asset_token_uri[0].decode('utf-8', errors='replace') except: log.exception("Invalid tokenURI for fungible collectible asset {}".format(asset_contract_address)) continue try: name_data = await self.eth.eth_call(to_address=asset_contract_address, data=NAME_CALL_DATA) except: log.exception("Error getting name for fungible collectible asset {}".format(asset_contract_address)) continue asset_name = decode_abi(['string'], data_decoder(name_data)) try: asset_name = asset_name[0].decode('utf-8', errors='replace') except: log.exception("Invalid name for fungible collectible asset {}".format(asset_contract_address)) continue try: creator_data = await self.eth.eth_call(to_address=asset_contract_address, data=CREATOR_CALL_DATA) except: log.exception("Error getting creator for fungible collectible asset {}".format(asset_contract_address)) continue asset_creator = decode_abi(['address'], data_decoder(creator_data))[0] try: total_supply_data = await self.eth.eth_call(to_address=asset_contract_address, data=TOTAL_SUPPLY_CALL_DATA) except: log.exception("Error getting total supply for fungible collectible asset {}".format(asset_contract_address)) continue total_supply = decode_abi(['uint256'], data_decoder(total_supply_data))[0] # owner is currently always the address that triggered the AssetCreate event tx = await self.eth.eth_getTransactionByHash(_log['transactionHash']) asset_owner = tx['from'] asset_image = None asset_description = None parsed_uri = urlparse(asset_token_uri) if asset_token_uri and parsed_uri.netloc and parsed_uri.scheme in ['http', 'https']: try: resp = await AsyncHTTPClient(max_clients=100).fetch(parsed_uri.geturl()) metadata = json_decode(resp.body) if "properties" in metadata: metadata = metadata['properties'] if 'name' in metadata: if type(metadata['name']) == dict and 'description' in metadata['name']: asset_name = metadata['name']['description'] elif type(metadata['name']) == str: asset_name = metadata['name'] if 'description' in metadata: if type(metadata['description']) == dict and 'description' in metadata['description']: asset_description = metadata['description']['description'] elif type(metadata['description']) == str: asset_description = metadata['description'] if 'image' in metadata: if type(metadata['image']) == dict and 'description' in metadata['image']: asset_image = metadata['image']['description'] elif type(metadata['image']) == str: asset_image = metadata['image'] except: log.exception("Error getting token metadata for {}:{} from {}".format( collectible_address, asset_contract_address, asset_token_uri)) pass if asset_image is None: if collectible['image_url_format_string'] is not None: asset_image = collectible['image_url_format_string'].format( contract_address=asset_contract_address, collectible_address=collectible_address, name=asset_name, token_uri=asset_token_uri, creator_address=asset_creator) async with self.pool.acquire() as con: await con.execute( "INSERT INTO fungible_collectibles (contract_address, collectible_address, name, description, token_uri, creator_address, last_block, image) " "VALUES ($1, $2, $3, $4, $5, $6, $7, $8) " "ON CONFLICT (contract_address) DO NOTHING", asset_contract_address, collectible_address, asset_name, asset_description, asset_token_uri, asset_creator, log_block_number, asset_image) await con.execute( "INSERT INTO fungible_collectible_balances (contract_address, owner_address, balance) " "VALUES ($1, $2, $3)", asset_contract_address, asset_owner, hex(total_supply)) asyncio.get_event_loop().create_task(self.process_block_for_asset_contract(asset_contract_address)) else: log.debug("No logs found for {} in blocks {}->{}".format(collectible_address, from_block_number, to_block_number)) ready = collectible['ready'] or to_block_number == latest_block_number async with self.pool.acquire() as con: await con.execute("UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3", to_block_number, ready, collectible_address) del self._processing[collectible_address] if to_block_number < latest_block_number or collectible_address in self._queue: self._queue.discard(collectible_address) asyncio.get_event_loop().create_task(self.process_block_for_asset_creation_contract(collectible_address))
async def process_block_for_asset_contract(self, contract_address): if contract_address in self._processing and not self._processing[contract_address].done(): log.debug("Already processing {}".format(contract_address)) self._queue.add(contract_address) return self._processing[contract_address] = asyncio.Task.current_task() async with self.pool.acquire() as con: latest_block_number = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") collectible = await con.fetchrow("SELECT * FROM fungible_collectibles WHERE contract_address = $1", contract_address) from_block_number = collectible['last_block'] + 1 if latest_block_number < from_block_number: del self._processing[contract_address] return to_block_number = min(from_block_number + 1000, latest_block_number) topics = [[ASSET_TRANSFER_TOPIC]] updates = {} req_start = time.time() while True: try: logs = await self.eth.eth_getLogs( fromBlock=from_block_number, toBlock=to_block_number, topics=topics, address=contract_address) if time.time() - req_start > 10: log.warning("eth_getLogs(fromBlock={}, toBlock={}, topics={}, address={}) took {} seconds to complete".format( from_block_number, to_block_number, topics, contract_address, time.time() - req_start)) break except JsonRPCError as e: if e.message != "Unknown block number": log.exception("unexpected error getting logs for fungible asset contract: {} (after {} seconds)".format(contract_address, time.time() - req_start)) await asyncio.sleep(random.random()) continue except: log.exception("unexpected error getting logs for fungible asset contract: {} (after {} seconds)".format(contract_address, time.time() - req_start)) # backoff randomly await asyncio.sleep(random.random()) continue if len(logs): for i, _log in enumerate(logs): log_block_number = int(_log['blockNumber'], 16) if log_block_number < from_block_number or log_block_number > to_block_number: log.error("go unexpected block number in logs: {} (fromBlock={}, toBlock={}, address={})".format( log_block_number, from_block_number, to_block_number, contract_address)) del self._processing[contract_address] return topic = _log['topics'][0] if topic == ASSET_TRANSFER_TOPIC: indexed_data = _log['topics'][1:] data_types = ['uint256'] try: data = decode_abi(data_types, data_decoder(_log['data'])) except: log.exception("Error decoding log data: {} {}".format(data_types, _log['data'])) del self._processing[contract_address] return arguments = [] try: for t, i in [('address', True), ('address', True), ('uint256', False)]: if i is True: arguments.append(decode_single(process_type(t), data_decoder(indexed_data.pop(0)))) else: arguments.append(data.pop(0)) except: log.exception("Error compiling event data") log.info("LOG: {}".format(_log)) del self._processing[contract_address] return from_address = arguments[0] to_address = arguments[1] value = parse_int(arguments[2]) async with self.pool.acquire() as con: if from_address and from_address not in updates: balance = await con.fetchval( "SELECT balance FROM fungible_collectible_balances WHERE contract_address = $1 AND owner_address = $2", contract_address, from_address) updates[from_address] = parse_int(balance) if balance is not None else 0 if to_address not in updates: balance = await con.fetchval( "SELECT balance FROM fungible_collectible_balances WHERE contract_address = $1 AND owner_address = $2", contract_address, to_address) updates[to_address] = parse_int(balance) if balance is not None else 0 updates[from_address] -= value updates[to_address] += value if len(updates) > 0: async with self.pool.acquire() as con: await con.executemany( "INSERT INTO fungible_collectible_balances (contract_address, owner_address, balance) " "VALUES ($1, $2, $3) " "ON CONFLICT (contract_address, owner_address) DO UPDATE " "SET balance = EXCLUDED.balance", [(contract_address, address, hex(value)) for address, value in updates.items()]) ready = collectible['ready'] or to_block_number == latest_block_number async with self.pool.acquire() as con: await con.execute("UPDATE fungible_collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3", to_block_number, ready, contract_address) del self._processing[contract_address] if to_block_number < latest_block_number or contract_address in self._queue: self._queue.discard(contract_address) asyncio.get_event_loop().create_task(self.process_block_for_asset_contract(contract_address))
async def process_block(self, blocknumber=None): if self._processing is True: return self._processing = True self.__call += 1 async with self.pool.acquire() as con: latest_block_number = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") if latest_block_number is None: log.warning("no blocks processed by block monitor yet") self._processing = False return collectible = await con.fetchrow("SELECT * FROM collectibles WHERE contract_address = $1", CRYPTO_PUNKS_CONTRACT_ADDRESS) if collectible is None: return from_block_number = collectible['last_block'] + 1 if latest_block_number < from_block_number: self._processing = False return to_block_number = min(from_block_number + 1000, latest_block_number) topics = [[TRANSFER_TOPIC, PUNK_BOUGHT_TOPIC, PUNK_TRANSFER_TOPIC]] while True: try: logs = await self.eth.eth_getLogs( fromBlock=from_block_number, toBlock=to_block_number, topics=topics, address=CRYPTO_PUNKS_CONTRACT_ADDRESS) break except: continue if len(logs): transactions = {} updates = [] for i, _log in enumerate(logs): tx = transactions.setdefault(_log['transactionHash'], {'function': 'unknown', 'done': False}) log_block_number = int(_log['blockNumber'], 16) assert log_block_number >= from_block_number and log_block_number <= to_block_number if tx['done'] is True: log.error("tried to reprocess transaction that was already added") continue topic = _log['topics'][0] if topic == TRANSFER_TOPIC: tx['to_address'] = decode_single(process_type('address'), data_decoder(_log['topics'][2])) elif topic == PUNK_TRANSFER_TOPIC: tx['token_id'] = decode_abi(['uint256'], data_decoder(_log['data']))[0] tx['function'] = 'transferPunk' elif topic == PUNK_BOUGHT_TOPIC: tx['token_id'] = parse_int(decode_single(process_type('address'), data_decoder(_log['topics'][1]))) to_address = decode_single(process_type('address'), data_decoder(_log['topics'][3])) if to_address == "0x0000000000000000000000000000000000000000": tx['function'] = 'acceptBidForPunk' else: tx['function'] = 'buyPunk' else: log.warning("got unknown topic: {}".format(topic)) continue if 'to_address' in tx and 'token_id' in tx: tx['done'] = True log.info("CryptoPunk #{} -> {} -> {}".format( tx['token_id'], tx['function'], tx['to_address'])) token_image = config['collectibles']['image_format'].format( contract_address=CRYPTO_PUNKS_CONTRACT_ADDRESS, token_id=tx['token_id']) updates.append((CRYPTO_PUNKS_CONTRACT_ADDRESS, hex(tx['token_id']), tx['to_address'], token_image)) async with self.pool.acquire() as con: await con.executemany( "INSERT INTO collectible_tokens (contract_address, token_id, owner_address, image) " "VALUES ($1, $2, $3, $4) " "ON CONFLICT (contract_address, token_id) DO UPDATE " "SET owner_address = EXCLUDED.owner_address", updates) ready = collectible['ready'] or to_block_number == latest_block_number async with self.pool.acquire() as con: await con.execute("UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3", to_block_number, ready, CRYPTO_PUNKS_CONTRACT_ADDRESS) self._processing = False if to_block_number < latest_block_number: asyncio.get_event_loop().create_task(self.process_block())
async def process_block_for_contract(self, collectible_address): if collectible_address in self._processing: return self._processing[collectible_address] = True async with self.pool.acquire() as con: latest_block_number = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") collectible = await con.fetchrow("SELECT * FROM collectibles WHERE contract_address = $1", collectible_address) if collectible['type'] == 1: events = await con.fetch("SELECT * FROM collectible_transfer_events " "WHERE collectible_address = $1", collectible_address) elif collectible['type'] == 721: # use default erc721 event # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md events = [{ 'collectible_address': collectible_address, 'contract_address': collectible_address, 'name': 'Transfer', 'topic_hash': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', 'arguments': ['address', 'address', 'uint256'], 'indexed_arguments': [True, True, False], 'to_address_offset': 1, 'token_id_offset': 2 }] from_block_number = collectible['last_block'] + 1 if latest_block_number < from_block_number: del self._processing[collectible_address] return to_block_number = min(from_block_number + 1000, latest_block_number) updates = {} for event in events: contract_address = event['contract_address'] while True: try: logs = await self.eth.eth_getLogs( fromBlock=from_block_number, toBlock=to_block_number, topics=[[event['topic_hash']]], address=contract_address) break except: log.exception("error getting logs for block") continue if len(logs): for _log in logs: indexed_data = _log['topics'][1:] data_types = [t for t, i in zip(event['arguments'], event['indexed_arguments']) if i is False] try: data = decode_abi(data_types, data_decoder(_log['data'])) except: log.exception("Error decoding log data: {} {}".format(data_types, _log['data'])) del self._processing[collectible_address] return arguments = [] try: for t, i in zip(event['arguments'], event['indexed_arguments']): if i is True: arguments.append(decode_single(process_type(t), data_decoder(indexed_data.pop(0)))) else: arguments.append(data.pop(0)) except: log.exception("Error compiling event data") to_address = arguments[event['to_address_offset']] token_id = parse_int(arguments[event['token_id_offset']]) log.debug("{} #{} -> {} -> {}".format(collectible['name'], token_id, event['name'], to_address)) token_image = config['collectibles']['image_format'].format( contract_address=collectible_address, token_id=token_id) updates[hex(token_id)] = (collectible_address, hex(token_id), to_address, token_image) if len(updates) > 0: async with self.pool.acquire() as con: await con.executemany( "INSERT INTO collectible_tokens (contract_address, token_id, owner_address, image) " "VALUES ($1, $2, $3, $4) " "ON CONFLICT (contract_address, token_id) DO UPDATE " "SET owner_address = EXCLUDED.owner_address", list(updates.values())) ready = collectible['ready'] or to_block_number == latest_block_number self.last_block = to_block_number async with self.pool.acquire() as con: await con.execute("UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3", to_block_number, ready, collectible_address) del self._processing[collectible_address] #log.info("Processed blocks #{} -> #{} for {} in {} seconds".format( # from_block_number, to_block_number, collectible['name'], time.time() - starttime)) if to_block_number < latest_block_number: asyncio.ensure_future(self.process_block_for_contract(contract_address))
async def process_block_for_contract(self, collectible_address): if collectible_address in self._processing and not self._processing[ collectible_address].done(): log.debug("Already processing {}".format(collectible_address)) return self._processing[collectible_address] = asyncio.Task.current_task() async with self.pool.acquire() as con: latest_block_number = await con.fetchval( "SELECT blocknumber FROM last_blocknumber") collectible = await con.fetchrow( "SELECT * FROM collectibles WHERE contract_address = $1", collectible_address) if collectible is None: log.error( "Unable to find collectible with contract_address {}". format(collectible_address)) del self._processing[collectible_address] return if collectible['type'] == 1: events = await con.fetch( "SELECT * FROM collectible_transfer_events " "WHERE collectible_address = $1", collectible_address) elif collectible['type'] == 3: # use default old (token id not indexed) erc721 event events = [{ 'collectible_address': collectible_address, 'contract_address': collectible_address, 'name': 'Transfer', 'topic_hash': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', 'arguments': ['address', 'address', 'uint256'], 'indexed_arguments': [True, True, False], 'to_address_offset': 1, 'token_id_offset': 2 }] elif collectible['type'] == 721: # use default erc721 event # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md events = [{ 'collectible_address': collectible_address, 'contract_address': collectible_address, 'name': 'Transfer', 'topic_hash': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', 'arguments': ['address', 'address', 'uint256'], 'indexed_arguments': [True, True, True], 'to_address_offset': 1, 'token_id_offset': 2 }] else: log.error("Collectible with unknown type {}".format( collectible_address)) del self._processing[collectible_address] return from_block_number = collectible['last_block'] + 1 if latest_block_number < from_block_number: del self._processing[collectible_address] log.info( "Aborting {} because latest block number < collectible's next block" .format(collectible_address)) return to_block_number = min(from_block_number + 1000, latest_block_number) updates = {} for event in events: contract_address = event['contract_address'] while True: try: logs = await self.eth.eth_getLogs( fromBlock=from_block_number, toBlock=to_block_number, topics=[[event['topic_hash']]], address=contract_address) break except Exception as e: if hasattr( e, 'message') and e.message != "Unknown block number": log.exception( "unexpected error getting logs for collectible at address: {}" .format(contract_address)) continue if len(logs): for _log in logs: indexed_data = _log['topics'][1:] data_types = [ t for t, i in zip(event['arguments'], event['indexed_arguments']) if i is False ] try: data = decode_abi(data_types, data_decoder(_log['data'])) except: log.exception("Error decoding log data: {} {}".format( data_types, _log['data'])) del self._processing[collectible_address] return arguments = [] try: for t, i in zip(event['arguments'], event['indexed_arguments']): if i is True: arguments.append( decode_single( process_type(t), data_decoder(indexed_data.pop(0)))) else: arguments.append(data.pop(0)) except: log.exception("Error compiling event data") log.info("EVENT: {}".format(event)) log.info("LOG: {}".format(_log)) del self._processing[collectible_address] return to_address = arguments[event['to_address_offset']] token_id = parse_int(arguments[event['token_id_offset']]) if collectible['ready'] is False: log.info("{} #{} -> {} -> {}".format( collectible['name'], token_id, event['name'], to_address)) updates[hex(token_id)] = (collectible_address, hex(token_id), to_address) if len(updates) > 0: new_tokens = [] for token_id in list(updates.keys()): async with self.pool.acquire() as con: token = await con.fetchrow( "SELECT * FROM collectible_tokens WHERE contract_address = $1 AND token_id = $2", collectible_address, token_id) if token is None: token_image = None token_name = None token_description = None token_uri = None token_uri_data = None if collectible_address == MLB_CONTRACT_ADDRESS: url = MLB_METADATA_URL.format(token_id) try: resp = await AsyncHTTPClient(max_clients=100 ).fetch(url) metadata = json_decode(resp.body) if 'fullName' not in metadata['result'][ 'mlbPlayerInfo']: token_name = None else: token_name = metadata['result'][ 'mlbPlayerInfo']['fullName'] token_image = metadata['result']['imagesURL'][ 'threeSixtyImages']['0'] except: log.exception( "Error getting token metadata for {}:{} from {}" .format(collectible_address, token_id, url)) pass else: # get token details while True: try: token_uri_data = await self.eth.eth_call( to_address=collectible_address, data="{}{:064x}".format( TOKEN_URI_CALL_DATA, int(token_id, 16))) break except JsonRPCError as e: if e.message == 'VM execution error.': break continue if token_uri_data and token_uri_data != "0x": try: token_uri = decode_abi( ['string'], data_decoder(token_uri_data))[0].decode( 'utf-8', errors='replace') except: log.exception("Error decoding tokenURI data") # if token_uri points to a valid url check if it points to json (for the erc721 metadata) parsed_uri = urlparse(token_uri) if token_uri and parsed_uri.netloc and parsed_uri.scheme in [ 'http', 'https' ]: try: resp = await AsyncHTTPClient( max_clients=100).fetch(parsed_uri.geturl()) metadata = json_decode(resp.body) properties = {} if "properties" in metadata and type( metadata['properties']) == dict: properties = metadata['properties'] name_prop = properties.get( 'name', metadata.get('name', None)) if name_prop: if type( name_prop ) == dict and 'description' in name_prop: token_name = name_prop['description'] elif type(name_prop) == str: token_name = name_prop description_prop = properties.get( 'description', metadata.get('description', None)) if description_prop: if type( description_prop ) == dict and 'description' in description_prop: token_description = description_prop[ 'description'] elif type(description_prop) == str: token_description = description_prop image_prop = properties.get( 'image', metadata.get('image', None)) if image_prop: if type( image_prop ) == dict and 'description' in image_prop: token_image = image_prop['description'] elif type(image_prop) == str: token_image = image_prop except: log.exception( "Error getting token metadata for {}:{} from {}" .format(collectible_address, token_id, token_uri)) pass elif token_uri is not None: log.warning( "token_uri is not a valid url: {}: {}".format( contract_address, token_uri)) if not token_image: if collectible['image_url_format_string'] is not None: image_format_string = collectible[ 'image_url_format_string'] else: image_format_string = config['collectibles'][ 'image_format'] token_image = image_format_string.format( contract_address=collectible_address, token_id_hex=token_id, token_id_int=int(token_id, 16), token_uri=token_uri) log.info("new '{}' collectible: {} {} {} {} {}".format( collectible['name'], token_id, token_uri, token_name, token_description, token_image)) new_token = updates.pop(token_id, ()) + ( token_uri, token_name, token_description, token_image) new_tokens.append(new_token) async with self.pool.acquire() as con: if len(new_tokens) > 0: await con.executemany( "INSERT INTO collectible_tokens (contract_address, token_id, owner_address, token_uri, name, description, image) " "VALUES ($1, $2, $3, $4, $5, $6, $7)", new_tokens) await con.executemany( "INSERT INTO collectible_tokens (contract_address, token_id, owner_address) " "VALUES ($1, $2, $3) " "ON CONFLICT (contract_address, token_id) DO UPDATE " "SET owner_address = EXCLUDED.owner_address", list(updates.values())) ready = collectible['ready'] or to_block_number == latest_block_number async with self.pool.acquire() as con: await con.execute( "UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3", to_block_number, ready, collectible_address) if collectible_address == MLB_CONTRACT_ADDRESS: async with self.pool.acquire() as con: tokens = await con.fetch( "SELECT * FROM collectible_tokens WHERE contract_address = $1 AND name IS NULL ORDER BY token_id LIMIT 100", collectible_address) updates = [] for token in tokens: token_id = token['token_id'] url = MLB_METADATA_URL.format(token_id) try: resp = await AsyncHTTPClient(max_clients=100).fetch(url) metadata = json_decode(resp.body) if 'mlbPlayerInfo' not in metadata['result']: continue if 'fullName' not in metadata['result']['mlbPlayerInfo']: token_name = None else: token_name = metadata['result']['mlbPlayerInfo'][ 'fullName'] token_image = metadata['result']['imagesURL'][ 'threeSixtyImages']['0'] if token_name == token['name'] and token_image == token[ 'image']: # nothing to update continue updates.append((collectible_address, token_id, token_name, token_image)) log.info("updated '{}' collectible: {} {} {} {} {}".format( collectible['name'], token_id, None, token_name, None, token_image)) except: log.exception( "Error getting token metadata for {}:{} from {}". format(collectible_address, token_id, url)) pass if updates: async with self.pool.acquire() as con: await con.executemany( "UPDATE collectible_tokens " "SET name = $3, image = $4 " "WHERE contract_address = $1 AND token_id = $2", updates) del self._processing[collectible_address] if to_block_number < latest_block_number: asyncio.get_event_loop().create_task( self.process_block_for_contract(collectible_address))