def encode_topic(topic):
name, types = _process_topic(topic)
for arg in types:
try:
process_type(arg)
except AssertionError:
raise ValueError("Invalid argument type")
return hex(event_id(name, types)), "{}({})".format(name, ",".join(types))
def decode_single(typ, data):
base, sub, _ = abi.process_type(typ)
if sub != '256':
raise NotImplementedError('havent gotten to this, {0}'.format((base, sub, _)))
if base == 'address':
raise NotImplementedError('havent gotten to this')
# return encode_hex(data[12:])
elif base == 'string' or base == 'bytes' or base == 'hash':
raise NotImplementedError('havent gotten to this')
# return data[:int(sub)] if len(sub) else data
elif base == 'uint':
return int(data, 16)
elif base == 'int':
o = int(data, 16)
return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o
elif base == 'ureal':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return big_endian_to_int(data) * 1.0 / 2 ** low
elif base == 'real':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high
elif base == 'bool':
raise NotImplementedError('havent gotten to this')
# return bool(int(data.encode('hex'), 16))
else:
raise ValueError("Unknown base: `{0}`".format(base))
def test_execution_of_call_with_single_bytes32(deploy_client,
deployed_contracts,
deploy_coinbase,
deploy_future_block_call):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(client_contract.setBytes32)
value = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'
signature = call.registerData.encoded_abi_signature
data = abi.encode_single(abi.process_type('bytes32'), value)
txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data)))
data_txn_hash = deploy_client.send_transaction(
to=call._meta.address,
data=txn_data,
)
data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash)
assert client_contract.v_bytes32() is None
call_txn_hash = call.execute()
deploy_client.wait_for_transaction(call_txn_hash)
assert client_contract.v_bytes32() == value
def validate_argument(arg_meta, value):
base, sub, arrlist = process_type(arg_meta['type'])
if base == 'int':
if not isinstance(value, (int, long)):
return False
exp = int(sub)
lower_bound = (-1 * 2 ** (exp - 1)) + 1
upper_bound = (2 ** (exp - 1)) - 1
return lower_bound <= value <= upper_bound
elif base == 'uint':
if not isinstance(value, (int, long)):
return False
exp = int(sub)
lower_bound = 0
upper_bound = (2 ** exp) - 1
return lower_bound <= value <= upper_bound
elif base == 'address':
if not isinstance(value, basestring):
return False
_value = value[2:] if value.startswith('0x') else value
if set(_value).difference('1234567890abcdef'):
return False
return len(_value) == 40
elif base == 'bytes':
if not isinstance(value, basestring):
return False
max_length = int(sub)
return len(value) <= max_length
else:
raise ValueError("Unsupported base: '{0}'".format(base))
def validate_argument(arg_meta, value):
base, sub, arrlist = process_type(arg_meta['type'])
if base == 'int':
if not isinstance(value, (int, long)):
return False
exp = int(sub)
lower_bound = (-1 * 2 ** (exp - 1)) + 1
upper_bound = (2 ** (exp - 1)) - 1
return lower_bound <= value <= upper_bound
elif base == 'uint':
if not isinstance(value, (int, long)):
return False
exp = int(sub)
lower_bound = 0
upper_bound = (2 ** exp) - 1
return lower_bound <= value <= upper_bound
elif base == 'address':
if not isinstance(value, basestring):
return False
_value = value[2:] if value.startswith('0x') else value
if set(_value).difference('1234567890abcdef'):
return False
return len(_value) == 40
elif base == 'bytes':
if not isinstance(value, basestring):
return False
max_length = int(sub)
return len(value) <= max_length
else:
raise ValueError("Unsupported base: '{0}'".format(base))
def decode_single(typ, data):
base, sub, _ = abi.process_type(typ)
if base == "address":
return "0x" + data[-40:]
elif base == "string" or base == "bytes" or base == "hash":
if sub:
bytes = ethereum_utils.int_to_32bytearray(int(data, 16))
while bytes and bytes[-1] == 0:
bytes.pop()
if bytes:
return "".join(chr(b) for b in bytes)
else:
num_bytes = int(data[64 + 2 : 128 + 2], 16)
bytes_as_hex = data[2 + 128 : 2 + 128 + (2 * num_bytes)]
return ethereum_utils.decode_hex(bytes_as_hex)
elif base == "uint":
return int(data, 16)
elif base == "int":
o = int(data, 16)
return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o
elif base == "ureal":
raise NotImplementedError("havent gotten to this")
high, low = [int(x) for x in sub.split("x")]
# return big_endian_to_int(data) * 1.0 / 2 ** low
elif base == "real":
raise NotImplementedError("havent gotten to this")
high, low = [int(x) for x in sub.split("x")]
# return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high
elif base == "bool":
return bool(int(data, 16))
else:
raise ValueError("Unknown base: `{0}`".format(base))
def decode_single(typ, data):
base, sub, _ = abi.process_type(typ)
if base == 'address':
return '0x' + data[-40:]
elif base == 'string' or base == 'bytes' or base == 'hash':
bytes = ethereum_utils.int_to_32bytearray(int(data, 16))
while bytes and bytes[-1] == 0:
bytes.pop()
if bytes:
return ''.join(chr(b) for b in bytes)
elif base == 'uint':
return int(data, 16)
elif base == 'int':
o = int(data, 16)
return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o
elif base == 'ureal':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return big_endian_to_int(data) * 1.0 / 2 ** low
elif base == 'real':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high
elif base == 'bool':
return bool(int(data, 16))
else:
raise ValueError("Unknown base: `{0}`".format(base))
def test_execution_of_call_with_many_values(deploy_client,
deployed_contracts,
deploy_coinbase,
deploy_future_block_call,
CallLib):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(client_contract.setMany)
values = (
1234567890,
-1234567890,
987654321,
'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13',
'd3cda913deb6f67967b99d67acdfa1712c293601',
'abcdefg',
)
types = (
'uint256',
'int256',
'uint256',
'bytes20',
'address',
'bytes',
)
signature = call.registerData.encoded_abi_signature
data = ''.join((
abi.encode_single(abi.process_type(t), v)
for t, v in zip(types, values)
))
txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data)))
data_txn_hash = deploy_client.send_transaction(
to=call._meta.address,
data=txn_data,
)
data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash)
assert client_contract.vm_a() == 0
assert client_contract.vm_b() == 0
assert client_contract.vm_c() == 0
assert client_contract.vm_d() == None
assert client_contract.vm_e() == '0x0000000000000000000000000000000000000000'
assert client_contract.vm_f() == ''
call_txn_hash = call.execute()
txn_r = deploy_client.wait_for_transaction(call_txn_hash)
call_logs = CallLib.CallExecuted.get_transaction_logs(call_txn_hash)
call_data = [CallLib.CallExecuted.get_log_data(l) for l in call_logs]
assert client_contract.vm_a() == values[0]
assert client_contract.vm_b() == values[1]
assert client_contract.vm_c() == values[2]
assert client_contract.vm_d() == values[3]
assert client_contract.vm_e() == '0xd3cda913deb6f67967b99d67acdfa1712c293601'
assert client_contract.vm_f() == values[5]
def encode_sym_abi(types):
proctypes = [abi.process_type(typ) for typ in types]
sizes = [abi.get_size(typ) for typ in proctypes]
headsize = 32 * len(types)
myhead, mytail = b'', b''
for i, typ in enumerate(types):
sym_arg_tag = 'symbolic_argument_' + str(i)
if sizes[i] is None:
myhead += abi.enc(abi.INT256, headsize + len(mytail))
mytail += abi.enc(proctypes[i], sym_arg_tag)
else:
myhead += abi.enc(proctypes[i], sym_arg_tag)
return myhead + mytail
def test_execution_of_call_with_single_uint(deploy_client, deployed_contracts,
deploy_future_block_call):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(
client_contract.setUInt,
call_data=abi.encode_single(abi.process_type('uint256'), 1234567890),
)
deploy_client.wait_for_block(call.targetBlock())
assert client_contract.v_uint() == 0
call_txn_hash = call.execute()
call_txn_receipt = deploy_client.wait_for_transaction(call_txn_hash)
assert client_contract.v_uint() == 1234567890
def test_execution_of_call_with_single_address(deploy_client,
deployed_contracts,
deploy_coinbase,
deploy_future_block_call):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(
client_contract.setAddress,
call_data=abi.encode_single(abi.process_type('address'), deploy_coinbase[2:]),
)
deploy_client.wait_for_block(call.targetBlock())
assert client_contract.v_address() == '0x0000000000000000000000000000000000000000'
call_txn_hash = call.execute()
call_txn_receipt = deploy_client.wait_for_transaction(call_txn_hash)
assert client_contract.v_address() == deploy_coinbase
def validate_argument(_type, value):
base, sub, arr_list = abi.process_type(_type)
if arr_list:
arr_value, remainder = arr_list[-1], arr_list[:-1]
if arr_value and len(value) != arr_value[0]:
return False
subtype = ''.join((base, sub, ''.join((str(v) for v in remainder))))
return all(validate_argument(subtype, v) for v in value)
elif base == 'int':
if not isinstance(value, (int, long)):
return False
exp = int(sub)
lower_bound = -1 * 2 ** exp / 2
upper_bound = (2 ** exp) / 2 - 1
return lower_bound <= value <= upper_bound
elif base == 'uint':
if not isinstance(value, (int, long)):
return False
exp = int(sub)
lower_bound = 0
upper_bound = (2 ** exp) - 1
return lower_bound <= value <= upper_bound
elif base == 'address':
if not isinstance(value, basestring):
return False
_value = value[2:] if value.startswith('0x') else value
if set(_value).difference('1234567890abcdef'):
return False
return len(_value) == 40
elif base == 'bytes':
if not isinstance(value, basestring):
return False
try:
max_length = int(sub)
except ValueError:
if sub == '':
return True
raise
return len(value) <= max_length
elif base == 'string':
return isinstance(value, basestring)
else:
raise ValueError("Unsupported base: '{0}'".format(base))
def test_execution_of_call_with_single_bytes32(deploy_client,
deployed_contracts,
deploy_coinbase,
deploy_future_block_call):
client_contract = deployed_contracts.TestCallExecution
value = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'
call = deploy_future_block_call(
client_contract.setBytes32,
call_data=abi.encode_single(abi.process_type('bytes32'), value),
)
deploy_client.wait_for_block(call.targetBlock())
assert client_contract.v_bytes32() is None
call_txn_hash = call.execute()
call_txn_receipt = deploy_client.wait_for_transaction(call_txn_hash)
assert client_contract.v_bytes32() == value
def test_execution_of_call_with_single_uint(deploy_client, deployed_contracts,
deploy_future_block_call):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(client_contract.setUInt)
signature = call.registerData.encoded_abi_signature
data = abi.encode_single(abi.process_type('uint256'), 1234567890)
txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data)))
data_txn_hash = deploy_client.send_transaction(
to=call._meta.address,
data=txn_data,
)
data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash)
assert client_contract.v_uint() == 0
call_txn_hash = call.execute()
deploy_client.wait_for_transaction(call_txn_hash)
assert client_contract.v_uint() == 1234567890
def test_execution_of_call_with_single_bytes(deploy_client,
deployed_contracts,
deploy_coinbase,
deploy_future_block_call,
CallLib):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(client_contract.setBytes)
value = 'abcd'
signature = call.registerData.encoded_abi_signature
data = abi.encode_single(abi.process_type('bytes'), value)
txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data)))
data_txn_hash = deploy_client.send_transaction(
to=call._meta.address,
data=txn_data,
)
data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash)
assert client_contract.v_bytes() == ''
assert call.callData() == data
#call_txn_hash = call.execute()
call_txn_hash = client_contract.setBytes(value)
txn_r = deploy_client.wait_for_transaction(call_txn_hash)
txn = deploy_client.get_transaction_by_hash(call_txn_hash)
assert txn['input'] == txn_data
call_logs = CallLib.CallExecuted.get_transaction_logs(call_txn_hash)
call_data = [CallLib.CallExecuted.get_log_data(l) for l in call_logs]
bytes_logs = client_contract.Bytes.get_transaction_logs(call_txn_hash)
bytes_data = [client_contract.Bytes.get_log_data(l) for l in bytes_logs]
assert client_contract.v_bytes() == value
def decode_single(typ, data):
base, sub, _ = abi.process_type(typ)
# ensure that we aren't trying to decode an empty response.
assert len(data) > 2
if base == 'address':
return '0x' + strip_0x_prefix(data[-40:])
elif base == 'string' or base == 'bytes' or base == 'hash':
if sub:
bytes = ethereum_utils.int_to_32bytearray(int(data, 16))
while bytes and bytes[-1] == 0:
bytes.pop()
if bytes:
return ''.join(chr(b) for b in bytes)
else:
num_bytes = int(data[64 + 2:128 + 2], 16)
bytes_as_hex = data[2 + 128:2 + 128 + (2 * num_bytes)]
return ethereum_utils.decode_hex(bytes_as_hex)
elif base == 'uint':
return int(data, 16)
elif base == 'int':
o = int(data, 16)
return (o - 2 ** int(sub)) if o >= 2 ** (int(sub) - 1) else o
elif base == 'ureal':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return big_endian_to_int(data) * 1.0 / 2 ** low
elif base == 'real':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high
elif base == 'bool':
return bool(int(data, 16))
else:
raise ValueError("Unknown base: `{0}`".format(base))
def decode_single(typ, data):
base, sub, _ = abi.process_type(typ)
# ensure that we aren't trying to decode an empty response.
assert len(data) > 2
if base == 'address':
return '0x' + strip_0x_prefix(data[-40:])
elif base == 'string' or base == 'bytes' or base == 'hash':
if sub:
bytes = ethereum_utils.int_to_32bytearray(int(data, 16))
while bytes and bytes[-1] == 0:
bytes.pop()
if bytes:
return ''.join(chr(b) for b in bytes)
else:
num_bytes = int(data[64 + 2:128 + 2], 16)
bytes_as_hex = data[2 + 128:2 + 128 + (2 * num_bytes)]
return ethereum_utils.decode_hex(bytes_as_hex)
elif base == 'uint':
return int(data, 16)
elif base == 'int':
o = int(data, 16)
return (o - 2**int(sub)) if o >= 2**(int(sub) - 1) else o
elif base == 'ureal':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return big_endian_to_int(data) * 1.0 / 2 ** low
elif base == 'real':
raise NotImplementedError('havent gotten to this')
high, low = [int(x) for x in sub.split('x')]
# return (big_endian_to_int(data) * 1.0 / 2 ** low) % 2 ** high
elif base == 'bool':
return bool(int(data, 16))
else:
raise ValueError("Unknown base: `{0}`".format(base))
def test_execution_of_call_with_single_address(deploy_client,
deployed_contracts,
deploy_coinbase,
deploy_future_block_call):
client_contract = deployed_contracts.TestCallExecution
call = deploy_future_block_call(client_contract.setAddress)
signature = call.registerData.encoded_abi_signature
data = abi.encode_single(abi.process_type('address'), deploy_coinbase[2:])
txn_data = ''.join((utils.encode_hex(signature), utils.encode_hex(data)))
data_txn_hash = deploy_client.send_transaction(
to=call._meta.address,
data=txn_data,
)
data_txn_receipt = deploy_client.wait_for_transaction(data_txn_hash)
assert client_contract.v_address() == '0x0000000000000000000000000000000000000000'
call_txn_hash = call.execute()
deploy_client.wait_for_transaction(call_txn_hash)
assert client_contract.v_address() == deploy_coinbase
async def process_block_for_contract(self, collectible_address):
if collectible_address in self._processing:
log.warning("Already processing {}".format(collectible_address))
return
self._processing[collectible_address] = True
async with self.pool.acquire() as con:
latest_block_number = await con.fetchval(
"SELECT blocknumber FROM last_blocknumber")
collectible = await con.fetchrow(
"SELECT * FROM collectibles WHERE contract_address = $1",
collectible_address)
if collectible is None:
log.error(
"Unable to find collectible with contract_address {}".
format(collectible_address))
del self._processing[collectible_address]
return
if collectible['type'] == 1:
events = await con.fetch(
"SELECT * FROM collectible_transfer_events "
"WHERE collectible_address = $1", collectible_address)
elif collectible['type'] == 721:
# use default erc721 event
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md
events = [{
'collectible_address': collectible_address,
'contract_address': collectible_address,
'name': 'Transfer',
'topic_hash':
'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
'arguments': ['address', 'address', 'uint256'],
'indexed_arguments': [True, True, False],
'to_address_offset': 1,
'token_id_offset': 2
}]
from_block_number = collectible['last_block'] + 1
if latest_block_number < from_block_number:
del self._processing[collectible_address]
log.info(
"Aborting {} because latest block number < collectible's next block"
.format(collectible_address))
return
to_block_number = min(from_block_number + 1000, latest_block_number)
updates = {}
for event in events:
contract_address = event['contract_address']
while True:
try:
logs = await self.eth.eth_getLogs(
fromBlock=from_block_number,
toBlock=to_block_number,
topics=[[event['topic_hash']]],
address=contract_address)
break
except:
log.exception("error getting logs for block")
continue
if len(logs):
for _log in logs:
indexed_data = _log['topics'][1:]
data_types = [
t for t, i in zip(event['arguments'],
event['indexed_arguments'])
if i is False
]
try:
data = decode_abi(data_types,
data_decoder(_log['data']))
except:
log.exception("Error decoding log data: {} {}".format(
data_types, _log['data']))
del self._processing[collectible_address]
return
arguments = []
try:
for t, i in zip(event['arguments'],
event['indexed_arguments']):
if i is True:
arguments.append(
decode_single(
process_type(t),
data_decoder(indexed_data.pop(0))))
else:
arguments.append(data.pop(0))
except:
log.exception("Error compiling event data")
log.info("EVENT: {}".format(event))
log.info("LOG: {}".format(_log))
del self._processing[collectible_address]
return
to_address = arguments[event['to_address_offset']]
token_id = parse_int(arguments[event['token_id_offset']])
log.debug("{} #{} -> {} -> {}".format(
collectible['name'], token_id, event['name'],
to_address))
updates[hex(token_id)] = (collectible_address,
hex(token_id), to_address)
if len(updates) > 0:
new_tokens = []
for token_id in list(updates.keys()):
async with self.pool.acquire() as con:
token = await con.fetchrow(
"SELECT * FROM collectible_tokens WHERE contract_address = $1 AND token_id = $2",
collectible_address, token_id)
if token is None:
# get token details
token_uri = None
token_uri_data = await self.eth.eth_call(
to_address=collectible_address,
data="{}{:064x}".format(TOKEN_URI_CALL_DATA,
int(token_id, 16)))
if token_uri_data and token_uri_data != "0x":
try:
token_uri = decode_abi(
['string'],
data_decoder(token_uri_data))[0].decode(
'utf-8', errors='replace')
except:
log.exception("Error decoding tokenURI data")
token_image = None
token_name = None
token_description = None
# if token_uri points to a valid url check if it points to json (for the erc721 metadata)
parsed_uri = urlparse(token_uri)
if token_uri and parsed_uri.netloc and parsed_uri.scheme in [
'http', 'https'
]:
try:
resp = await AsyncHTTPClient(
max_clients=100).fetch(parsed_uri.geturl())
metadata = json_decode(resp.body)
if "properties" in metadata:
metadata = metadata['properties']
if 'name' in metadata:
if type(
metadata['name']
) == dict and 'description' in metadata['name']:
token_name = metadata['name'][
'description']
elif type(metadata['name']) == str:
token_name = metadata['name']
if 'description' in metadata:
if type(
metadata['description']
) == dict and 'description' in metadata[
'description']:
token_description = metadata[
'description']['description']
elif type(metadata['description']) == str:
token_description = metadata['description']
if 'image' in metadata:
if type(
metadata['image']
) == dict and 'description' in metadata[
'image']:
token_image = metadata['image'][
'description']
elif type(metadata['image']) == str:
token_image = metadata['image']
except:
log.exception(
"Error getting token metadata for {}:{} from {}"
.format(collectible_address, token_id,
token_uri))
pass
if not token_image:
if collectible['image_url_format_string'] is not None:
image_format_string = collectible[
'image_url_format_string']
else:
image_format_string = config['collectibles'][
'image_format']
token_image = image_format_string.format(
contract_address=collectible_address,
token_id_hex=token_id,
token_id_int=int(token_id, 16),
token_uri=token_uri)
new_token = updates.pop(token_id, ()) + (
token_uri, token_name, token_description, token_image)
new_tokens.append(new_token)
async with self.pool.acquire() as con:
if len(new_tokens) > 0:
await con.executemany(
"INSERT INTO collectible_tokens (contract_address, token_id, owner_address, token_uri, name, description, image) "
"VALUES ($1, $2, $3, $4, $5, $6, $7)", new_tokens)
await con.executemany(
"INSERT INTO collectible_tokens (contract_address, token_id, owner_address) "
"VALUES ($1, $2, $3) "
"ON CONFLICT (contract_address, token_id) DO UPDATE "
"SET owner_address = EXCLUDED.owner_address",
list(updates.values()))
ready = collectible['ready'] or to_block_number == latest_block_number
self.last_block = to_block_number
async with self.pool.acquire() as con:
await con.execute(
"UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3",
to_block_number, ready, collectible_address)
del self._processing[collectible_address]
if to_block_number < latest_block_number:
asyncio.get_event_loop().create_task(
self.process_block_for_contract(collectible_address))
async def process_block_for_asset_creation_contract(self, collectible_address):
if collectible_address in self._processing and not self._processing[collectible_address].done():
log.debug("Already processing {}".format(collectible_address))
self._queue.add(collectible_address)
return
self._processing[collectible_address] = asyncio.Task.current_task()
async with self.pool.acquire() as con:
latest_block_number = await con.fetchval(
"SELECT blocknumber FROM last_blocknumber")
collectible = await con.fetchrow("SELECT * FROM collectibles WHERE contract_address = $1",
collectible_address)
from_block_number = collectible['last_block'] + 1
if latest_block_number < from_block_number:
del self._processing[collectible_address]
return
to_block_number = min(from_block_number + 1000, latest_block_number)
topics = [[ASSET_CREATED_TOPIC]]
log.debug("Getting logs for {} from blocks {}->{}".format(collectible_address, from_block_number, to_block_number))
req_start = time.time()
while True:
try:
logs = await self.eth.eth_getLogs(
fromBlock=from_block_number, toBlock=to_block_number,
topics=topics,
address=collectible['contract_address'])
if time.time() - req_start > 10:
log.warning("eth_getLogs(fromBlock={}, toBlock={}, topics={}, address={}) took {} seconds to complete".format(
from_block_number, to_block_number, topics, collectible['contract_address'], time.time() - req_start))
break
except JsonRPCError as e:
if e.message != "Unknown block number":
log.exception("unexpected error getting logs for fungible creation contract: {} (after {} seconds)".format(collectible_address, time.time() - req_start))
await asyncio.sleep(random.random())
continue
except:
log.exception("unexpected error getting logs for fungible creation contract: {} (after {} seconds)".format(collectible_address, time.time() - req_start))
await asyncio.sleep(random.random())
continue
if len(logs):
log.debug("Found {} logs for {} in blocks {}->{}".format(len(logs), collectible_address, from_block_number, to_block_number))
for i, _log in enumerate(logs):
log_block_number = int(_log['blockNumber'], 16)
if log_block_number < from_block_number or log_block_number > to_block_number:
log.error("go unexpected block number in logs: {} (fromBlock={}, toBlock={}, collectible_address={})".format(
log_block_number, from_block_number, to_block_number, collectible['contract_address']))
del self._processing[collectible_address]
return
topic = _log['topics'][0]
if topic != ASSET_CREATED_TOPIC:
continue
asset_contract_address = decode_single(
process_type('address'), data_decoder(_log['topics'][1]))
try:
token_uri_data = await self.eth.eth_call(to_address=asset_contract_address, data=TOKEN_URI_CALL_DATA)
except:
log.exception("Error getting token uri for fungible collectible asset {}".format(asset_contract_address))
continue
asset_token_uri = decode_abi(['string'], data_decoder(token_uri_data))
try:
asset_token_uri = asset_token_uri[0].decode('utf-8', errors='replace')
except:
log.exception("Invalid tokenURI for fungible collectible asset {}".format(asset_contract_address))
continue
try:
name_data = await self.eth.eth_call(to_address=asset_contract_address, data=NAME_CALL_DATA)
except:
log.exception("Error getting name for fungible collectible asset {}".format(asset_contract_address))
continue
asset_name = decode_abi(['string'], data_decoder(name_data))
try:
asset_name = asset_name[0].decode('utf-8', errors='replace')
except:
log.exception("Invalid name for fungible collectible asset {}".format(asset_contract_address))
continue
try:
creator_data = await self.eth.eth_call(to_address=asset_contract_address, data=CREATOR_CALL_DATA)
except:
log.exception("Error getting creator for fungible collectible asset {}".format(asset_contract_address))
continue
asset_creator = decode_abi(['address'], data_decoder(creator_data))[0]
try:
total_supply_data = await self.eth.eth_call(to_address=asset_contract_address, data=TOTAL_SUPPLY_CALL_DATA)
except:
log.exception("Error getting total supply for fungible collectible asset {}".format(asset_contract_address))
continue
total_supply = decode_abi(['uint256'], data_decoder(total_supply_data))[0]
# owner is currently always the address that triggered the AssetCreate event
tx = await self.eth.eth_getTransactionByHash(_log['transactionHash'])
asset_owner = tx['from']
asset_image = None
asset_description = None
parsed_uri = urlparse(asset_token_uri)
if asset_token_uri and parsed_uri.netloc and parsed_uri.scheme in ['http', 'https']:
try:
resp = await AsyncHTTPClient(max_clients=100).fetch(parsed_uri.geturl())
metadata = json_decode(resp.body)
if "properties" in metadata:
metadata = metadata['properties']
if 'name' in metadata:
if type(metadata['name']) == dict and 'description' in metadata['name']:
asset_name = metadata['name']['description']
elif type(metadata['name']) == str:
asset_name = metadata['name']
if 'description' in metadata:
if type(metadata['description']) == dict and 'description' in metadata['description']:
asset_description = metadata['description']['description']
elif type(metadata['description']) == str:
asset_description = metadata['description']
if 'image' in metadata:
if type(metadata['image']) == dict and 'description' in metadata['image']:
asset_image = metadata['image']['description']
elif type(metadata['image']) == str:
asset_image = metadata['image']
except:
log.exception("Error getting token metadata for {}:{} from {}".format(
collectible_address, asset_contract_address, asset_token_uri))
pass
if asset_image is None:
if collectible['image_url_format_string'] is not None:
asset_image = collectible['image_url_format_string'].format(
contract_address=asset_contract_address,
collectible_address=collectible_address,
name=asset_name,
token_uri=asset_token_uri,
creator_address=asset_creator)
async with self.pool.acquire() as con:
await con.execute(
"INSERT INTO fungible_collectibles (contract_address, collectible_address, name, description, token_uri, creator_address, last_block, image) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8) "
"ON CONFLICT (contract_address) DO NOTHING",
asset_contract_address, collectible_address, asset_name, asset_description, asset_token_uri, asset_creator, log_block_number, asset_image)
await con.execute(
"INSERT INTO fungible_collectible_balances (contract_address, owner_address, balance) "
"VALUES ($1, $2, $3)",
asset_contract_address, asset_owner, hex(total_supply))
asyncio.get_event_loop().create_task(self.process_block_for_asset_contract(asset_contract_address))
else:
log.debug("No logs found for {} in blocks {}->{}".format(collectible_address, from_block_number, to_block_number))
ready = collectible['ready'] or to_block_number == latest_block_number
async with self.pool.acquire() as con:
await con.execute("UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3",
to_block_number, ready, collectible_address)
del self._processing[collectible_address]
if to_block_number < latest_block_number or collectible_address in self._queue:
self._queue.discard(collectible_address)
asyncio.get_event_loop().create_task(self.process_block_for_asset_creation_contract(collectible_address))
async def process_block_for_asset_contract(self, contract_address):
if contract_address in self._processing and not self._processing[contract_address].done():
log.debug("Already processing {}".format(contract_address))
self._queue.add(contract_address)
return
self._processing[contract_address] = asyncio.Task.current_task()
async with self.pool.acquire() as con:
latest_block_number = await con.fetchval(
"SELECT blocknumber FROM last_blocknumber")
collectible = await con.fetchrow("SELECT * FROM fungible_collectibles WHERE contract_address = $1",
contract_address)
from_block_number = collectible['last_block'] + 1
if latest_block_number < from_block_number:
del self._processing[contract_address]
return
to_block_number = min(from_block_number + 1000, latest_block_number)
topics = [[ASSET_TRANSFER_TOPIC]]
updates = {}
req_start = time.time()
while True:
try:
logs = await self.eth.eth_getLogs(
fromBlock=from_block_number, toBlock=to_block_number,
topics=topics,
address=contract_address)
if time.time() - req_start > 10:
log.warning("eth_getLogs(fromBlock={}, toBlock={}, topics={}, address={}) took {} seconds to complete".format(
from_block_number, to_block_number, topics, contract_address, time.time() - req_start))
break
except JsonRPCError as e:
if e.message != "Unknown block number":
log.exception("unexpected error getting logs for fungible asset contract: {} (after {} seconds)".format(contract_address, time.time() - req_start))
await asyncio.sleep(random.random())
continue
except:
log.exception("unexpected error getting logs for fungible asset contract: {} (after {} seconds)".format(contract_address, time.time() - req_start))
# backoff randomly
await asyncio.sleep(random.random())
continue
if len(logs):
for i, _log in enumerate(logs):
log_block_number = int(_log['blockNumber'], 16)
if log_block_number < from_block_number or log_block_number > to_block_number:
log.error("go unexpected block number in logs: {} (fromBlock={}, toBlock={}, address={})".format(
log_block_number, from_block_number, to_block_number, contract_address))
del self._processing[contract_address]
return
topic = _log['topics'][0]
if topic == ASSET_TRANSFER_TOPIC:
indexed_data = _log['topics'][1:]
data_types = ['uint256']
try:
data = decode_abi(data_types, data_decoder(_log['data']))
except:
log.exception("Error decoding log data: {} {}".format(data_types, _log['data']))
del self._processing[contract_address]
return
arguments = []
try:
for t, i in [('address', True), ('address', True), ('uint256', False)]:
if i is True:
arguments.append(decode_single(process_type(t), data_decoder(indexed_data.pop(0))))
else:
arguments.append(data.pop(0))
except:
log.exception("Error compiling event data")
log.info("LOG: {}".format(_log))
del self._processing[contract_address]
return
from_address = arguments[0]
to_address = arguments[1]
value = parse_int(arguments[2])
async with self.pool.acquire() as con:
if from_address and from_address not in updates:
balance = await con.fetchval(
"SELECT balance FROM fungible_collectible_balances WHERE contract_address = $1 AND owner_address = $2",
contract_address, from_address)
updates[from_address] = parse_int(balance) if balance is not None else 0
if to_address not in updates:
balance = await con.fetchval(
"SELECT balance FROM fungible_collectible_balances WHERE contract_address = $1 AND owner_address = $2",
contract_address, to_address)
updates[to_address] = parse_int(balance) if balance is not None else 0
updates[from_address] -= value
updates[to_address] += value
if len(updates) > 0:
async with self.pool.acquire() as con:
await con.executemany(
"INSERT INTO fungible_collectible_balances (contract_address, owner_address, balance) "
"VALUES ($1, $2, $3) "
"ON CONFLICT (contract_address, owner_address) DO UPDATE "
"SET balance = EXCLUDED.balance",
[(contract_address, address, hex(value)) for address, value in updates.items()])
ready = collectible['ready'] or to_block_number == latest_block_number
async with self.pool.acquire() as con:
await con.execute("UPDATE fungible_collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3",
to_block_number, ready, contract_address)
del self._processing[contract_address]
if to_block_number < latest_block_number or contract_address in self._queue:
self._queue.discard(contract_address)
asyncio.get_event_loop().create_task(self.process_block_for_asset_contract(contract_address))
async def process_block(self, blocknumber=None):
if self._processing is True:
return
self._processing = True
self.__call += 1
async with self.pool.acquire() as con:
latest_block_number = await con.fetchval(
"SELECT blocknumber FROM last_blocknumber")
if latest_block_number is None:
log.warning("no blocks processed by block monitor yet")
self._processing = False
return
collectible = await con.fetchrow("SELECT * FROM collectibles WHERE contract_address = $1",
CRYPTO_PUNKS_CONTRACT_ADDRESS)
if collectible is None:
return
from_block_number = collectible['last_block'] + 1
if latest_block_number < from_block_number:
self._processing = False
return
to_block_number = min(from_block_number + 1000, latest_block_number)
topics = [[TRANSFER_TOPIC, PUNK_BOUGHT_TOPIC, PUNK_TRANSFER_TOPIC]]
while True:
try:
logs = await self.eth.eth_getLogs(
fromBlock=from_block_number, toBlock=to_block_number,
topics=topics,
address=CRYPTO_PUNKS_CONTRACT_ADDRESS)
break
except:
continue
if len(logs):
transactions = {}
updates = []
for i, _log in enumerate(logs):
tx = transactions.setdefault(_log['transactionHash'], {'function': 'unknown', 'done': False})
log_block_number = int(_log['blockNumber'], 16)
assert log_block_number >= from_block_number and log_block_number <= to_block_number
if tx['done'] is True:
log.error("tried to reprocess transaction that was already added")
continue
topic = _log['topics'][0]
if topic == TRANSFER_TOPIC:
tx['to_address'] = decode_single(process_type('address'),
data_decoder(_log['topics'][2]))
elif topic == PUNK_TRANSFER_TOPIC:
tx['token_id'] = decode_abi(['uint256'], data_decoder(_log['data']))[0]
tx['function'] = 'transferPunk'
elif topic == PUNK_BOUGHT_TOPIC:
tx['token_id'] = parse_int(decode_single(process_type('address'),
data_decoder(_log['topics'][1])))
to_address = decode_single(process_type('address'),
data_decoder(_log['topics'][3]))
if to_address == "0x0000000000000000000000000000000000000000":
tx['function'] = 'acceptBidForPunk'
else:
tx['function'] = 'buyPunk'
else:
log.warning("got unknown topic: {}".format(topic))
continue
if 'to_address' in tx and 'token_id' in tx:
tx['done'] = True
log.info("CryptoPunk #{} -> {} -> {}".format(
tx['token_id'], tx['function'], tx['to_address']))
token_image = config['collectibles']['image_format'].format(
contract_address=CRYPTO_PUNKS_CONTRACT_ADDRESS,
token_id=tx['token_id'])
updates.append((CRYPTO_PUNKS_CONTRACT_ADDRESS, hex(tx['token_id']), tx['to_address'], token_image))
async with self.pool.acquire() as con:
await con.executemany(
"INSERT INTO collectible_tokens (contract_address, token_id, owner_address, image) "
"VALUES ($1, $2, $3, $4) "
"ON CONFLICT (contract_address, token_id) DO UPDATE "
"SET owner_address = EXCLUDED.owner_address",
updates)
ready = collectible['ready'] or to_block_number == latest_block_number
async with self.pool.acquire() as con:
await con.execute("UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3",
to_block_number, ready, CRYPTO_PUNKS_CONTRACT_ADDRESS)
self._processing = False
if to_block_number < latest_block_number:
asyncio.get_event_loop().create_task(self.process_block())
async def process_block_for_contract(self, collectible_address):
if collectible_address in self._processing:
return
self._processing[collectible_address] = True
async with self.pool.acquire() as con:
latest_block_number = await con.fetchval(
"SELECT blocknumber FROM last_blocknumber")
collectible = await con.fetchrow("SELECT * FROM collectibles WHERE contract_address = $1",
collectible_address)
if collectible['type'] == 1:
events = await con.fetch("SELECT * FROM collectible_transfer_events "
"WHERE collectible_address = $1",
collectible_address)
elif collectible['type'] == 721:
# use default erc721 event
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md
events = [{
'collectible_address': collectible_address,
'contract_address': collectible_address,
'name': 'Transfer',
'topic_hash': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
'arguments': ['address', 'address', 'uint256'],
'indexed_arguments': [True, True, False],
'to_address_offset': 1,
'token_id_offset': 2
}]
from_block_number = collectible['last_block'] + 1
if latest_block_number < from_block_number:
del self._processing[collectible_address]
return
to_block_number = min(from_block_number + 1000, latest_block_number)
updates = {}
for event in events:
contract_address = event['contract_address']
while True:
try:
logs = await self.eth.eth_getLogs(
fromBlock=from_block_number, toBlock=to_block_number,
topics=[[event['topic_hash']]],
address=contract_address)
break
except:
log.exception("error getting logs for block")
continue
if len(logs):
for _log in logs:
indexed_data = _log['topics'][1:]
data_types = [t for t, i in zip(event['arguments'], event['indexed_arguments']) if i is False]
try:
data = decode_abi(data_types, data_decoder(_log['data']))
except:
log.exception("Error decoding log data: {} {}".format(data_types, _log['data']))
del self._processing[collectible_address]
return
arguments = []
try:
for t, i in zip(event['arguments'], event['indexed_arguments']):
if i is True:
arguments.append(decode_single(process_type(t), data_decoder(indexed_data.pop(0))))
else:
arguments.append(data.pop(0))
except:
log.exception("Error compiling event data")
to_address = arguments[event['to_address_offset']]
token_id = parse_int(arguments[event['token_id_offset']])
log.debug("{} #{} -> {} -> {}".format(collectible['name'], token_id,
event['name'], to_address))
token_image = config['collectibles']['image_format'].format(
contract_address=collectible_address,
token_id=token_id)
updates[hex(token_id)] = (collectible_address, hex(token_id), to_address, token_image)
if len(updates) > 0:
async with self.pool.acquire() as con:
await con.executemany(
"INSERT INTO collectible_tokens (contract_address, token_id, owner_address, image) "
"VALUES ($1, $2, $3, $4) "
"ON CONFLICT (contract_address, token_id) DO UPDATE "
"SET owner_address = EXCLUDED.owner_address",
list(updates.values()))
ready = collectible['ready'] or to_block_number == latest_block_number
self.last_block = to_block_number
async with self.pool.acquire() as con:
await con.execute("UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3",
to_block_number, ready, collectible_address)
del self._processing[collectible_address]
#log.info("Processed blocks #{} -> #{} for {} in {} seconds".format(
# from_block_number, to_block_number, collectible['name'], time.time() - starttime))
if to_block_number < latest_block_number:
asyncio.ensure_future(self.process_block_for_contract(contract_address))
async def process_block_for_contract(self, collectible_address):
if collectible_address in self._processing and not self._processing[
collectible_address].done():
log.debug("Already processing {}".format(collectible_address))
return
self._processing[collectible_address] = asyncio.Task.current_task()
async with self.pool.acquire() as con:
latest_block_number = await con.fetchval(
"SELECT blocknumber FROM last_blocknumber")
collectible = await con.fetchrow(
"SELECT * FROM collectibles WHERE contract_address = $1",
collectible_address)
if collectible is None:
log.error(
"Unable to find collectible with contract_address {}".
format(collectible_address))
del self._processing[collectible_address]
return
if collectible['type'] == 1:
events = await con.fetch(
"SELECT * FROM collectible_transfer_events "
"WHERE collectible_address = $1", collectible_address)
elif collectible['type'] == 3:
# use default old (token id not indexed) erc721 event
events = [{
'collectible_address': collectible_address,
'contract_address': collectible_address,
'name': 'Transfer',
'topic_hash':
'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
'arguments': ['address', 'address', 'uint256'],
'indexed_arguments': [True, True, False],
'to_address_offset': 1,
'token_id_offset': 2
}]
elif collectible['type'] == 721:
# use default erc721 event
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md
events = [{
'collectible_address': collectible_address,
'contract_address': collectible_address,
'name': 'Transfer',
'topic_hash':
'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
'arguments': ['address', 'address', 'uint256'],
'indexed_arguments': [True, True, True],
'to_address_offset': 1,
'token_id_offset': 2
}]
else:
log.error("Collectible with unknown type {}".format(
collectible_address))
del self._processing[collectible_address]
return
from_block_number = collectible['last_block'] + 1
if latest_block_number < from_block_number:
del self._processing[collectible_address]
log.info(
"Aborting {} because latest block number < collectible's next block"
.format(collectible_address))
return
to_block_number = min(from_block_number + 1000, latest_block_number)
updates = {}
for event in events:
contract_address = event['contract_address']
while True:
try:
logs = await self.eth.eth_getLogs(
fromBlock=from_block_number,
toBlock=to_block_number,
topics=[[event['topic_hash']]],
address=contract_address)
break
except Exception as e:
if hasattr(
e,
'message') and e.message != "Unknown block number":
log.exception(
"unexpected error getting logs for collectible at address: {}"
.format(contract_address))
continue
if len(logs):
for _log in logs:
indexed_data = _log['topics'][1:]
data_types = [
t for t, i in zip(event['arguments'],
event['indexed_arguments'])
if i is False
]
try:
data = decode_abi(data_types,
data_decoder(_log['data']))
except:
log.exception("Error decoding log data: {} {}".format(
data_types, _log['data']))
del self._processing[collectible_address]
return
arguments = []
try:
for t, i in zip(event['arguments'],
event['indexed_arguments']):
if i is True:
arguments.append(
decode_single(
process_type(t),
data_decoder(indexed_data.pop(0))))
else:
arguments.append(data.pop(0))
except:
log.exception("Error compiling event data")
log.info("EVENT: {}".format(event))
log.info("LOG: {}".format(_log))
del self._processing[collectible_address]
return
to_address = arguments[event['to_address_offset']]
token_id = parse_int(arguments[event['token_id_offset']])
if collectible['ready'] is False:
log.info("{} #{} -> {} -> {}".format(
collectible['name'], token_id, event['name'],
to_address))
updates[hex(token_id)] = (collectible_address,
hex(token_id), to_address)
if len(updates) > 0:
new_tokens = []
for token_id in list(updates.keys()):
async with self.pool.acquire() as con:
token = await con.fetchrow(
"SELECT * FROM collectible_tokens WHERE contract_address = $1 AND token_id = $2",
collectible_address, token_id)
if token is None:
token_image = None
token_name = None
token_description = None
token_uri = None
token_uri_data = None
if collectible_address == MLB_CONTRACT_ADDRESS:
url = MLB_METADATA_URL.format(token_id)
try:
resp = await AsyncHTTPClient(max_clients=100
).fetch(url)
metadata = json_decode(resp.body)
if 'fullName' not in metadata['result'][
'mlbPlayerInfo']:
token_name = None
else:
token_name = metadata['result'][
'mlbPlayerInfo']['fullName']
token_image = metadata['result']['imagesURL'][
'threeSixtyImages']['0']
except:
log.exception(
"Error getting token metadata for {}:{} from {}"
.format(collectible_address, token_id, url))
pass
else:
# get token details
while True:
try:
token_uri_data = await self.eth.eth_call(
to_address=collectible_address,
data="{}{:064x}".format(
TOKEN_URI_CALL_DATA, int(token_id,
16)))
break
except JsonRPCError as e:
if e.message == 'VM execution error.':
break
continue
if token_uri_data and token_uri_data != "0x":
try:
token_uri = decode_abi(
['string'],
data_decoder(token_uri_data))[0].decode(
'utf-8', errors='replace')
except:
log.exception("Error decoding tokenURI data")
# if token_uri points to a valid url check if it points to json (for the erc721 metadata)
parsed_uri = urlparse(token_uri)
if token_uri and parsed_uri.netloc and parsed_uri.scheme in [
'http', 'https'
]:
try:
resp = await AsyncHTTPClient(
max_clients=100).fetch(parsed_uri.geturl())
metadata = json_decode(resp.body)
properties = {}
if "properties" in metadata and type(
metadata['properties']) == dict:
properties = metadata['properties']
name_prop = properties.get(
'name', metadata.get('name', None))
if name_prop:
if type(
name_prop
) == dict and 'description' in name_prop:
token_name = name_prop['description']
elif type(name_prop) == str:
token_name = name_prop
description_prop = properties.get(
'description',
metadata.get('description', None))
if description_prop:
if type(
description_prop
) == dict and 'description' in description_prop:
token_description = description_prop[
'description']
elif type(description_prop) == str:
token_description = description_prop
image_prop = properties.get(
'image', metadata.get('image', None))
if image_prop:
if type(
image_prop
) == dict and 'description' in image_prop:
token_image = image_prop['description']
elif type(image_prop) == str:
token_image = image_prop
except:
log.exception(
"Error getting token metadata for {}:{} from {}"
.format(collectible_address, token_id,
token_uri))
pass
elif token_uri is not None:
log.warning(
"token_uri is not a valid url: {}: {}".format(
contract_address, token_uri))
if not token_image:
if collectible['image_url_format_string'] is not None:
image_format_string = collectible[
'image_url_format_string']
else:
image_format_string = config['collectibles'][
'image_format']
token_image = image_format_string.format(
contract_address=collectible_address,
token_id_hex=token_id,
token_id_int=int(token_id, 16),
token_uri=token_uri)
log.info("new '{}' collectible: {} {} {} {} {}".format(
collectible['name'], token_id, token_uri, token_name,
token_description, token_image))
new_token = updates.pop(token_id, ()) + (
token_uri, token_name, token_description, token_image)
new_tokens.append(new_token)
async with self.pool.acquire() as con:
if len(new_tokens) > 0:
await con.executemany(
"INSERT INTO collectible_tokens (contract_address, token_id, owner_address, token_uri, name, description, image) "
"VALUES ($1, $2, $3, $4, $5, $6, $7)", new_tokens)
await con.executemany(
"INSERT INTO collectible_tokens (contract_address, token_id, owner_address) "
"VALUES ($1, $2, $3) "
"ON CONFLICT (contract_address, token_id) DO UPDATE "
"SET owner_address = EXCLUDED.owner_address",
list(updates.values()))
ready = collectible['ready'] or to_block_number == latest_block_number
async with self.pool.acquire() as con:
await con.execute(
"UPDATE collectibles SET last_block = $1, ready = $2 WHERE contract_address = $3",
to_block_number, ready, collectible_address)
if collectible_address == MLB_CONTRACT_ADDRESS:
async with self.pool.acquire() as con:
tokens = await con.fetch(
"SELECT * FROM collectible_tokens WHERE contract_address = $1 AND name IS NULL ORDER BY token_id LIMIT 100",
collectible_address)
updates = []
for token in tokens:
token_id = token['token_id']
url = MLB_METADATA_URL.format(token_id)
try:
resp = await AsyncHTTPClient(max_clients=100).fetch(url)
metadata = json_decode(resp.body)
if 'mlbPlayerInfo' not in metadata['result']:
continue
if 'fullName' not in metadata['result']['mlbPlayerInfo']:
token_name = None
else:
token_name = metadata['result']['mlbPlayerInfo'][
'fullName']
token_image = metadata['result']['imagesURL'][
'threeSixtyImages']['0']
if token_name == token['name'] and token_image == token[
'image']:
# nothing to update
continue
updates.append((collectible_address, token_id, token_name,
token_image))
log.info("updated '{}' collectible: {} {} {} {} {}".format(
collectible['name'], token_id, None, token_name, None,
token_image))
except:
log.exception(
"Error getting token metadata for {}:{} from {}".
format(collectible_address, token_id, url))
pass
if updates:
async with self.pool.acquire() as con:
await con.executemany(
"UPDATE collectible_tokens "
"SET name = $3, image = $4 "
"WHERE contract_address = $1 AND token_id = $2",
updates)
del self._processing[collectible_address]
if to_block_number < latest_block_number:
asyncio.get_event_loop().create_task(
self.process_block_for_contract(collectible_address))
