async def update_token_metadatas_deferred(
self,
collectionTokenIds: List[Tuple[str, str]],
shouldForce: bool = False) -> None:
if len(collectionTokenIds) == 0:
return
if not shouldForce:
query = (TokenMetadatasTable.select().where(
TokenMetadatasTable.c.updatedDate > date_util.
datetime_from_now(days=-_TOKEN_UPDATE_MIN_DAYS)).where(
sqlalchemy.tuple_(
TokenMetadatasTable.c.registryAddress,
TokenMetadatasTable.c.tokenId).in_(collectionTokenIds))
)
recentlyUpdatedTokenMetadatas = await self.retriever.query_token_metadatas(
query=query)
recentlyUpdatedTokenIds = set(
(tokenMetadata.registryAddress, tokenMetadata.tokenId)
for tokenMetadata in recentlyUpdatedTokenMetadatas)
logging.info(
f'Skipping {len(recentlyUpdatedTokenIds)} collectionTokenIds because they have been updated recently.'
)
collectionTokenIds = set(
collectionTokenIds) - recentlyUpdatedTokenIds
messages = [
UpdateTokenMetadataMessageContent(
registryAddress=registryAddress,
tokenId=tokenId,
shouldForce=shouldForce).to_message()
for (registryAddress, tokenId) in collectionTokenIds
]
await self.tokenQueue.send_messages(messages=messages)
async def list_token_metadatas(self, fieldFilters: Optional[Sequence[FieldFilter]] = None, orders: Optional[Sequence[Order]] = None, limit: Optional[int] = None, connection: Optional[DatabaseConnection] = None) -> Sequence[TokenMetadata]:
query = TokenMetadatasTable.select()
if fieldFilters:
query = self._apply_field_filters(query=query, table=TokenMetadatasTable, fieldFilters=fieldFilters)
if orders:
query = self._apply_orders(query=query, table=TokenMetadatasTable, orders=orders)
if limit:
query = query.limit(limit)
return await self.query_token_metadatas(query=query, connection=connection)
async def get_token_metadata_by_registry_address_token_id(self, registryAddress: str, tokenId: str, connection: Optional[DatabaseConnection] = None) -> TokenMetadata:
query = TokenMetadatasTable.select() \
.where(TokenMetadatasTable.c.registryAddress == registryAddress) \
.where(TokenMetadatasTable.c.tokenId == tokenId)
result = await self.database.execute(query=query, connection=connection)
row = result.first()
if not row:
raise NotFoundException(message=f'TokenMetadata with registry:{registryAddress} tokenId:{tokenId} not found')
tokenMetdata = token_metadata_from_row(row)
return tokenMetdata
async def reprocess_metadata(startId: int, endId: int, batchSize: int):
databaseConnectionString = Database.create_psql_connection_string(
username=os.environ["DB_USERNAME"],
password=os.environ["DB_PASSWORD"],
host=os.environ["DB_HOST"],
port=os.environ["DB_PORT"],
name=os.environ["DB_NAME"])
database = Database(connectionString=databaseConnectionString)
saver = Saver(database)
tokenMetadataProcessor = TokenMetadataProcessor(requester=None,
ethClient=None,
s3manager=None,
bucketName=None)
await database.connect()
currentId = startId
while currentId < endId:
start = currentId
end = min(currentId + batchSize, endId)
logging.info(f'Working on {start} to {end}...')
async with database.transaction():
query = TokenMetadatasTable.select()
query = query.where(TokenMetadatasTable.c.tokenMetadataId >= start)
query = query.where(TokenMetadatasTable.c.tokenMetadataId < end)
query = query.where(
TokenMetadatasTable.c.metadataUrl.startswith('data:'))
query = query.where(TokenMetadatasTable.c.name == None)
tokenMetadatasToChange = [
token_metadata_from_row(row)
async for row in database.execute(query=query)
]
logging.info(
f'Updating {len(tokenMetadatasToChange)} transfers...')
for tokenMetadata in tokenMetadatasToChange:
try:
tokenMetadataDict = tokenMetadataProcessor._resolve_data(
dataString=tokenMetadata.metadataUrl,
registryAddress=tokenMetadata.registryAddress,
tokenId=tokenMetadata.tokenId)
if tokenMetadataDict:
logging.info(
f'Processed: {tokenMetadata.tokenMetadataId}')
await saver.update_token_metadata(
tokenMetadataId=tokenMetadata.tokenMetadataId,
name=tokenMetadataDict.get('name'),
imageUrl=tokenMetadataDict.get('image'),
description=tokenMetadataDict.get('description'),
attributes=tokenMetadataDict.get('attributes', []))
except Exception as e:
logging.exception(
f'Error processing {tokenMetadata.tokenMetadataId}: {e}'
)
currentId = currentId + batchSize
await database.disconnect()
async def process_token_ownerships(startTokenId: int, endTokenId: int, batchSize: int):
databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"])
database = Database(connectionString=databaseConnectionString)
saver = Saver(database=database)
retriever = Retriever(database=database)
s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'])
workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue')
tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue')
awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'])
ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester)
requester = Requester()
tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET'])
openseaApiKey = os.environ['OPENSEA_API_KEY']
tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever)
collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET'])
tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor)
revueApiKey = os.environ['REVUE_API_KEY']
slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications')
await database.connect()
await workQueue.connect()
await s3manager.connect()
await tokenQueue.connect()
await database.connect()
await slackClient.post(text=f'process_token_ownerships → 🚧 started: {startTokenId}-{endTokenId}')
try:
currentTokenId = startTokenId
while currentTokenId < endTokenId:
start = currentTokenId
end = min(currentTokenId + batchSize, endTokenId)
currentTokenId = end
logging.info(f'Working on {start}-{end}')
query = TokenMetadatasTable.select() \
.where(TokenMetadatasTable.c.tokenMetadataId >= start) \
.where(TokenMetadatasTable.c.tokenMetadataId < end)
tokenMetadatas = await retriever.query_token_metadatas(query=query)
await asyncio.gather(*[process_token_ownership(tokenManager=tokenManager, registryAddress=tokenMetadata.registryAddress, tokenId=tokenMetadata.tokenId) for tokenMetadata in tokenMetadatas])
await slackClient.post(text=f'process_token_ownerships → ✅ completed : {startTokenId}-{endTokenId}')
except Exception as exception:
await slackClient.post(text=f'process_token_ownerships → � error: {startTokenId}-{endTokenId}\n```{str(exception)}```')
raise exception
finally:
await database.disconnect()
await workQueue.disconnect()
await tokenQueue.disconnect()
await s3manager.disconnect()
async def _update_token_metadatas(tokensToProcess: Sequence[tuple], tokenManager: TokenManager, retriever: Retriever):
query = (
TokenMetadatasTable.select()
.where(sqlalchemy.tuple_(TokenMetadatasTable.c.registryAddress, TokenMetadatasTable.c.tokenId).in_(tokensToProcess))
)
recentlyUpdatedTokenMetadatas = await retriever.query_token_metadatas(query=query)
recentlyUpdatedTokenIds = set((tokenMetadata.registryAddress, tokenMetadata.tokenId) for tokenMetadata in recentlyUpdatedTokenMetadatas)
tokensToUpdate = set(tokensToProcess) - recentlyUpdatedTokenIds
print('len(tokensToUpdate)', len(tokensToUpdate))
for tokensToUpdateChunk in list_util.generate_chunks(lst=list(tokensToUpdate), chunkSize=10):
tokenProcessResults = await asyncio.gather(*[tokenManager.update_token_metadata(registryAddress=registryAddress, tokenId=tokenId) for (registryAddress, tokenId) in tokensToUpdateChunk], return_exceptions=True)
tokenProcessSuccessCount = tokenProcessResults.count(None)
if tokenProcessSuccessCount:
print(f'{tokenProcessSuccessCount} / {len(tokenProcessResults)} token updates succeeded')
# NOTE(krishan711): if less than 90% of things succeed, bail out
if len(tokenProcessResults) >= 100 and tokenProcessSuccessCount / len(tokenProcessResults) < 0.9:
raise Exception('Less than 90% of token updates failed!')
async def reprocess_metadata(startId: Optional[int], endId: Optional[int],
batchSize: Optional[int]):
databaseConnectionString = Database.create_psql_connection_string(
username=os.environ["DB_USERNAME"],
password=os.environ["DB_PASSWORD"],
host=os.environ["DB_HOST"],
port=os.environ["DB_PORT"],
name=os.environ["DB_NAME"])
database = Database(connectionString=databaseConnectionString)
saver = Saver(database=database)
retriever = Retriever(database=database)
s3manager = S3Manager(region='eu-west-1',
accessKeyId=os.environ['AWS_KEY'],
accessKeySecret=os.environ['AWS_SECRET'])
tokenQueue = SqsMessageQueue(
region='eu-west-1',
accessKeyId=os.environ['AWS_KEY'],
accessKeySecret=os.environ['AWS_SECRET'],
queueUrl=
'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue')
awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'],
accessKeySecret=os.environ['AWS_SECRET'])
requester = Requester()
ethClient = RestEthClient(
url=
'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com',
requester=awsRequester)
tokenMetadataProcessor = TokenMetadataProcessor(
requester=requester,
ethClient=ethClient,
s3manager=s3manager,
bucketName=os.environ['S3_BUCKET'])
openseaApiKey = os.environ['OPENSEA_API_KEY']
collectionProcessor = CollectionProcessor(
requester=requester,
ethClient=ethClient,
openseaApiKey=openseaApiKey,
s3manager=s3manager,
bucketName=os.environ['S3_BUCKET'])
tokenManger = TokenManager(saver=saver,
retriever=retriever,
tokenQueue=tokenQueue,
collectionProcessor=collectionProcessor,
tokenMetadataProcessor=tokenMetadataProcessor)
await s3manager.connect()
await tokenQueue.connect()
await database.connect()
if not startId:
startId = 0
if not endId:
maxTokenMetadata = await retriever.list_token_metadatas(
limit=1,
orders=[
Order(fieldName=TokenMetadatasTable.c.tokenMetadataId.key,
direction=Direction.DESCENDING)
])
print(maxTokenMetadata)
endId = maxTokenMetadata[0].tokenMetadataId + 1
currentId = startId
while currentId < endId:
start = currentId
end = min(currentId + batchSize, endId)
query = TokenMetadatasTable.select()
query = query.where(TokenMetadatasTable.c.tokenMetadataId >= start)
query = query.where(TokenMetadatasTable.c.tokenMetadataId < end)
query = query.where(
TokenMetadatasTable.c.updatedDate < datetime.datetime(2022, 2, 13))
query = query.order_by(TokenMetadatasTable.c.tokenMetadataId.asc())
tokenMetadatasToChange = [
token_metadata_from_row(row)
for row in await database.execute(query=query)
]
logging.info(f'Working on {start} - {end}')
logging.info(f'Updating {len(tokenMetadatasToChange)} transfers...')
await asyncio.gather(*[
_reprocess_metadata_from_s3(
tokenMetadataProcessor=tokenMetadataProcessor,
s3manager=s3manager,
tokenManger=tokenManger,
tokenMetadata=tokenMetadata)
for tokenMetadata in tokenMetadatasToChange
])
currentId = currentId + batchSize
await s3manager.disconnect()
await tokenQueue.disconnect()
await awsRequester.close_connections()
await requester.close_connections()
await database.disconnect()