async def process_tokens_from_old_transfers(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) requester = Requester() tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever) collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor) revueApiKey = os.environ['REVUE_API_KEY'] await database.connect() await workQueue.connect() await s3manager.connect() await tokenQueue.connect() cache = set() registryCache = set() currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(currentBlockNumber + batchSize, endBlockNumber) currentBlockNumber = end logging.info(f'Working on {start}-{end}...') query = ( sqlalchemy.select(TokenTransfersTable.c.registryAddress, TokenTransfersTable.c.tokenId) .where(TokenTransfersTable.c.blockNumber >= start) .where(TokenTransfersTable.c.blockNumber < end) ) result = await database.execute(query=query,) tokensToProcess = set() collectionsToProcess = set() for (registryAddress, tokenId) in result: if (registryAddress, tokenId) in cache: continue cache.add((registryAddress, tokenId)) tokensToProcess.add((registryAddress, tokenId)) if registryAddress in registryCache: continue registryCache.add(registryAddress) collectionsToProcess.add(registryAddress) print('len(tokensToProcess)', len(tokensToProcess)) print('len(collectionsToProcess)', len(collectionsToProcess)) try: await _update_token_metadatas(tokensToProcess=tokensToProcess, tokenManager=tokenManager, retriever=retriever) await _update_collections(collectionsToProcess=collectionsToProcess, tokenManager=tokenManager, retriever=retriever) except: logging.error(f'Failed during: {start}-{end}') raise await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await s3manager.disconnect()
async def main(): requestIdHolder = RequestIdHolder() name = os.environ.get('NAME', 'notd-api') version = os.environ.get('VERSION', 'local') environment = os.environ.get('ENV', 'dev') isRunningDebugMode = environment == 'dev' if isRunningDebugMode: logging.init_basic_logging() else: logging.init_json_logging(name=name, version=version, environment=environment, requestIdHolder=requestIdHolder) databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) requester = Requester() tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever) collectionActivityProcessor = CollectionActivityProcessor(retriever=retriever) revueApiKey = os.environ['REVUE_API_KEY'] tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor, collectionActivityProcessor=collectionActivityProcessor) notdManager = NotdManager(blockProcessor=blockProcessor, saver=saver, retriever=retriever, workQueue=workQueue, tokenManager=tokenManager, requester=requester, revueApiKey=revueApiKey) processor = NotdMessageProcessor(notdManager=notdManager) slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') workQueueProcessor = MessageQueueProcessor(queue=workQueue, messageProcessor=processor, slackClient=slackClient, requestIdHolder=requestIdHolder) tokenQueueProcessor = MessageQueueProcessor(queue=tokenQueue, messageProcessor=processor, slackClient=slackClient, requestIdHolder=requestIdHolder) await database.connect() await s3manager.connect() await workQueue.connect() await tokenQueue.connect() try: while True: hasProcessedWork = await workQueueProcessor.execute_batch(batchSize=3, longPollSeconds=1, shouldProcessInParallel=True) if hasProcessedWork: continue hasProcessedToken = await tokenQueueProcessor.execute_batch(batchSize=10, longPollSeconds=1, shouldProcessInParallel=True) if hasProcessedToken: continue logging.info('No message received.. sleeping') time.sleep(60) finally: await database.disconnect() await s3manager.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections()
async def reprocess_metadata(startId: int, endId: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database) tokenMetadataProcessor = TokenMetadataProcessor(requester=None, ethClient=None, s3manager=None, bucketName=None) await database.connect() currentId = startId while currentId < endId: start = currentId end = min(currentId + batchSize, endId) logging.info(f'Working on {start} to {end}...') async with database.transaction(): query = TokenMetadatasTable.select() query = query.where(TokenMetadatasTable.c.tokenMetadataId >= start) query = query.where(TokenMetadatasTable.c.tokenMetadataId < end) query = query.where( TokenMetadatasTable.c.metadataUrl.startswith('data:')) query = query.where(TokenMetadatasTable.c.name == None) tokenMetadatasToChange = [ token_metadata_from_row(row) async for row in database.execute(query=query) ] logging.info( f'Updating {len(tokenMetadatasToChange)} transfers...') for tokenMetadata in tokenMetadatasToChange: try: tokenMetadataDict = tokenMetadataProcessor._resolve_data( dataString=tokenMetadata.metadataUrl, registryAddress=tokenMetadata.registryAddress, tokenId=tokenMetadata.tokenId) if tokenMetadataDict: logging.info( f'Processed: {tokenMetadata.tokenMetadataId}') await saver.update_token_metadata( tokenMetadataId=tokenMetadata.tokenMetadataId, name=tokenMetadataDict.get('name'), imageUrl=tokenMetadataDict.get('image'), description=tokenMetadataDict.get('description'), attributes=tokenMetadataDict.get('attributes', [])) except Exception as e: logging.exception( f'Error processing {tokenMetadata.tokenMetadataId}: {e}' ) currentId = currentId + batchSize await database.disconnect()
async def process_token_ownerships(startTokenId: int, endTokenId: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) requester = Requester() tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever) collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor) revueApiKey = os.environ['REVUE_API_KEY'] slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') await database.connect() await workQueue.connect() await s3manager.connect() await tokenQueue.connect() await database.connect() await slackClient.post(text=f'process_token_ownerships → 🚧 started: {startTokenId}-{endTokenId}') try: currentTokenId = startTokenId while currentTokenId < endTokenId: start = currentTokenId end = min(currentTokenId + batchSize, endTokenId) currentTokenId = end logging.info(f'Working on {start}-{end}') query = TokenMetadatasTable.select() \ .where(TokenMetadatasTable.c.tokenMetadataId >= start) \ .where(TokenMetadatasTable.c.tokenMetadataId < end) tokenMetadatas = await retriever.query_token_metadatas(query=query) await asyncio.gather(*[process_token_ownership(tokenManager=tokenManager, registryAddress=tokenMetadata.registryAddress, tokenId=tokenMetadata.tokenId) for tokenMetadata in tokenMetadatas]) await slackClient.post(text=f'process_token_ownerships → ✅ completed : {startTokenId}-{endTokenId}') except Exception as exception: await slackClient.post(text=f'process_token_ownerships → � error: {startTokenId}-{endTokenId}\n```{str(exception)}```') raise exception finally: await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await s3manager.disconnect()
async def process_collection(address: str, shouldDefer: bool): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') requester = Requester() awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) revueApiKey = os.environ['REVUE_API_KEY'] tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor) notdManager = NotdManager(blockProcessor=blockProcessor, saver=saver, retriever=retriever, workQueue=workQueue, tokenManager=tokenManager, requester=requester, revueApiKey=revueApiKey) await database.connect() await s3manager.connect() await workQueue.connect() await tokenQueue.connect() retrievedCollectionTokenMetadatas = await retriever.list_token_metadatas( fieldFilters=[ StringFieldFilter(fieldName=TokenTransfersTable.c.registryAddress.key, eq=address), ], ) for tokenMetadata in retrievedCollectionTokenMetadatas: if shouldDefer: await notdManager.update_token_metadata_deferred(registryAddress=address, tokenId=tokenMetadata.tokenId, shouldForce=True) else: await notdManager.update_token_metadata(registryAddress=address, tokenId=tokenMetadata.tokenId, shouldForce=True) await database.disconnect() await s3manager.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections()
async def reprocess_metadata(startId: Optional[int], endId: Optional[int], batchSize: Optional[int]): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) requester = Requester() ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) tokenMetadataProcessor = TokenMetadataProcessor( requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] collectionProcessor = CollectionProcessor( requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManger = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor) await s3manager.connect() await tokenQueue.connect() await database.connect() if not startId: startId = 0 if not endId: maxTokenMetadata = await retriever.list_token_metadatas( limit=1, orders=[ Order(fieldName=TokenMetadatasTable.c.tokenMetadataId.key, direction=Direction.DESCENDING) ]) print(maxTokenMetadata) endId = maxTokenMetadata[0].tokenMetadataId + 1 currentId = startId while currentId < endId: start = currentId end = min(currentId + batchSize, endId) query = TokenMetadatasTable.select() query = query.where(TokenMetadatasTable.c.tokenMetadataId >= start) query = query.where(TokenMetadatasTable.c.tokenMetadataId < end) query = query.where( TokenMetadatasTable.c.updatedDate < datetime.datetime(2022, 2, 13)) query = query.order_by(TokenMetadatasTable.c.tokenMetadataId.asc()) tokenMetadatasToChange = [ token_metadata_from_row(row) for row in await database.execute(query=query) ] logging.info(f'Working on {start} - {end}') logging.info(f'Updating {len(tokenMetadatasToChange)} transfers...') await asyncio.gather(*[ _reprocess_metadata_from_s3( tokenMetadataProcessor=tokenMetadataProcessor, s3manager=s3manager, tokenManger=tokenManger, tokenMetadata=tokenMetadata) for tokenMetadata in tokenMetadatasToChange ]) currentId = currentId + batchSize await s3manager.disconnect() await tokenQueue.disconnect() await awsRequester.close_connections() await requester.close_connections() await database.disconnect()
async def reprocess_transfers(startBlockNumber: int, endBlockNumber: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) requester = Requester() slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) notdManager = NotdManager(blockProcessor=None, saver=saver, retriever=retriever, workQueue=None, tokenManager=None, requester=requester, revueApiKey=None) await database.connect() await slackClient.post( text= f'reprocess_transfers → 🚧 started: {startBlockNumber}-{endBlockNumber}' ) try: currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: logging.info(f'Working on {currentBlockNumber}') try: retrievedTokenTransfers = await blockProcessor.process_block( blockNumber=currentBlockNumber) except Exception as exception: logging.info( f'Got exception whilst getting blocks: {str(exception)}. Will retry in 10 secs.' ) time.sleep(60) currentBlockNumber -= 1 continue await notdManager._save_block_transfers( blockNumber=currentBlockNumber, retrievedTokenTransfers=retrievedTokenTransfers) currentBlockNumber = currentBlockNumber + 1 await slackClient.post( text= f'reprocess_transfers → ✅ completed : {startBlockNumber}-{endBlockNumber}' ) except Exception as exception: await slackClient.post( text= f'reprocess_transfers → � error: {startBlockNumber}-{endBlockNumber}\n```{str(exception)}```' ) raise exception finally: await database.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def run(registryAddress: Optional[str]): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') requester = Requester() awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=None, tokenMetadataProcessor=None, tokenOwnershipProcessor=None) # NOTE(krishan711): use tokenqueue so its lower prioritized work notdManager = NotdManager(blockProcessor=blockProcessor, saver=saver, retriever=retriever, workQueue=tokenQueue, tokenManager=tokenManager, requester=requester, revueApiKey=None) await database.connect() await workQueue.connect() await tokenQueue.connect() if registryAddress: registryAddresses = [registryAddress] else: query = sqlalchemy.select(TokenCollectionsTable.c.address).filter( TokenCollectionsTable.c.doesSupportErc1155 == True).order_by( TokenCollectionsTable.c.collectionId.desc()) results = await database.execute(query=query) registryAddresses = [ registryAddress for (registryAddress, ) in results ] print( f'Starting to reprocess blocks for {len(registryAddresses)} collections' ) for index, registryAddress in enumerate(registryAddresses): print( f'Reprocessing blocks for collection: {registryAddress} (index: {index})' ) minDate = datetime.datetime(2022, 4, 8, 9, 0) query = ( sqlalchemy.select(sqlalchemy.distinct(BlocksTable.c.blockNumber)) \ .join(TokenTransfersTable, TokenTransfersTable.c.blockNumber == BlocksTable.c.blockNumber) \ .filter(TokenTransfersTable.c.registryAddress == registryAddress) .filter(BlocksTable.c.updatedDate < minDate) ) results = await database.execute(query=query) blockNumbers = set(blockNumber for (blockNumber, ) in results) print(f'Processing {len(blockNumbers)} blocks') if len(blockNumbers) == 0: continue # await notdManager.process_blocks_deferred(blockNumbers=blockNumbers) for blockNumberChunk in list_util.generate_chunks( lst=list(blockNumbers), chunkSize=5): await asyncio.gather(*[ notdManager.process_block(blockNumber=blockNumber) for blockNumber in blockNumberChunk ]) query = ( sqlalchemy.select(TokenMetadatasTable.c.tokenId) \ .filter(TokenMetadatasTable.c.registryAddress == registryAddress) ) results = await database.execute(query=query) collectionTokenIds = [(registryAddress, tokenId) for (tokenId, ) in results] await tokenManager.update_token_ownerships_deferred( collectionTokenIds=collectionTokenIds) await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def reprocess_collections(startId: int, endId: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database) retriever = Retriever(database) openseaApiKey = os.environ['OPENSEA_API_KEY'] awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) requester = Requester() # ethClient = RestEthClient(url=f'https://mainnet.infura.io/v3/{os.environ["INFURA_PROJECT_ID"]}', requester=requester) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) requester = Requester() collectionProcessor = CollectionProcessor( requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=None, collectionProcessor=collectionProcessor, tokenMetadataProcessor=None) await database.connect() await s3manager.connect() currentId = startId while currentId < endId: start = currentId end = min(currentId + batchSize, endId) logging.info(f'Working on {start} to {end}...') async with database.transaction(): query = TokenCollectionsTable.select() query = query.where(TokenCollectionsTable.c.collectionId >= start) query = query.where(TokenCollectionsTable.c.collectionId < end) collectionsToChange = [ collection_from_row(row) async for row in database.iterate(query=query) ] logging.info(f'Updating {len(collectionsToChange)} collections...') for collection in collectionsToChange: logging.info(f'Updating collection: {collection.address}') try: await tokenManager.update_collection( address=collection.address, shouldForce=True) except Exception as e: logging.exception( f'Error processing {collection.collectionId}: {e}') currentId = currentId + batchSize await database.disconnect() await s3manager.disconnect() await requester.close_connections() await awsRequester.close_connections()
if isRunningDebugMode: logging.init_basic_logging() else: logging.init_json_logging(name=name, version=version, environment=environment, requestIdHolder=requestIdHolder) databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue' ) tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue'
async def reprocess_bad_blocks(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') requester = Requester() slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) # NOTE(krishan711): use tokenqueue so its lower prioritized work notdManager = NotdManager(blockProcessor=None, saver=saver, retriever=retriever, workQueue=tokenQueue, tokenManager=None, requester=requester, revueApiKey=None) await database.connect() await workQueue.connect() await tokenQueue.connect() await slackClient.post( text= f'reprocess_bad_blocks → 🚧 started: {startBlockNumber}-{endBlockNumber}' ) try: currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(start + batchSize, endBlockNumber) logging.info(f'Working on {start}-{end}') blockNumbers = set(range(start, end)) processedBlocksQuey = (sqlalchemy.select( BlocksTable.c.blockNumber).where( BlocksTable.c.blockNumber >= start).where( BlocksTable.c.blockNumber < end)) results = await database.execute(query=processedBlocksQuey) processedBlocks = {blockNumber for (blockNumber, ) in results} logging.info(f'Ignoring {len(processedBlocks)} processedBlocks') blockNumbers = list(blockNumbers - processedBlocks) # blockUncleCounts = [] # for chunk in list_util.generate_chunks(lst=blockNumbers, chunkSize=10): # blockUncleCounts += await asyncio.gather(*[blockProcessor.ethClient.get_block_uncle_count(blockNumber=blockNumber) for blockNumber in chunk]) # blocksWithUncles = {blockNumber for (blockNumber, uncleCount) in zip(blockNumbers, blockUncleCounts) if uncleCount > 0} blocksWithUncles = set() logging.info(f'Found {len(blocksWithUncles)} blocks with uncles') blocksWithDuplicatesQuery = (sqlalchemy.select( TokenTransfersTable.c.blockNumber, sqlalchemy.func.count( sqlalchemy.func.distinct(TokenTransfersTable.c.blockHash)) ).where(TokenTransfersTable.c.blockNumber >= start).where( TokenTransfersTable.c.blockNumber < end).group_by( TokenTransfersTable.c.blockNumber)) results = await database.execute(query=blocksWithDuplicatesQuery) blocksWithDuplicates = { blockNumber for (blockNumber, blockHashCount) in results if blockHashCount > 1 } logging.info( f'Found {len(blocksWithDuplicates)} blocks with multiple blockHashes' ) badBlockTransactionsQuery = (sqlalchemy.select( TokenTransfersTable.c.transactionHash).where( TokenTransfersTable.c.blockNumber.in_( blocksWithDuplicates))) results = await database.execute(query=badBlockTransactionsQuery) badBlockTransactions = { transactionHash for (transactionHash, ) in results } logging.info( f'Found {len(badBlockTransactions)} transactions in bad blocks' ) badBlockTransactionActualBlocks = set() for chunk in list_util.generate_chunks( lst=list(badBlockTransactions), chunkSize=10): transactionReceipts = await asyncio.gather(*[ blockProcessor.get_transaction_receipt( transactionHash=transactionHash) for transactionHash in chunk ]) badBlockTransactionActualBlocks.update({ transactionReceipt['blockNumber'] for transactionReceipt in transactionReceipts if transactionReceipt is not None }) badBlockTransactionBlocksQuery = (sqlalchemy.select( sqlalchemy.func.distinct( TokenTransfersTable.c.blockNumber)).where( TokenTransfersTable.c.transactionHash.in_( badBlockTransactions))) results = await database.execute( query=badBlockTransactionBlocksQuery) badBlockTransactionBlocks = { blockNumber for (blockNumber, ) in results } allBadBlocks = blocksWithUncles.union( badBlockTransactionActualBlocks).union( blocksWithDuplicates).union(badBlockTransactionBlocks) logging.info(f'Found {len(allBadBlocks)} blocks to reprocess') await notdManager.process_blocks_deferred(blockNumbers=allBadBlocks ) insertQuery = BlocksTable.insert().from_select( [ BlocksTable.c.createdDate.key, BlocksTable.c.updatedDate.key, BlocksTable.c.blockNumber.key, BlocksTable.c.blockHash.key, BlocksTable.c.blockDate.key ], sqlalchemy.select( sqlalchemy.func.min(TokenTransfersTable.c.blockDate) + datetime.timedelta(minutes=15), sqlalchemy.func.min(TokenTransfersTable.c.blockDate) + datetime.timedelta(minutes=15), TokenTransfersTable.c.blockNumber, TokenTransfersTable.c.blockHash, sqlalchemy.func.min( TokenTransfersTable.c.blockDate)).where( TokenTransfersTable.c.blockNumber.in_( set(blockNumbers) - allBadBlocks)). where(TokenTransfersTable.c.blockNumber >= start).where( TokenTransfersTable.c.blockNumber < end).group_by( TokenTransfersTable.c.blockNumber, TokenTransfersTable.c.blockHash)) async with database.create_transaction() as connection: await database.execute(connection=connection, query=insertQuery) currentBlockNumber = end await slackClient.post( text= f'reprocess_bad_blocks → ✅ completed : {startBlockNumber}-{endBlockNumber}' ) except Exception as exception: await slackClient.post( text= f'reprocess_bad_blocks → � error: {startBlockNumber}-{endBlockNumber}\n```{str(exception)}```' ) raise exception finally: await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def backfill_collection_activities(startBlock: int, endBlock: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) retriever = Retriever(database=database) saver = Saver(database=database) tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') collectionActivityProcessor = CollectionActivityProcessor( retriever=retriever) tokenManager = TokenManager( saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=None, tokenMetadataProcessor=None, tokenOwnershipProcessor=None, collectionActivityProcessor=collectionActivityProcessor) await database.connect() await tokenQueue.connect() currentBlockNumber = startBlock while currentBlockNumber < endBlock: endBlockNumber = min(currentBlockNumber + batchSize, endBlock) logging.info(f'Working on {currentBlockNumber} to {endBlockNumber}...') tokenTransfers = await retriever.list_token_transfers( fieldFilters=[ IntegerFieldFilter(BlocksTable.c.blockNumber.key, gte=currentBlockNumber), IntegerFieldFilter(BlocksTable.c.blockNumber.key, lte=endBlockNumber), ], orders=[ Order(fieldName=BlocksTable.c.blockDate.key, direction=Direction.ASCENDING) ], ) if len(tokenTransfers) == 0: print( f"Skipping {currentBlockNumber} to {endBlockNumber} with 0 transfers " ) else: collectionHourlyActivities = await retriever.list_collections_activity( fieldFilters=[ DateFieldFilter(CollectionHourlyActivityTable.c.date.key, gte=date_hour_from_datetime( tokenTransfers[0].blockDate)), DateFieldFilter(CollectionHourlyActivityTable.c.date.key, lte=date_hour_from_datetime( tokenTransfers[-1].blockDate)), ], ) processedPairs = { (collectionHourlyActivity.address, collectionHourlyActivity.date) for collectionHourlyActivity in collectionHourlyActivities } registryDatePairs = { (tokenTransfer.registryAddress, date_hour_from_datetime(tokenTransfer.blockDate)) for tokenTransfer in tokenTransfers if (tokenTransfer.registryAddress, date_hour_from_datetime(tokenTransfer.blockDate) ) not in processedPairs } print( f'Processing {len(registryDatePairs)} pairs from {len(tokenTransfers)} transfers' ) # messages = [UpdateActivityForCollectionMessageContent(address=address, startDate=startDate).to_message() for (address, startDate) in registryDatePairs] # await tokenQueue.send_messages(messages=messages) for pairChunk in list_util.generate_chunks( lst=list(registryDatePairs), chunkSize=50): await asyncio.gather(*[ tokenManager.update_activity_for_collection( address=registryAddress, startDate=startDate) for registryAddress, startDate in pairChunk ]) currentBlockNumber = endBlockNumber await database.disconnect() await tokenQueue.disconnect()