async def daily_new_registries(): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) await database.connect() query = TokenTransfersTable.select() query = query.where( TokenTransfersTable.c.registryAddress.in_( TokenTransfersTable.select().with_only_columns([ TokenTransfersTable.c.registryAddress ]).group_by(TokenTransfersTable.c.registryAddress).having( sqlalchemyfunc.count(TokenTransfersTable.c.registryAddress) == 1))) query = query.where( sqlalchemyfunc.date(TokenTransfersTable.c.blockDate) == sqlalchemyfunc.current_date()) rows = await database.fetch_all(query) for row in rows: logging.info( f'New Tokens: registry address {row[2]} and tokenId {row[5]}') await database.disconnect()
async def check_all_processed(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) await database.connect() currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(currentBlockNumber + batchSize, endBlockNumber) logging.info(f'Working on {start} to {end}...') result = await database.execute(f''' update tbl_token_transfers set amount_2 = amount where block_number >= {start} and block_number < {end} and amount_2 is null; ''') print('result:', result) currentBlockNumber += batchSize await database.disconnect()
async def process_tokens_from_old_transfers(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) requester = Requester() tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever) collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor) revueApiKey = os.environ['REVUE_API_KEY'] await database.connect() await workQueue.connect() await s3manager.connect() await tokenQueue.connect() cache = set() registryCache = set() currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(currentBlockNumber + batchSize, endBlockNumber) currentBlockNumber = end logging.info(f'Working on {start}-{end}...') query = ( sqlalchemy.select(TokenTransfersTable.c.registryAddress, TokenTransfersTable.c.tokenId) .where(TokenTransfersTable.c.blockNumber >= start) .where(TokenTransfersTable.c.blockNumber < end) ) result = await database.execute(query=query,) tokensToProcess = set() collectionsToProcess = set() for (registryAddress, tokenId) in result: if (registryAddress, tokenId) in cache: continue cache.add((registryAddress, tokenId)) tokensToProcess.add((registryAddress, tokenId)) if registryAddress in registryCache: continue registryCache.add(registryAddress) collectionsToProcess.add(registryAddress) print('len(tokensToProcess)', len(tokensToProcess)) print('len(collectionsToProcess)', len(collectionsToProcess)) try: await _update_token_metadatas(tokensToProcess=tokensToProcess, tokenManager=tokenManager, retriever=retriever) await _update_collections(collectionsToProcess=collectionsToProcess, tokenManager=tokenManager, retriever=retriever) except: logging.error(f'Failed during: {start}-{end}') raise await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await s3manager.disconnect()
async def main(): requestIdHolder = RequestIdHolder() name = os.environ.get('NAME', 'notd-api') version = os.environ.get('VERSION', 'local') environment = os.environ.get('ENV', 'dev') isRunningDebugMode = environment == 'dev' if isRunningDebugMode: logging.init_basic_logging() else: logging.init_json_logging(name=name, version=version, environment=environment, requestIdHolder=requestIdHolder) databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) requester = Requester() tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever) collectionActivityProcessor = CollectionActivityProcessor(retriever=retriever) revueApiKey = os.environ['REVUE_API_KEY'] tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor, collectionActivityProcessor=collectionActivityProcessor) notdManager = NotdManager(blockProcessor=blockProcessor, saver=saver, retriever=retriever, workQueue=workQueue, tokenManager=tokenManager, requester=requester, revueApiKey=revueApiKey) processor = NotdMessageProcessor(notdManager=notdManager) slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') workQueueProcessor = MessageQueueProcessor(queue=workQueue, messageProcessor=processor, slackClient=slackClient, requestIdHolder=requestIdHolder) tokenQueueProcessor = MessageQueueProcessor(queue=tokenQueue, messageProcessor=processor, slackClient=slackClient, requestIdHolder=requestIdHolder) await database.connect() await s3manager.connect() await workQueue.connect() await tokenQueue.connect() try: while True: hasProcessedWork = await workQueueProcessor.execute_batch(batchSize=3, longPollSeconds=1, shouldProcessInParallel=True) if hasProcessedWork: continue hasProcessedToken = await tokenQueueProcessor.execute_batch(batchSize=10, longPollSeconds=1, shouldProcessInParallel=True) if hasProcessedToken: continue logging.info('No message received.. sleeping') time.sleep(60) finally: await database.disconnect() await s3manager.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections()
async def reprocess_metadata(startId: int, endId: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database) tokenMetadataProcessor = TokenMetadataProcessor(requester=None, ethClient=None, s3manager=None, bucketName=None) await database.connect() currentId = startId while currentId < endId: start = currentId end = min(currentId + batchSize, endId) logging.info(f'Working on {start} to {end}...') async with database.transaction(): query = TokenMetadatasTable.select() query = query.where(TokenMetadatasTable.c.tokenMetadataId >= start) query = query.where(TokenMetadatasTable.c.tokenMetadataId < end) query = query.where( TokenMetadatasTable.c.metadataUrl.startswith('data:')) query = query.where(TokenMetadatasTable.c.name == None) tokenMetadatasToChange = [ token_metadata_from_row(row) async for row in database.execute(query=query) ] logging.info( f'Updating {len(tokenMetadatasToChange)} transfers...') for tokenMetadata in tokenMetadatasToChange: try: tokenMetadataDict = tokenMetadataProcessor._resolve_data( dataString=tokenMetadata.metadataUrl, registryAddress=tokenMetadata.registryAddress, tokenId=tokenMetadata.tokenId) if tokenMetadataDict: logging.info( f'Processed: {tokenMetadata.tokenMetadataId}') await saver.update_token_metadata( tokenMetadataId=tokenMetadata.tokenMetadataId, name=tokenMetadataDict.get('name'), imageUrl=tokenMetadataDict.get('image'), description=tokenMetadataDict.get('description'), attributes=tokenMetadataDict.get('attributes', [])) except Exception as e: logging.exception( f'Error processing {tokenMetadata.tokenMetadataId}: {e}' ) currentId = currentId + batchSize await database.disconnect()
async def calculate_token_fields(startCollectionId: Optional[int], endCollectionId: Optional[int]): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) bucketName = os.environ['S3_BUCKET'] await database.connect() await s3manager.connect() query = TokenCollectionsTable.select() if startCollectionId: query = query.where(TokenCollectionsTable.c.collectionId >= startCollectionId) if endCollectionId: query = query.where(TokenCollectionsTable.c.collectionId < endCollectionId) collections = [collection_from_row(row) async for row in database.iterate(query=query)] rows = [] fields = set() for collection in collections: logging.info(f'Working on {collection.address}') collectionDirectory = f'{bucketName}/token-metadatas/{collection.address}/' index = 0 async for tokenFile in s3manager.generate_directory_files(s3Directory=collectionDirectory): logging.info(f'Working on file {tokenFile.bucket}/{tokenFile.path}') if index > 3: break try: tokenDict = json.loads(await s3manager.read_file(sourcePath=f'{tokenFile.bucket}/{tokenFile.path}')) tokenDict['tokenId'] = tokenFile.path.split('/')[2] if tokenDict.get('attributes'): tokenDict['attributes'] = ",".join(list(set(key for attribute in tokenDict.get('attributes', []) for key in attribute.keys()))) if isinstance(tokenDict.get('attributes', []), List) else [attribute for attribute in tokenDict.get('attributes')] else: tokenDict['attributes'] = None tokenDict['description'] = tokenDict["description"][:10] if tokenDict.get('description') else None tokenDict['collection'] = collection.address fields.update(tokenDict.keys()) rows.append(tokenDict) except Exception as exception: logging.exception(exception) index += 1 with open(f'./output{startCollectionId}-{endCollectionId}.tsv', 'w') as outFile: dictWriter = csv.DictWriter(outFile, fields, delimiter='\t') dictWriter.writeheader() dictWriter.writerows(rows) fieldCounts = defaultdict(int) for row in rows: for key, value in row.items(): if value: fieldCounts[key] += 1 print(fieldCounts) await database.disconnect() await s3manager.disconnect()
async def process_token_ownerships(startTokenId: int, endTokenId: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) requester = Requester() tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] tokenOwnershipProcessor = TokenOwnershipProcessor(retriever=retriever) collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor, tokenOwnershipProcessor=tokenOwnershipProcessor) revueApiKey = os.environ['REVUE_API_KEY'] slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') await database.connect() await workQueue.connect() await s3manager.connect() await tokenQueue.connect() await database.connect() await slackClient.post(text=f'process_token_ownerships → 🚧 started: {startTokenId}-{endTokenId}') try: currentTokenId = startTokenId while currentTokenId < endTokenId: start = currentTokenId end = min(currentTokenId + batchSize, endTokenId) currentTokenId = end logging.info(f'Working on {start}-{end}') query = TokenMetadatasTable.select() \ .where(TokenMetadatasTable.c.tokenMetadataId >= start) \ .where(TokenMetadatasTable.c.tokenMetadataId < end) tokenMetadatas = await retriever.query_token_metadatas(query=query) await asyncio.gather(*[process_token_ownership(tokenManager=tokenManager, registryAddress=tokenMetadata.registryAddress, tokenId=tokenMetadata.tokenId) for tokenMetadata in tokenMetadatas]) await slackClient.post(text=f'process_token_ownerships → ✅ completed : {startTokenId}-{endTokenId}') except Exception as exception: await slackClient.post(text=f'process_token_ownerships → � error: {startTokenId}-{endTokenId}\n```{str(exception)}```') raise exception finally: await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await s3manager.disconnect()
async def fix_address(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) await database.connect() currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(currentBlockNumber + batchSize, endBlockNumber) logging.info(f'Working on {start} to {end}...') async with database.transaction(): query = TokenTransfersTable.select() query = query.where(TokenTransfersTable.c.blockNumber >= start) query = query.where(TokenTransfersTable.c.blockNumber < end) query = query.where( or_( sqlalchemyfunc.length( TokenTransfersTable.c.toAddress) != 42, sqlalchemyfunc.length(TokenTransfersTable.c.toAddress) != 42, )) tokenTransfersToChange = [ token_transfer_from_row(row) async for row in database.iterate(query=query) ] logging.info( f'Updating {len(tokenTransfersToChange)} transfers...') for tokenTransfer in tokenTransfersToChange: query = TokenTransfersTable.update( TokenTransfersTable.c.tokenTransferId == tokenTransfer.tokenTransferId) values = { TokenTransfersTable.c.toAddress.key: normalize_address(tokenTransfer.toAddress), TokenTransfersTable.c.fromAddress.key: normalize_address(tokenTransfer.fromAddress), } await database.execute(query=query, values=values) currentBlockNumber = end await database.disconnect()
async def check_all_processed(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') await database.connect() await workQueue.connect() currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(currentBlockNumber + batchSize, endBlockNumber) logging.info(f'Working on {start} - {end}...') async with database.transaction(): query = TokenTransfersTable.select() \ .with_only_columns([TokenTransfersTable.c.blockNumber]) \ .filter(TokenTransfersTable.c.blockNumber >= start) \ .filter(TokenTransfersTable.c.blockNumber < end) \ .distinct(TokenTransfersTable.c.blockNumber) processedBlocks = [ row[0] for row in await database.fetch_all(query) ] unprocessedBlocks = set(range(start, end)) - set(processedBlocks) logging.info( f'Processing {len(unprocessedBlocks)} blocks in {start} - {end}') await workQueue.send_message(message=ProcessBlocksMessageContent( blockNumbers=unprocessedBlocks).to_message()) currentBlockNumber = currentBlockNumber + batchSize await database.disconnect() await workQueue.disconnect()
async def process_collection(address: str, shouldDefer: bool): databaseConnectionString = Database.create_psql_connection_string(username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) workQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') requester = Requester() awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) tokenMetadataProcessor = TokenMetadataProcessor(requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] collectionProcessor = CollectionProcessor(requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) revueApiKey = os.environ['REVUE_API_KEY'] tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor) notdManager = NotdManager(blockProcessor=blockProcessor, saver=saver, retriever=retriever, workQueue=workQueue, tokenManager=tokenManager, requester=requester, revueApiKey=revueApiKey) await database.connect() await s3manager.connect() await workQueue.connect() await tokenQueue.connect() retrievedCollectionTokenMetadatas = await retriever.list_token_metadatas( fieldFilters=[ StringFieldFilter(fieldName=TokenTransfersTable.c.registryAddress.key, eq=address), ], ) for tokenMetadata in retrievedCollectionTokenMetadatas: if shouldDefer: await notdManager.update_token_metadata_deferred(registryAddress=address, tokenId=tokenMetadata.tokenId, shouldForce=True) else: await notdManager.update_token_metadata(registryAddress=address, tokenId=tokenMetadata.tokenId, shouldForce=True) await database.disconnect() await s3manager.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections()
async def owned_tokens(ownerAddress: Optional[str]): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) retriever = Retriever(database=database) await database.connect() boughtTokens = [] soldTokens = [] async with database.transaction(): query = TokenTransfersTable.select() query = query.where(TokenTransfersTable.c.toAddress == ownerAddress) async for row in retriever.database.iterate(query=query): tokenTransfer = token_transfer_from_row(row) boughtTokens.append(tokenTransfer) query = TokenTransfersTable.select() query = query.where(TokenTransfersTable.c.fromAddress == ownerAddress) async for row in retriever.database.iterate(query=query): tokenTransfer = token_transfer_from_row(row) soldTokens.append(tokenTransfer) uniqueBoughtTokens = set(boughtTokens) uniqueSoldTokens = set(soldTokens) tokensOwned = uniqueBoughtTokens - uniqueSoldTokens for tokenTransfer in tokensOwned: logging.info( f'Tokens Owned: registry_address: {tokenTransfer.registryAddress}, token_id: {tokenTransfer.tokenId}' ) await database.disconnect() logging.info(f'Got {len(tokensOwned)} total owned')
async def reprocess_metadata(startId: Optional[int], endId: Optional[int], batchSize: Optional[int]): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) requester = Requester() ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) tokenMetadataProcessor = TokenMetadataProcessor( requester=requester, ethClient=ethClient, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) openseaApiKey = os.environ['OPENSEA_API_KEY'] collectionProcessor = CollectionProcessor( requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManger = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=collectionProcessor, tokenMetadataProcessor=tokenMetadataProcessor) await s3manager.connect() await tokenQueue.connect() await database.connect() if not startId: startId = 0 if not endId: maxTokenMetadata = await retriever.list_token_metadatas( limit=1, orders=[ Order(fieldName=TokenMetadatasTable.c.tokenMetadataId.key, direction=Direction.DESCENDING) ]) print(maxTokenMetadata) endId = maxTokenMetadata[0].tokenMetadataId + 1 currentId = startId while currentId < endId: start = currentId end = min(currentId + batchSize, endId) query = TokenMetadatasTable.select() query = query.where(TokenMetadatasTable.c.tokenMetadataId >= start) query = query.where(TokenMetadatasTable.c.tokenMetadataId < end) query = query.where( TokenMetadatasTable.c.updatedDate < datetime.datetime(2022, 2, 13)) query = query.order_by(TokenMetadatasTable.c.tokenMetadataId.asc()) tokenMetadatasToChange = [ token_metadata_from_row(row) for row in await database.execute(query=query) ] logging.info(f'Working on {start} - {end}') logging.info(f'Updating {len(tokenMetadatasToChange)} transfers...') await asyncio.gather(*[ _reprocess_metadata_from_s3( tokenMetadataProcessor=tokenMetadataProcessor, s3manager=s3manager, tokenManger=tokenManger, tokenMetadata=tokenMetadata) for tokenMetadata in tokenMetadatasToChange ]) currentId = currentId + batchSize await s3manager.disconnect() await tokenQueue.disconnect() await awsRequester.close_connections() await requester.close_connections() await database.disconnect()
name = os.environ.get('NAME', 'mdtp-api') version = os.environ.get('VERSION', 'local') environment = os.environ.get('ENV', 'dev') isRunningDebugMode = environment == 'dev' if isRunningDebugMode: logging.init_basic_logging() else: logging.init_json_logging(name=name, version=version, environment=environment, requestIdHolder=requestIdHolder) databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl='https://sqs.eu-west-1.amazonaws.com/097520841056/mdtp-work-queue' ) s3Manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'])
async def reprocess_transfers(startBlockNumber: int, endBlockNumber: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) requester = Requester() slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) notdManager = NotdManager(blockProcessor=None, saver=saver, retriever=retriever, workQueue=None, tokenManager=None, requester=requester, revueApiKey=None) await database.connect() await slackClient.post( text= f'reprocess_transfers → 🚧 started: {startBlockNumber}-{endBlockNumber}' ) try: currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: logging.info(f'Working on {currentBlockNumber}') try: retrievedTokenTransfers = await blockProcessor.process_block( blockNumber=currentBlockNumber) except Exception as exception: logging.info( f'Got exception whilst getting blocks: {str(exception)}. Will retry in 10 secs.' ) time.sleep(60) currentBlockNumber -= 1 continue await notdManager._save_block_transfers( blockNumber=currentBlockNumber, retrievedTokenTransfers=retrievedTokenTransfers) currentBlockNumber = currentBlockNumber + 1 await slackClient.post( text= f'reprocess_transfers → ✅ completed : {startBlockNumber}-{endBlockNumber}' ) except Exception as exception: await slackClient.post( text= f'reprocess_transfers → � error: {startBlockNumber}-{endBlockNumber}\n```{str(exception)}```' ) raise exception finally: await database.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def reprocess_bad_blocks(startBlockNumber: int, endBlockNumber: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') requester = Requester() slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='notd-notifications') awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) # NOTE(krishan711): use tokenqueue so its lower prioritized work notdManager = NotdManager(blockProcessor=None, saver=saver, retriever=retriever, workQueue=tokenQueue, tokenManager=None, requester=requester, revueApiKey=None) await database.connect() await workQueue.connect() await tokenQueue.connect() await slackClient.post( text= f'reprocess_bad_blocks → 🚧 started: {startBlockNumber}-{endBlockNumber}' ) try: currentBlockNumber = startBlockNumber while currentBlockNumber < endBlockNumber: start = currentBlockNumber end = min(start + batchSize, endBlockNumber) logging.info(f'Working on {start}-{end}') blockNumbers = set(range(start, end)) processedBlocksQuey = (sqlalchemy.select( BlocksTable.c.blockNumber).where( BlocksTable.c.blockNumber >= start).where( BlocksTable.c.blockNumber < end)) results = await database.execute(query=processedBlocksQuey) processedBlocks = {blockNumber for (blockNumber, ) in results} logging.info(f'Ignoring {len(processedBlocks)} processedBlocks') blockNumbers = list(blockNumbers - processedBlocks) # blockUncleCounts = [] # for chunk in list_util.generate_chunks(lst=blockNumbers, chunkSize=10): # blockUncleCounts += await asyncio.gather(*[blockProcessor.ethClient.get_block_uncle_count(blockNumber=blockNumber) for blockNumber in chunk]) # blocksWithUncles = {blockNumber for (blockNumber, uncleCount) in zip(blockNumbers, blockUncleCounts) if uncleCount > 0} blocksWithUncles = set() logging.info(f'Found {len(blocksWithUncles)} blocks with uncles') blocksWithDuplicatesQuery = (sqlalchemy.select( TokenTransfersTable.c.blockNumber, sqlalchemy.func.count( sqlalchemy.func.distinct(TokenTransfersTable.c.blockHash)) ).where(TokenTransfersTable.c.blockNumber >= start).where( TokenTransfersTable.c.blockNumber < end).group_by( TokenTransfersTable.c.blockNumber)) results = await database.execute(query=blocksWithDuplicatesQuery) blocksWithDuplicates = { blockNumber for (blockNumber, blockHashCount) in results if blockHashCount > 1 } logging.info( f'Found {len(blocksWithDuplicates)} blocks with multiple blockHashes' ) badBlockTransactionsQuery = (sqlalchemy.select( TokenTransfersTable.c.transactionHash).where( TokenTransfersTable.c.blockNumber.in_( blocksWithDuplicates))) results = await database.execute(query=badBlockTransactionsQuery) badBlockTransactions = { transactionHash for (transactionHash, ) in results } logging.info( f'Found {len(badBlockTransactions)} transactions in bad blocks' ) badBlockTransactionActualBlocks = set() for chunk in list_util.generate_chunks( lst=list(badBlockTransactions), chunkSize=10): transactionReceipts = await asyncio.gather(*[ blockProcessor.get_transaction_receipt( transactionHash=transactionHash) for transactionHash in chunk ]) badBlockTransactionActualBlocks.update({ transactionReceipt['blockNumber'] for transactionReceipt in transactionReceipts if transactionReceipt is not None }) badBlockTransactionBlocksQuery = (sqlalchemy.select( sqlalchemy.func.distinct( TokenTransfersTable.c.blockNumber)).where( TokenTransfersTable.c.transactionHash.in_( badBlockTransactions))) results = await database.execute( query=badBlockTransactionBlocksQuery) badBlockTransactionBlocks = { blockNumber for (blockNumber, ) in results } allBadBlocks = blocksWithUncles.union( badBlockTransactionActualBlocks).union( blocksWithDuplicates).union(badBlockTransactionBlocks) logging.info(f'Found {len(allBadBlocks)} blocks to reprocess') await notdManager.process_blocks_deferred(blockNumbers=allBadBlocks ) insertQuery = BlocksTable.insert().from_select( [ BlocksTable.c.createdDate.key, BlocksTable.c.updatedDate.key, BlocksTable.c.blockNumber.key, BlocksTable.c.blockHash.key, BlocksTable.c.blockDate.key ], sqlalchemy.select( sqlalchemy.func.min(TokenTransfersTable.c.blockDate) + datetime.timedelta(minutes=15), sqlalchemy.func.min(TokenTransfersTable.c.blockDate) + datetime.timedelta(minutes=15), TokenTransfersTable.c.blockNumber, TokenTransfersTable.c.blockHash, sqlalchemy.func.min( TokenTransfersTable.c.blockDate)).where( TokenTransfersTable.c.blockNumber.in_( set(blockNumbers) - allBadBlocks)). where(TokenTransfersTable.c.blockNumber >= start).where( TokenTransfersTable.c.blockNumber < end).group_by( TokenTransfersTable.c.blockNumber, TokenTransfersTable.c.blockHash)) async with database.create_transaction() as connection: await database.execute(connection=connection, query=insertQuery) currentBlockNumber = end await slackClient.post( text= f'reprocess_bad_blocks → ✅ completed : {startBlockNumber}-{endBlockNumber}' ) except Exception as exception: await slackClient.post( text= f'reprocess_bad_blocks → � error: {startBlockNumber}-{endBlockNumber}\n```{str(exception)}```' ) raise exception finally: await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def run(registryAddress: Optional[str]): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-work-queue') tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') requester = Requester() awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) blockProcessor = BlockProcessor(ethClient=ethClient) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=None, tokenMetadataProcessor=None, tokenOwnershipProcessor=None) # NOTE(krishan711): use tokenqueue so its lower prioritized work notdManager = NotdManager(blockProcessor=blockProcessor, saver=saver, retriever=retriever, workQueue=tokenQueue, tokenManager=tokenManager, requester=requester, revueApiKey=None) await database.connect() await workQueue.connect() await tokenQueue.connect() if registryAddress: registryAddresses = [registryAddress] else: query = sqlalchemy.select(TokenCollectionsTable.c.address).filter( TokenCollectionsTable.c.doesSupportErc1155 == True).order_by( TokenCollectionsTable.c.collectionId.desc()) results = await database.execute(query=query) registryAddresses = [ registryAddress for (registryAddress, ) in results ] print( f'Starting to reprocess blocks for {len(registryAddresses)} collections' ) for index, registryAddress in enumerate(registryAddresses): print( f'Reprocessing blocks for collection: {registryAddress} (index: {index})' ) minDate = datetime.datetime(2022, 4, 8, 9, 0) query = ( sqlalchemy.select(sqlalchemy.distinct(BlocksTable.c.blockNumber)) \ .join(TokenTransfersTable, TokenTransfersTable.c.blockNumber == BlocksTable.c.blockNumber) \ .filter(TokenTransfersTable.c.registryAddress == registryAddress) .filter(BlocksTable.c.updatedDate < minDate) ) results = await database.execute(query=query) blockNumbers = set(blockNumber for (blockNumber, ) in results) print(f'Processing {len(blockNumbers)} blocks') if len(blockNumbers) == 0: continue # await notdManager.process_blocks_deferred(blockNumbers=blockNumbers) for blockNumberChunk in list_util.generate_chunks( lst=list(blockNumbers), chunkSize=5): await asyncio.gather(*[ notdManager.process_block(blockNumber=blockNumber) for blockNumber in blockNumberChunk ]) query = ( sqlalchemy.select(TokenMetadatasTable.c.tokenId) \ .filter(TokenMetadatasTable.c.registryAddress == registryAddress) ) results = await database.execute(query=query) collectionTokenIds = [(registryAddress, tokenId) for (tokenId, ) in results] await tokenManager.update_token_ownerships_deferred( collectionTokenIds=collectionTokenIds) await database.disconnect() await workQueue.disconnect() await tokenQueue.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def create_consolidated_metadata(address: str, outputFilename: str): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) retriever = Retriever(database=database) await database.connect() address = chain_util.normalize_address(value=address) collection = await retriever.get_collection_by_address(address=address) tokens = await retriever.list_token_metadatas(fieldFilters=[ StringFieldFilter(fieldName=TokenMetadatasTable.c.registryAddress.key, eq=address) ]) with open(outputFilename, 'w') as outputFile: outputFile.write( json.dumps({ 'address': collection.address, 'name': collection.name, 'symbol': collection.symbol, 'description': collection.description, 'imageUrl': collection.imageUrl, 'twitterUsername': collection.twitterUsername, 'instagramUsername': collection.instagramUsername, 'wikiUrl': collection.wikiUrl, 'openseaSlug': collection.openseaSlug, 'url': collection.url, 'discordUrl': collection.discordUrl, 'bannerImageUrl': collection.bannerImageUrl, 'doesSupportErc721': collection.doesSupportErc721, 'doesSupportErc1155': collection.doesSupportErc1155, 'tokens': [{ 'registryAddress': token.registryAddress, 'tokenId': token.tokenId, 'metadataUrl': token.metadataUrl, 'name': token.name, 'description': token.description, 'imageUrl': token.imageUrl, 'animationUrl': token.animationUrl, 'youtubeUrl': token.youtubeUrl, 'backgroundColor': token.backgroundColor, 'frameImageUrl': token.frameImageUrl, 'attributes': token.attributes, } for token in tokens], })) await database.disconnect()
async def main(): requestIdHolder = RequestIdHolder() name = os.environ.get('NAME', 'mdtp-worker') version = os.environ.get('VERSION', 'local') environment = os.environ.get('ENV', 'dev') isRunningDebugMode = environment == 'dev' if isRunningDebugMode: logging.init_basic_logging() else: logging.init_json_logging(name=name, version=version, environment=environment, requestIdHolder=requestIdHolder) databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database=database) retriever = Retriever(database=database) workQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/mdtp-work-queue') s3Manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) # NOTE(krishan711): The AWS eth instance is much slower with getLogs so fails when calling ContractStore.get_latest_update_block_number # awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) # ethClient = RestEthClient(url='https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) requester = Requester() ethClient = RestEthClient(url=os.environ['ALCHEMY_MAINNET_URL'], requester=requester) rinkebyEthClient = RestEthClient(url=os.environ['ALCHEMY_URL'], requester=requester) mumbaiEthClient = RestEthClient( url='https://matic-mumbai.chainstacklabs.com', requester=requester) contractStore = create_contract_store(ethClient=ethClient, rinkebyEthClient=rinkebyEthClient, mumbaiEthClient=mumbaiEthClient) infuraIpfsAuth = BasicAuthentication( username=os.environ['INFURA_IPFS_PROJECT_ID'], password=os.environ['INFURA_IPFS_PROJECT_SECRET']) infuraIpfsRequester = Requester( headers={'authorization': f'Basic {infuraIpfsAuth.to_string()}'}) ipfsManager = IpfsManager(requester=infuraIpfsRequester) imageManager = ImageManager(requester=requester, s3Manager=s3Manager, ipfsManager=ipfsManager) manager = MdtpManager(requester=requester, retriever=retriever, saver=saver, s3Manager=s3Manager, contractStore=contractStore, workQueue=workQueue, imageManager=imageManager, ipfsManager=ipfsManager) processor = MdtpMessageProcessor(manager=manager) slackClient = SlackClient(webhookUrl=os.environ['SLACK_WEBHOOK_URL'], requester=requester, defaultSender='worker', defaultChannel='mdtp-notifications') messageQueueProcessor = MessageQueueProcessor(queue=workQueue, messageProcessor=processor, slackClient=slackClient) await database.connect() await s3Manager.connect() await workQueue.connect() await messageQueueProcessor.run() await requester.close_connections() await workQueue.disconnect() await s3Manager.disconnect() await database.disconnect()
async def reprocess_collections(startId: int, endId: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) saver = Saver(database) retriever = Retriever(database) openseaApiKey = os.environ['OPENSEA_API_KEY'] awsRequester = AwsRequester(accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) requester = Requester() # ethClient = RestEthClient(url=f'https://mainnet.infura.io/v3/{os.environ["INFURA_PROJECT_ID"]}', requester=requester) ethClient = RestEthClient( url= 'https://nd-foldvvlb25awde7kbqfvpgvrrm.ethereum.managedblockchain.eu-west-1.amazonaws.com', requester=awsRequester) s3manager = S3Manager(region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET']) requester = Requester() collectionProcessor = CollectionProcessor( requester=requester, ethClient=ethClient, openseaApiKey=openseaApiKey, s3manager=s3manager, bucketName=os.environ['S3_BUCKET']) tokenManager = TokenManager(saver=saver, retriever=retriever, tokenQueue=None, collectionProcessor=collectionProcessor, tokenMetadataProcessor=None) await database.connect() await s3manager.connect() currentId = startId while currentId < endId: start = currentId end = min(currentId + batchSize, endId) logging.info(f'Working on {start} to {end}...') async with database.transaction(): query = TokenCollectionsTable.select() query = query.where(TokenCollectionsTable.c.collectionId >= start) query = query.where(TokenCollectionsTable.c.collectionId < end) collectionsToChange = [ collection_from_row(row) async for row in database.iterate(query=query) ] logging.info(f'Updating {len(collectionsToChange)} collections...') for collection in collectionsToChange: logging.info(f'Updating collection: {collection.address}') try: await tokenManager.update_collection( address=collection.address, shouldForce=True) except Exception as e: logging.exception( f'Error processing {collection.collectionId}: {e}') currentId = currentId + batchSize await database.disconnect() await s3manager.disconnect() await requester.close_connections() await awsRequester.close_connections()
async def backfill_collection_activities(startBlock: int, endBlock: int, batchSize: int): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) retriever = Retriever(database=database) saver = Saver(database=database) tokenQueue = SqsMessageQueue( region='eu-west-1', accessKeyId=os.environ['AWS_KEY'], accessKeySecret=os.environ['AWS_SECRET'], queueUrl= 'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue') collectionActivityProcessor = CollectionActivityProcessor( retriever=retriever) tokenManager = TokenManager( saver=saver, retriever=retriever, tokenQueue=tokenQueue, collectionProcessor=None, tokenMetadataProcessor=None, tokenOwnershipProcessor=None, collectionActivityProcessor=collectionActivityProcessor) await database.connect() await tokenQueue.connect() currentBlockNumber = startBlock while currentBlockNumber < endBlock: endBlockNumber = min(currentBlockNumber + batchSize, endBlock) logging.info(f'Working on {currentBlockNumber} to {endBlockNumber}...') tokenTransfers = await retriever.list_token_transfers( fieldFilters=[ IntegerFieldFilter(BlocksTable.c.blockNumber.key, gte=currentBlockNumber), IntegerFieldFilter(BlocksTable.c.blockNumber.key, lte=endBlockNumber), ], orders=[ Order(fieldName=BlocksTable.c.blockDate.key, direction=Direction.ASCENDING) ], ) if len(tokenTransfers) == 0: print( f"Skipping {currentBlockNumber} to {endBlockNumber} with 0 transfers " ) else: collectionHourlyActivities = await retriever.list_collections_activity( fieldFilters=[ DateFieldFilter(CollectionHourlyActivityTable.c.date.key, gte=date_hour_from_datetime( tokenTransfers[0].blockDate)), DateFieldFilter(CollectionHourlyActivityTable.c.date.key, lte=date_hour_from_datetime( tokenTransfers[-1].blockDate)), ], ) processedPairs = { (collectionHourlyActivity.address, collectionHourlyActivity.date) for collectionHourlyActivity in collectionHourlyActivities } registryDatePairs = { (tokenTransfer.registryAddress, date_hour_from_datetime(tokenTransfer.blockDate)) for tokenTransfer in tokenTransfers if (tokenTransfer.registryAddress, date_hour_from_datetime(tokenTransfer.blockDate) ) not in processedPairs } print( f'Processing {len(registryDatePairs)} pairs from {len(tokenTransfers)} transfers' ) # messages = [UpdateActivityForCollectionMessageContent(address=address, startDate=startDate).to_message() for (address, startDate) in registryDatePairs] # await tokenQueue.send_messages(messages=messages) for pairChunk in list_util.generate_chunks( lst=list(registryDatePairs), chunkSize=50): await asyncio.gather(*[ tokenManager.update_activity_for_collection( address=registryAddress, startDate=startDate) for registryAddress, startDate in pairChunk ]) currentBlockNumber = endBlockNumber await database.disconnect() await tokenQueue.disconnect()