async def _get_token_metadata_by_registry_address_token_id( self, registryAddress: str, tokenId: str, shouldProcessIfNotFound: bool = True, sleepSecondsBeforeProcess: float = 0) -> TokenMetadata: registryAddress = chain_util.normalize_address(value=registryAddress) try: tokenMetadata = await self.retriever.get_token_metadata_by_registry_address_token_id( registryAddress=registryAddress, tokenId=tokenId) except NotFoundException: if not shouldProcessIfNotFound: raise await asyncio.sleep(sleepSecondsBeforeProcess) await self.update_token_metadata(registryAddress=registryAddress, tokenId=tokenId, shouldForce=True) tokenMetadata = await self.retriever.get_token_metadata_by_registry_address_token_id( registryAddress=registryAddress, tokenId=tokenId) return tokenMetadata
async def get_collection_token_recent_transfers( self, registryAddress: str, tokenId: str, limit: int, offset: int) -> List[TokenTransfer]: registryAddress = chain_util.normalize_address(value=registryAddress) tokenTransfers = await self.retriever.list_token_transfers( fieldFilters=[ StringFieldFilter( fieldName=TokenTransfersTable.c.registryAddress.key, eq=registryAddress), StringFieldFilter(fieldName=TokenTransfersTable.c.tokenId.key, eq=tokenId), ], orders=[ Order(fieldName=TokenTransfersTable.c.blockNumber.key, direction=Direction.DESCENDING) ], limit=limit, offset=offset, ) return tokenTransfers
async def update_token_metadata(self, registryAddress: str, tokenId: str, shouldForce: bool = False) -> None: registryAddress = chain_util.normalize_address(value=registryAddress) if not shouldForce: recentlyUpdatedTokens = await self.retriever.list_token_metadatas( fieldFilters=[ StringFieldFilter( fieldName=TokenMetadatasTable.c.registryAddress.key, eq=registryAddress), StringFieldFilter( fieldName=TokenMetadatasTable.c.tokenId.key, eq=tokenId), DateFieldFilter( fieldName=TokenMetadatasTable.c.updatedDate.key, gt=date_util.datetime_from_now( days=-_TOKEN_UPDATE_MIN_DAYS)) ], ) if len(recentlyUpdatedTokens) > 0: logging.info( 'Skipping token because it has been updated recently.') return collection = await self._get_collection_by_address( address=registryAddress, shouldProcessIfNotFound=True, sleepSecondsBeforeProcess=0.1 * random.randint(1, 10)) try: retrievedTokenMetadata = await self.tokenMetadataProcessor.retrieve_token_metadata( registryAddress=registryAddress, tokenId=tokenId, collection=collection) except (TokenDoesNotExistException, TokenHasNoMetadataException): logging.info( f'Failed to retrieve metadata for token: {registryAddress}: {tokenId}' ) retrievedTokenMetadata = TokenMetadataProcessor.get_default_token_metadata( registryAddress=registryAddress, tokenId=tokenId) await self.save_token_metadata( retrievedTokenMetadata=retrievedTokenMetadata)
async def list_collection_tokens_by_owner( self, address: str, ownerAddress: str) -> List[Token]: address = chain_util.normalize_address(value=address) collection = await self.get_collection_by_address(address=address) tokens = [] if collection.doesSupportErc721: tokenOwnerships = await self.retriever.list_token_ownerships( fieldFilters=[ StringFieldFilter( fieldName=TokenOwnershipsTable.c.registryAddress.key, eq=address), StringFieldFilter( fieldName=TokenOwnershipsTable.c.ownerAddress.key, eq=ownerAddress), ]) tokens += [ Token(registryAddress=tokenOwnership.registryAddress, tokenId=tokenOwnership.tokenId) for tokenOwnership in tokenOwnerships ] elif collection.doesSupportErc1155: tokenMultiOwnerships = await self.retriever.list_token_multi_ownerships( fieldFilters=[ StringFieldFilter(fieldName=TokenMultiOwnershipsTable.c. registryAddress.key, eq=address), StringFieldFilter( fieldName=TokenMultiOwnershipsTable.c.ownerAddress.key, eq=ownerAddress), StringFieldFilter( fieldName=TokenMultiOwnershipsTable.c.quantity.key, eq=0), ]) tokens += [ Token(registryAddress=tokenMultiOwnership.registryAddress, tokenId=tokenMultiOwnership.tokenId) for tokenMultiOwnership in tokenMultiOwnerships ] return tokens
async def calculate_collection_hourly_activity( self, address: str, startDate: datetime.datetime) -> RetrievedCollectionHourlyActivity: address = chain_util.normalize_address(address) startDate = date_hour_from_datetime(startDate) tokenTransfers = await self.retriever.list_token_transfers( fieldFilters=[ StringFieldFilter(TokenTransfersTable.c.registryAddress.key, eq=address), DateFieldFilter(BlocksTable.c.blockDate.key, gte=startDate), DateFieldFilter(BlocksTable.c.blockDate.key, lt=date_util.datetime_from_datetime( dt=startDate, hours=1)), ], ) saleCount = 0 transferCount = 0 totalValue = 0 averageValue = 0 minimumValue = 0 maximumValue = 0 for tokenTransfer in tokenTransfers: if tokenTransfer.value > 0: saleCount += tokenTransfer.amount totalValue += tokenTransfer.value minimumValue = min( minimumValue, tokenTransfer.value ) if minimumValue > 0 else tokenTransfer.value maximumValue = max(maximumValue, tokenTransfer.value) transferCount += tokenTransfer.amount averageValue = totalValue / saleCount if saleCount > 0 else 0 return RetrievedCollectionHourlyActivity(address=address, date=startDate, transferCount=transferCount, saleCount=saleCount, totalValue=totalValue, minimumValue=minimumValue, maximumValue=maximumValue, averageValue=averageValue)
async def update_collection_deferred(self, address: str, shouldForce: bool = False) -> None: address = chain_util.normalize_address(value=address) if not shouldForce: recentlyUpdatedCollections = await self.retriever.list_collections( fieldFilters=[ StringFieldFilter( fieldName=TokenCollectionsTable.c.address.key, eq=address), DateFieldFilter( fieldName=TokenCollectionsTable.c.updatedDate.key, gt=date_util.datetime_from_now( days=-_COLLECTION_UPDATE_MIN_DAYS)) ], ) if len(recentlyUpdatedCollections) > 0: logging.info( 'Skipping collection because it has been updated recently.' ) return await self.tokenQueue.send_message( message=UpdateCollectionMessageContent( address=address).to_message())
async def _update_token_single_ownership(self, registryAddress: str, tokenId: str) -> None: registryAddress = chain_util.normalize_address(value=registryAddress) async with self.saver.create_transaction() as connection: try: tokenOwnership = await self.retriever.get_token_ownership_by_registry_address_token_id( connection=connection, registryAddress=registryAddress, tokenId=tokenId) except NotFoundException: tokenOwnership = None try: retrievedTokenOwnership = await self.tokenOwnershipProcessor.calculate_token_single_ownership( registryAddress=registryAddress, tokenId=tokenId) except NoOwnershipException: logging.error( f'No ownership found for {registryAddress}:{tokenId}') return if tokenOwnership: await self.saver.update_token_ownership( connection=connection, tokenOwnershipId=tokenOwnership.tokenOwnershipId, ownerAddress=retrievedTokenOwnership.ownerAddress, transferDate=retrievedTokenOwnership.transferDate, transferValue=retrievedTokenOwnership.transferValue, transferTransactionHash=retrievedTokenOwnership. transferTransactionHash) else: await self.saver.create_token_ownership( connection=connection, registryAddress=retrievedTokenOwnership.registryAddress, tokenId=retrievedTokenOwnership.tokenId, ownerAddress=retrievedTokenOwnership.ownerAddress, transferDate=retrievedTokenOwnership.transferDate, transferValue=retrievedTokenOwnership.transferValue, transferTransactionHash=retrievedTokenOwnership. transferTransactionHash)
async def get_collection_by_address(self, address: str) -> Collection: address = chain_util.normalize_address(value=address) return await self._get_collection_by_address( address=address, shouldProcessIfNotFound=True)
async def _update_token_multi_ownership(self, registryAddress: str, tokenId: str) -> None: registryAddress = chain_util.normalize_address(value=registryAddress) latestTransfers = await self.retriever.list_token_transfers( fieldFilters=[ StringFieldFilter( fieldName=TokenTransfersTable.c.registryAddress.key, eq=registryAddress), StringFieldFilter(fieldName=TokenTransfersTable.c.tokenId.key, eq=tokenId), ], orders=[ Order(fieldName=BlocksTable.c.updatedDate.key, direction=Direction.DESCENDING) ], limit=1) if len(latestTransfers) == 0: return latestTransfer = latestTransfers[0] latestOwnerships = await self.retriever.list_token_multi_ownerships( fieldFilters=[ StringFieldFilter( fieldName=TokenMultiOwnershipsTable.c.registryAddress.key, eq=registryAddress), StringFieldFilter( fieldName=TokenMultiOwnershipsTable.c.tokenId.key, eq=tokenId), ], orders=[ Order(fieldName=TokenMultiOwnershipsTable.c.updatedDate.key, direction=Direction.DESCENDING) ], limit=1) latestOwnership = latestOwnerships[0] if len( latestOwnerships) > 0 else None if latestOwnership is not None and latestOwnership.updatedDate > latestTransfer.updatedDate: logging.info( f'Skipping updating token_multi_ownership because last record ({latestOwnership.updatedDate}) is newer that last transfer update ({latestTransfer.updatedDate})' ) return retrievedTokenMultiOwnerships = await self.tokenOwnershipProcessor.calculate_token_multi_ownership( registryAddress=registryAddress, tokenId=tokenId) async with self.saver.create_transaction() as connection: currentTokenMultiOwnerships = await self.retriever.list_token_multi_ownerships( connection=connection, fieldFilters=[ StringFieldFilter(fieldName=TokenMultiOwnershipsTable.c. registryAddress.key, eq=registryAddress), StringFieldFilter( fieldName=TokenMultiOwnershipsTable.c.tokenId.key, eq=tokenId), ]) existingOwnershipTuplesMap = { self._uniqueness_tuple_from_token_multi_ownership( retrievedTokenMultiOwnership=tokenMultiOwnership): tokenMultiOwnership for tokenMultiOwnership in currentTokenMultiOwnerships } existingOwnershipTuples = set(existingOwnershipTuplesMap.keys()) retrievedOwnershipTuplesMaps = { self._uniqueness_tuple_from_token_multi_ownership( retrievedTokenMultiOwnership=retrievedTokenMultiOwnership): retrievedTokenMultiOwnership for retrievedTokenMultiOwnership in retrievedTokenMultiOwnerships } retrievedOwnershipTuples = set(retrievedOwnershipTuplesMaps.keys()) tokenMultiOwnershipIdsToDelete = [] for existingTuple, existingTokenMultiOwnership in existingOwnershipTuplesMap.items( ): if existingTuple in retrievedOwnershipTuples: continue tokenMultiOwnershipIdsToDelete.append( existingTokenMultiOwnership.tokenMultiOwnershipId) await self.saver.delete_token_multi_ownerships( connection=connection, tokenMultiOwnershipIds=tokenMultiOwnershipIdsToDelete) retrievedTokenMultiOwnershipsToSave = [] for retrievedTuple, retrievedTokenMultiOwnership in retrievedOwnershipTuplesMaps.items( ): if retrievedTuple in existingOwnershipTuples: continue retrievedTokenMultiOwnershipsToSave.append( retrievedTokenMultiOwnership) await self.saver.create_token_multi_ownerships( connection=connection, retrievedTokenMultiOwnerships= retrievedTokenMultiOwnershipsToSave) logging.info( f'Saving multi ownerships: saved {len(retrievedTokenMultiOwnershipsToSave)}, deleted {len(tokenMultiOwnershipIdsToDelete)}, kept {len(existingOwnershipTuples - retrievedOwnershipTuples) - len(tokenMultiOwnershipIdsToDelete)}' ) # NOTE(krishan711): if nothing changed, force update one so that it doesn't update again if len(existingOwnershipTuplesMap) > 0 and len( retrievedTokenMultiOwnershipsToSave) == 0 and len( tokenMultiOwnershipIdsToDelete) == 0: firstOwnership = list(existingOwnershipTuplesMap.values())[0] await self.saver.update_token_multi_ownership( connection=connection, tokenMultiOwnershipId=firstOwnership.tokenMultiOwnershipId, ownerAddress=firstOwnership.ownerAddress)
async def update_token_ownership_deferred(self, registryAddress: str, tokenId: str) -> None: registryAddress = chain_util.normalize_address(value=registryAddress) await self.tokenQueue.send_message( message=UpdateTokenOwnershipMessageContent( registryAddress=registryAddress, tokenId=tokenId).to_message())
async def update_collection(self, address: str, shouldForce: bool = False) -> None: address = chain_util.normalize_address(value=address) if not shouldForce: recentlyUpdatedCollections = await self.retriever.list_collections( fieldFilters=[ StringFieldFilter( fieldName=TokenCollectionsTable.c.address.key, eq=address), DateFieldFilter( fieldName=TokenCollectionsTable.c.updatedDate.key, gt=date_util.datetime_from_now( days=-_COLLECTION_UPDATE_MIN_DAYS)) ], ) if len(recentlyUpdatedCollections) > 0: logging.info( 'Skipping collection because it has been updated recently.' ) return try: retrievedCollection = await self.collectionProcessor.retrieve_collection( address=address) except CollectionDoesNotExist: logging.info( f'Failed to retrieve non-existant collection: {address}') return async with self.saver.create_transaction() as connection: try: collection = await self.retriever.get_collection_by_address( connection=connection, address=address) except NotFoundException: collection = None if collection: await self.saver.update_collection( connection=connection, collectionId=collection.collectionId, name=retrievedCollection.name, symbol=retrievedCollection.symbol, description=retrievedCollection.description, imageUrl=retrievedCollection.imageUrl, twitterUsername=retrievedCollection.twitterUsername, instagramUsername=retrievedCollection.instagramUsername, wikiUrl=retrievedCollection.wikiUrl, openseaSlug=retrievedCollection.openseaSlug, url=retrievedCollection.url, discordUrl=retrievedCollection.discordUrl, bannerImageUrl=retrievedCollection.bannerImageUrl, doesSupportErc721=retrievedCollection.doesSupportErc721, doesSupportErc1155=retrievedCollection.doesSupportErc1155) else: await self.saver.create_collection( connection=connection, address=address, name=retrievedCollection.name, symbol=retrievedCollection.symbol, description=retrievedCollection.description, imageUrl=retrievedCollection.imageUrl, twitterUsername=retrievedCollection.twitterUsername, instagramUsername=retrievedCollection.instagramUsername, wikiUrl=retrievedCollection.wikiUrl, openseaSlug=retrievedCollection.openseaSlug, url=retrievedCollection.url, discordUrl=retrievedCollection.discordUrl, bannerImageUrl=retrievedCollection.bannerImageUrl, doesSupportErc721=retrievedCollection.doesSupportErc721, doesSupportErc1155=retrievedCollection.doesSupportErc1155)
async def create_consolidated_metadata(address: str, outputFilename: str): databaseConnectionString = Database.create_psql_connection_string( username=os.environ["DB_USERNAME"], password=os.environ["DB_PASSWORD"], host=os.environ["DB_HOST"], port=os.environ["DB_PORT"], name=os.environ["DB_NAME"]) database = Database(connectionString=databaseConnectionString) retriever = Retriever(database=database) await database.connect() address = chain_util.normalize_address(value=address) collection = await retriever.get_collection_by_address(address=address) tokens = await retriever.list_token_metadatas(fieldFilters=[ StringFieldFilter(fieldName=TokenMetadatasTable.c.registryAddress.key, eq=address) ]) with open(outputFilename, 'w') as outputFile: outputFile.write( json.dumps({ 'address': collection.address, 'name': collection.name, 'symbol': collection.symbol, 'description': collection.description, 'imageUrl': collection.imageUrl, 'twitterUsername': collection.twitterUsername, 'instagramUsername': collection.instagramUsername, 'wikiUrl': collection.wikiUrl, 'openseaSlug': collection.openseaSlug, 'url': collection.url, 'discordUrl': collection.discordUrl, 'bannerImageUrl': collection.bannerImageUrl, 'doesSupportErc721': collection.doesSupportErc721, 'doesSupportErc1155': collection.doesSupportErc1155, 'tokens': [{ 'registryAddress': token.registryAddress, 'tokenId': token.tokenId, 'metadataUrl': token.metadataUrl, 'name': token.name, 'description': token.description, 'imageUrl': token.imageUrl, 'animationUrl': token.animationUrl, 'youtubeUrl': token.youtubeUrl, 'backgroundColor': token.backgroundColor, 'frameImageUrl': token.frameImageUrl, 'attributes': token.attributes, } for token in tokens], })) await database.disconnect()
async def get_collection_statistics(self, address: str) -> CollectionStatistics: address = chain_util.normalize_address(value=address) startDate = date_util.start_of_day() endDate = date_util.start_of_day( dt=date_util.datetime_from_datetime(dt=startDate, days=1)) itemCount = len( await self.retriever.list_token_metadatas(fieldFilters=[ StringFieldFilter( fieldName=TokenMetadatasTable.c.registryAddress.key, eq=address), ])) holderCount = len( await self.retriever.list_token_ownerships(fieldFilters=[ StringFieldFilter( fieldName=TokenOwnershipsTable.c.registryAddress.key, eq=address) ])) allCollectionActivities = await self.retriever.list_collections_activity( fieldFilters=[ StringFieldFilter( fieldName=CollectionHourlyActivityTable.c.address.key, eq=address) ]) totalTradeVolume = sum([ collectionActivity.totalValue for collectionActivity in allCollectionActivities ]) dayCollectionActivities = await self.retriever.list_collections_activity( fieldFilters=[ StringFieldFilter( fieldName=CollectionHourlyActivityTable.c.address.key, eq=address), DateFieldFilter( fieldName=CollectionHourlyActivityTable.c.date.key, gte=startDate, lt=endDate), ]) saleCount = 0 transferCount = 0 tradeVolume24Hours = 0 lowestSaleLast24Hours = 0 highestSaleLast24Hours = 0 for collectionActivity in dayCollectionActivities: saleCount += collectionActivity.saleCount tradeVolume24Hours += collectionActivity.totalValue lowestSaleLast24Hours = min( lowestSaleLast24Hours, collectionActivity.minimumValue ) if lowestSaleLast24Hours > 0 else collectionActivity.minimumValue highestSaleLast24Hours = max(highestSaleLast24Hours, collectionActivity.maximumValue) transferCount += collectionActivity.transferCount return CollectionStatistics( itemCount=itemCount, holderCount=holderCount, saleCount=saleCount, transferCount=transferCount, totalTradeVolume=totalTradeVolume, lowestSaleLast24Hours=lowestSaleLast24Hours, highestSaleLast24Hours=highestSaleLast24Hours, tradeVolume24Hours=tradeVolume24Hours, )