async def _get_token_metadata_by_registry_address_token_id(
self,
registryAddress: str,
tokenId: str,
shouldProcessIfNotFound: bool = True,
sleepSecondsBeforeProcess: float = 0) -> TokenMetadata:
registryAddress = chain_util.normalize_address(value=registryAddress)
try:
tokenMetadata = await self.retriever.get_token_metadata_by_registry_address_token_id(
registryAddress=registryAddress, tokenId=tokenId)
except NotFoundException:
if not shouldProcessIfNotFound:
raise
await asyncio.sleep(sleepSecondsBeforeProcess)
await self.update_token_metadata(registryAddress=registryAddress,
tokenId=tokenId,
shouldForce=True)
tokenMetadata = await self.retriever.get_token_metadata_by_registry_address_token_id(
registryAddress=registryAddress, tokenId=tokenId)
return tokenMetadata
async def get_collection_token_recent_transfers(
self, registryAddress: str, tokenId: str, limit: int,
offset: int) -> List[TokenTransfer]:
registryAddress = chain_util.normalize_address(value=registryAddress)
tokenTransfers = await self.retriever.list_token_transfers(
fieldFilters=[
StringFieldFilter(
fieldName=TokenTransfersTable.c.registryAddress.key,
eq=registryAddress),
StringFieldFilter(fieldName=TokenTransfersTable.c.tokenId.key,
eq=tokenId),
],
orders=[
Order(fieldName=TokenTransfersTable.c.blockNumber.key,
direction=Direction.DESCENDING)
],
limit=limit,
offset=offset,
)
return tokenTransfers
async def update_token_metadata(self,
registryAddress: str,
tokenId: str,
shouldForce: bool = False) -> None:
registryAddress = chain_util.normalize_address(value=registryAddress)
if not shouldForce:
recentlyUpdatedTokens = await self.retriever.list_token_metadatas(
fieldFilters=[
StringFieldFilter(
fieldName=TokenMetadatasTable.c.registryAddress.key,
eq=registryAddress),
StringFieldFilter(
fieldName=TokenMetadatasTable.c.tokenId.key,
eq=tokenId),
DateFieldFilter(
fieldName=TokenMetadatasTable.c.updatedDate.key,
gt=date_util.datetime_from_now(
days=-_TOKEN_UPDATE_MIN_DAYS))
], )
if len(recentlyUpdatedTokens) > 0:
logging.info(
'Skipping token because it has been updated recently.')
return
collection = await self._get_collection_by_address(
address=registryAddress,
shouldProcessIfNotFound=True,
sleepSecondsBeforeProcess=0.1 * random.randint(1, 10))
try:
retrievedTokenMetadata = await self.tokenMetadataProcessor.retrieve_token_metadata(
registryAddress=registryAddress,
tokenId=tokenId,
collection=collection)
except (TokenDoesNotExistException, TokenHasNoMetadataException):
logging.info(
f'Failed to retrieve metadata for token: {registryAddress}: {tokenId}'
)
retrievedTokenMetadata = TokenMetadataProcessor.get_default_token_metadata(
registryAddress=registryAddress, tokenId=tokenId)
await self.save_token_metadata(
retrievedTokenMetadata=retrievedTokenMetadata)
async def list_collection_tokens_by_owner(
self, address: str, ownerAddress: str) -> List[Token]:
address = chain_util.normalize_address(value=address)
collection = await self.get_collection_by_address(address=address)
tokens = []
if collection.doesSupportErc721:
tokenOwnerships = await self.retriever.list_token_ownerships(
fieldFilters=[
StringFieldFilter(
fieldName=TokenOwnershipsTable.c.registryAddress.key,
eq=address),
StringFieldFilter(
fieldName=TokenOwnershipsTable.c.ownerAddress.key,
eq=ownerAddress),
])
tokens += [
Token(registryAddress=tokenOwnership.registryAddress,
tokenId=tokenOwnership.tokenId)
for tokenOwnership in tokenOwnerships
]
elif collection.doesSupportErc1155:
tokenMultiOwnerships = await self.retriever.list_token_multi_ownerships(
fieldFilters=[
StringFieldFilter(fieldName=TokenMultiOwnershipsTable.c.
registryAddress.key,
eq=address),
StringFieldFilter(
fieldName=TokenMultiOwnershipsTable.c.ownerAddress.key,
eq=ownerAddress),
StringFieldFilter(
fieldName=TokenMultiOwnershipsTable.c.quantity.key,
eq=0),
])
tokens += [
Token(registryAddress=tokenMultiOwnership.registryAddress,
tokenId=tokenMultiOwnership.tokenId)
for tokenMultiOwnership in tokenMultiOwnerships
]
return tokens
async def calculate_collection_hourly_activity(
self, address: str,
startDate: datetime.datetime) -> RetrievedCollectionHourlyActivity:
address = chain_util.normalize_address(address)
startDate = date_hour_from_datetime(startDate)
tokenTransfers = await self.retriever.list_token_transfers(
fieldFilters=[
StringFieldFilter(TokenTransfersTable.c.registryAddress.key,
eq=address),
DateFieldFilter(BlocksTable.c.blockDate.key, gte=startDate),
DateFieldFilter(BlocksTable.c.blockDate.key,
lt=date_util.datetime_from_datetime(
dt=startDate, hours=1)),
], )
saleCount = 0
transferCount = 0
totalValue = 0
averageValue = 0
minimumValue = 0
maximumValue = 0
for tokenTransfer in tokenTransfers:
if tokenTransfer.value > 0:
saleCount += tokenTransfer.amount
totalValue += tokenTransfer.value
minimumValue = min(
minimumValue, tokenTransfer.value
) if minimumValue > 0 else tokenTransfer.value
maximumValue = max(maximumValue, tokenTransfer.value)
transferCount += tokenTransfer.amount
averageValue = totalValue / saleCount if saleCount > 0 else 0
return RetrievedCollectionHourlyActivity(address=address,
date=startDate,
transferCount=transferCount,
saleCount=saleCount,
totalValue=totalValue,
minimumValue=minimumValue,
maximumValue=maximumValue,
averageValue=averageValue)
async def update_collection_deferred(self,
address: str,
shouldForce: bool = False) -> None:
address = chain_util.normalize_address(value=address)
if not shouldForce:
recentlyUpdatedCollections = await self.retriever.list_collections(
fieldFilters=[
StringFieldFilter(
fieldName=TokenCollectionsTable.c.address.key,
eq=address),
DateFieldFilter(
fieldName=TokenCollectionsTable.c.updatedDate.key,
gt=date_util.datetime_from_now(
days=-_COLLECTION_UPDATE_MIN_DAYS))
], )
if len(recentlyUpdatedCollections) > 0:
logging.info(
'Skipping collection because it has been updated recently.'
)
return
await self.tokenQueue.send_message(
message=UpdateCollectionMessageContent(
address=address).to_message())
async def _update_token_single_ownership(self, registryAddress: str,
tokenId: str) -> None:
registryAddress = chain_util.normalize_address(value=registryAddress)
async with self.saver.create_transaction() as connection:
try:
tokenOwnership = await self.retriever.get_token_ownership_by_registry_address_token_id(
connection=connection,
registryAddress=registryAddress,
tokenId=tokenId)
except NotFoundException:
tokenOwnership = None
try:
retrievedTokenOwnership = await self.tokenOwnershipProcessor.calculate_token_single_ownership(
registryAddress=registryAddress, tokenId=tokenId)
except NoOwnershipException:
logging.error(
f'No ownership found for {registryAddress}:{tokenId}')
return
if tokenOwnership:
await self.saver.update_token_ownership(
connection=connection,
tokenOwnershipId=tokenOwnership.tokenOwnershipId,
ownerAddress=retrievedTokenOwnership.ownerAddress,
transferDate=retrievedTokenOwnership.transferDate,
transferValue=retrievedTokenOwnership.transferValue,
transferTransactionHash=retrievedTokenOwnership.
transferTransactionHash)
else:
await self.saver.create_token_ownership(
connection=connection,
registryAddress=retrievedTokenOwnership.registryAddress,
tokenId=retrievedTokenOwnership.tokenId,
ownerAddress=retrievedTokenOwnership.ownerAddress,
transferDate=retrievedTokenOwnership.transferDate,
transferValue=retrievedTokenOwnership.transferValue,
transferTransactionHash=retrievedTokenOwnership.
transferTransactionHash)
async def get_collection_by_address(self, address: str) -> Collection:
address = chain_util.normalize_address(value=address)
return await self._get_collection_by_address(
address=address, shouldProcessIfNotFound=True)
async def _update_token_multi_ownership(self, registryAddress: str,
tokenId: str) -> None:
registryAddress = chain_util.normalize_address(value=registryAddress)
latestTransfers = await self.retriever.list_token_transfers(
fieldFilters=[
StringFieldFilter(
fieldName=TokenTransfersTable.c.registryAddress.key,
eq=registryAddress),
StringFieldFilter(fieldName=TokenTransfersTable.c.tokenId.key,
eq=tokenId),
],
orders=[
Order(fieldName=BlocksTable.c.updatedDate.key,
direction=Direction.DESCENDING)
],
limit=1)
if len(latestTransfers) == 0:
return
latestTransfer = latestTransfers[0]
latestOwnerships = await self.retriever.list_token_multi_ownerships(
fieldFilters=[
StringFieldFilter(
fieldName=TokenMultiOwnershipsTable.c.registryAddress.key,
eq=registryAddress),
StringFieldFilter(
fieldName=TokenMultiOwnershipsTable.c.tokenId.key,
eq=tokenId),
],
orders=[
Order(fieldName=TokenMultiOwnershipsTable.c.updatedDate.key,
direction=Direction.DESCENDING)
],
limit=1)
latestOwnership = latestOwnerships[0] if len(
latestOwnerships) > 0 else None
if latestOwnership is not None and latestOwnership.updatedDate > latestTransfer.updatedDate:
logging.info(
f'Skipping updating token_multi_ownership because last record ({latestOwnership.updatedDate}) is newer that last transfer update ({latestTransfer.updatedDate})'
)
return
retrievedTokenMultiOwnerships = await self.tokenOwnershipProcessor.calculate_token_multi_ownership(
registryAddress=registryAddress, tokenId=tokenId)
async with self.saver.create_transaction() as connection:
currentTokenMultiOwnerships = await self.retriever.list_token_multi_ownerships(
connection=connection,
fieldFilters=[
StringFieldFilter(fieldName=TokenMultiOwnershipsTable.c.
registryAddress.key,
eq=registryAddress),
StringFieldFilter(
fieldName=TokenMultiOwnershipsTable.c.tokenId.key,
eq=tokenId),
])
existingOwnershipTuplesMap = {
self._uniqueness_tuple_from_token_multi_ownership(
retrievedTokenMultiOwnership=tokenMultiOwnership):
tokenMultiOwnership
for tokenMultiOwnership in currentTokenMultiOwnerships
}
existingOwnershipTuples = set(existingOwnershipTuplesMap.keys())
retrievedOwnershipTuplesMaps = {
self._uniqueness_tuple_from_token_multi_ownership(
retrievedTokenMultiOwnership=retrievedTokenMultiOwnership):
retrievedTokenMultiOwnership
for retrievedTokenMultiOwnership in
retrievedTokenMultiOwnerships
}
retrievedOwnershipTuples = set(retrievedOwnershipTuplesMaps.keys())
tokenMultiOwnershipIdsToDelete = []
for existingTuple, existingTokenMultiOwnership in existingOwnershipTuplesMap.items(
):
if existingTuple in retrievedOwnershipTuples:
continue
tokenMultiOwnershipIdsToDelete.append(
existingTokenMultiOwnership.tokenMultiOwnershipId)
await self.saver.delete_token_multi_ownerships(
connection=connection,
tokenMultiOwnershipIds=tokenMultiOwnershipIdsToDelete)
retrievedTokenMultiOwnershipsToSave = []
for retrievedTuple, retrievedTokenMultiOwnership in retrievedOwnershipTuplesMaps.items(
):
if retrievedTuple in existingOwnershipTuples:
continue
retrievedTokenMultiOwnershipsToSave.append(
retrievedTokenMultiOwnership)
await self.saver.create_token_multi_ownerships(
connection=connection,
retrievedTokenMultiOwnerships=
retrievedTokenMultiOwnershipsToSave)
logging.info(
f'Saving multi ownerships: saved {len(retrievedTokenMultiOwnershipsToSave)}, deleted {len(tokenMultiOwnershipIdsToDelete)}, kept {len(existingOwnershipTuples - retrievedOwnershipTuples) - len(tokenMultiOwnershipIdsToDelete)}'
)
# NOTE(krishan711): if nothing changed, force update one so that it doesn't update again
if len(existingOwnershipTuplesMap) > 0 and len(
retrievedTokenMultiOwnershipsToSave) == 0 and len(
tokenMultiOwnershipIdsToDelete) == 0:
firstOwnership = list(existingOwnershipTuplesMap.values())[0]
await self.saver.update_token_multi_ownership(
connection=connection,
tokenMultiOwnershipId=firstOwnership.tokenMultiOwnershipId,
ownerAddress=firstOwnership.ownerAddress)
async def update_token_ownership_deferred(self, registryAddress: str,
tokenId: str) -> None:
registryAddress = chain_util.normalize_address(value=registryAddress)
await self.tokenQueue.send_message(
message=UpdateTokenOwnershipMessageContent(
registryAddress=registryAddress, tokenId=tokenId).to_message())
async def update_collection(self,
address: str,
shouldForce: bool = False) -> None:
address = chain_util.normalize_address(value=address)
if not shouldForce:
recentlyUpdatedCollections = await self.retriever.list_collections(
fieldFilters=[
StringFieldFilter(
fieldName=TokenCollectionsTable.c.address.key,
eq=address),
DateFieldFilter(
fieldName=TokenCollectionsTable.c.updatedDate.key,
gt=date_util.datetime_from_now(
days=-_COLLECTION_UPDATE_MIN_DAYS))
], )
if len(recentlyUpdatedCollections) > 0:
logging.info(
'Skipping collection because it has been updated recently.'
)
return
try:
retrievedCollection = await self.collectionProcessor.retrieve_collection(
address=address)
except CollectionDoesNotExist:
logging.info(
f'Failed to retrieve non-existant collection: {address}')
return
async with self.saver.create_transaction() as connection:
try:
collection = await self.retriever.get_collection_by_address(
connection=connection, address=address)
except NotFoundException:
collection = None
if collection:
await self.saver.update_collection(
connection=connection,
collectionId=collection.collectionId,
name=retrievedCollection.name,
symbol=retrievedCollection.symbol,
description=retrievedCollection.description,
imageUrl=retrievedCollection.imageUrl,
twitterUsername=retrievedCollection.twitterUsername,
instagramUsername=retrievedCollection.instagramUsername,
wikiUrl=retrievedCollection.wikiUrl,
openseaSlug=retrievedCollection.openseaSlug,
url=retrievedCollection.url,
discordUrl=retrievedCollection.discordUrl,
bannerImageUrl=retrievedCollection.bannerImageUrl,
doesSupportErc721=retrievedCollection.doesSupportErc721,
doesSupportErc1155=retrievedCollection.doesSupportErc1155)
else:
await self.saver.create_collection(
connection=connection,
address=address,
name=retrievedCollection.name,
symbol=retrievedCollection.symbol,
description=retrievedCollection.description,
imageUrl=retrievedCollection.imageUrl,
twitterUsername=retrievedCollection.twitterUsername,
instagramUsername=retrievedCollection.instagramUsername,
wikiUrl=retrievedCollection.wikiUrl,
openseaSlug=retrievedCollection.openseaSlug,
url=retrievedCollection.url,
discordUrl=retrievedCollection.discordUrl,
bannerImageUrl=retrievedCollection.bannerImageUrl,
doesSupportErc721=retrievedCollection.doesSupportErc721,
doesSupportErc1155=retrievedCollection.doesSupportErc1155)
async def create_consolidated_metadata(address: str, outputFilename: str):
databaseConnectionString = Database.create_psql_connection_string(
username=os.environ["DB_USERNAME"],
password=os.environ["DB_PASSWORD"],
host=os.environ["DB_HOST"],
port=os.environ["DB_PORT"],
name=os.environ["DB_NAME"])
database = Database(connectionString=databaseConnectionString)
retriever = Retriever(database=database)
await database.connect()
address = chain_util.normalize_address(value=address)
collection = await retriever.get_collection_by_address(address=address)
tokens = await retriever.list_token_metadatas(fieldFilters=[
StringFieldFilter(fieldName=TokenMetadatasTable.c.registryAddress.key,
eq=address)
])
with open(outputFilename, 'w') as outputFile:
outputFile.write(
json.dumps({
'address':
collection.address,
'name':
collection.name,
'symbol':
collection.symbol,
'description':
collection.description,
'imageUrl':
collection.imageUrl,
'twitterUsername':
collection.twitterUsername,
'instagramUsername':
collection.instagramUsername,
'wikiUrl':
collection.wikiUrl,
'openseaSlug':
collection.openseaSlug,
'url':
collection.url,
'discordUrl':
collection.discordUrl,
'bannerImageUrl':
collection.bannerImageUrl,
'doesSupportErc721':
collection.doesSupportErc721,
'doesSupportErc1155':
collection.doesSupportErc1155,
'tokens': [{
'registryAddress': token.registryAddress,
'tokenId': token.tokenId,
'metadataUrl': token.metadataUrl,
'name': token.name,
'description': token.description,
'imageUrl': token.imageUrl,
'animationUrl': token.animationUrl,
'youtubeUrl': token.youtubeUrl,
'backgroundColor': token.backgroundColor,
'frameImageUrl': token.frameImageUrl,
'attributes': token.attributes,
} for token in tokens],
}))
await database.disconnect()
async def get_collection_statistics(self,
address: str) -> CollectionStatistics:
address = chain_util.normalize_address(value=address)
startDate = date_util.start_of_day()
endDate = date_util.start_of_day(
dt=date_util.datetime_from_datetime(dt=startDate, days=1))
itemCount = len(
await self.retriever.list_token_metadatas(fieldFilters=[
StringFieldFilter(
fieldName=TokenMetadatasTable.c.registryAddress.key,
eq=address),
]))
holderCount = len(
await self.retriever.list_token_ownerships(fieldFilters=[
StringFieldFilter(
fieldName=TokenOwnershipsTable.c.registryAddress.key,
eq=address)
]))
allCollectionActivities = await self.retriever.list_collections_activity(
fieldFilters=[
StringFieldFilter(
fieldName=CollectionHourlyActivityTable.c.address.key,
eq=address)
])
totalTradeVolume = sum([
collectionActivity.totalValue
for collectionActivity in allCollectionActivities
])
dayCollectionActivities = await self.retriever.list_collections_activity(
fieldFilters=[
StringFieldFilter(
fieldName=CollectionHourlyActivityTable.c.address.key,
eq=address),
DateFieldFilter(
fieldName=CollectionHourlyActivityTable.c.date.key,
gte=startDate,
lt=endDate),
])
saleCount = 0
transferCount = 0
tradeVolume24Hours = 0
lowestSaleLast24Hours = 0
highestSaleLast24Hours = 0
for collectionActivity in dayCollectionActivities:
saleCount += collectionActivity.saleCount
tradeVolume24Hours += collectionActivity.totalValue
lowestSaleLast24Hours = min(
lowestSaleLast24Hours, collectionActivity.minimumValue
) if lowestSaleLast24Hours > 0 else collectionActivity.minimumValue
highestSaleLast24Hours = max(highestSaleLast24Hours,
collectionActivity.maximumValue)
transferCount += collectionActivity.transferCount
return CollectionStatistics(
itemCount=itemCount,
holderCount=holderCount,
saleCount=saleCount,
transferCount=transferCount,
totalTradeVolume=totalTradeVolume,
lowestSaleLast24Hours=lowestSaleLast24Hours,
highestSaleLast24Hours=highestSaleLast24Hours,
tradeVolume24Hours=tradeVolume24Hours,
)