async def update_activity_for_all_collections(self) -> None:
# TODO(femi-ogunkola): Account for deleted transactions
collectionActivity = await self.retriever.list_collections_activity(
orders=[
Order(fieldName=CollectionHourlyActivityTable.c.date.key,
direction=Direction.DESCENDING)
],
limit=1)
if len(collectionActivity) > 0:
lastestProcessedDate = collectionActivity[0].date
else:
lastestProcessedDate = date_util.start_of_day()
newTokenTransfers = await self.retriever.list_token_transfers(
fieldFilters=[
DateFieldFilter(fieldName=BlocksTable.c.updatedDate.key,
gte=lastestProcessedDate)
])
registryDatePairs = {(tokenTransfer.registryAddress,
date_hour_from_datetime(tokenTransfer.blockDate))
for tokenTransfer in newTokenTransfers}
logging.info(
f'Scheduling processing for {len(registryDatePairs)} collection, hour pairs'
)
messages = [
UpdateActivityForCollectionMessageContent(
address=address, startDate=startDate).to_message()
for (address, startDate) in registryDatePairs
]
await self.tokenQueue.send_messages(messages=messages)
async def update_activity_for_collection_deferred(
self, address: str, startDate: datetime.datetime) -> None:
address = chain_util.normalize_address(address)
startDate = date_hour_from_datetime(startDate)
await self.tokenQueue.send_message(
message=UpdateActivityForCollectionMessageContent(
address=address, startDate=startDate).to_message())
async def update_activity_for_collection(
self, address: str, startDate: datetime.datetime) -> None:
address = chain_util.normalize_address(address)
startDate = date_hour_from_datetime(startDate)
retrievedCollectionActivity = await self.collectionActivityProcessor.calculate_collection_hourly_activity(
address=address, startDate=startDate)
async with self.saver.create_transaction() as connection:
collectionActivity = await self.retriever.list_collections_activity(
connection=connection,
fieldFilters=[
StringFieldFilter(
fieldName=CollectionHourlyActivityTable.c.address.key,
eq=address),
DateFieldFilter(
fieldName=CollectionHourlyActivityTable.c.date.key,
eq=startDate)
])
if len(collectionActivity) > 0:
await self.saver.update_collection_hourly_activity(
connection=connection,
collectionActivityId=collectionActivity[0].
collectionActivityId,
address=address,
date=retrievedCollectionActivity.date,
transferCount=retrievedCollectionActivity.transferCount,
saleCount=retrievedCollectionActivity.saleCount,
totalValue=retrievedCollectionActivity.totalValue,
minimumValue=retrievedCollectionActivity.minimumValue,
maximumValue=retrievedCollectionActivity.maximumValue,
averageValue=retrievedCollectionActivity.averageValue,
)
else:
if retrievedCollectionActivity.transferCount == 0:
logging.info(
f'Not creating activity with transferCount==0')
else:
await self.saver.create_collection_hourly_activity(
connection=connection,
address=retrievedCollectionActivity.address,
date=retrievedCollectionActivity.date,
transferCount=retrievedCollectionActivity.
transferCount,
saleCount=retrievedCollectionActivity.saleCount,
totalValue=retrievedCollectionActivity.totalValue,
minimumValue=retrievedCollectionActivity.minimumValue,
maximumValue=retrievedCollectionActivity.maximumValue,
averageValue=retrievedCollectionActivity.averageValue,
)
async def calculate_collection_hourly_activity(
self, address: str,
startDate: datetime.datetime) -> RetrievedCollectionHourlyActivity:
address = chain_util.normalize_address(address)
startDate = date_hour_from_datetime(startDate)
tokenTransfers = await self.retriever.list_token_transfers(
fieldFilters=[
StringFieldFilter(TokenTransfersTable.c.registryAddress.key,
eq=address),
DateFieldFilter(BlocksTable.c.blockDate.key, gte=startDate),
DateFieldFilter(BlocksTable.c.blockDate.key,
lt=date_util.datetime_from_datetime(
dt=startDate, hours=1)),
], )
saleCount = 0
transferCount = 0
totalValue = 0
averageValue = 0
minimumValue = 0
maximumValue = 0
for tokenTransfer in tokenTransfers:
if tokenTransfer.value > 0:
saleCount += tokenTransfer.amount
totalValue += tokenTransfer.value
minimumValue = min(
minimumValue, tokenTransfer.value
) if minimumValue > 0 else tokenTransfer.value
maximumValue = max(maximumValue, tokenTransfer.value)
transferCount += tokenTransfer.amount
averageValue = totalValue / saleCount if saleCount > 0 else 0
return RetrievedCollectionHourlyActivity(address=address,
date=startDate,
transferCount=transferCount,
saleCount=saleCount,
totalValue=totalValue,
minimumValue=minimumValue,
maximumValue=maximumValue,
averageValue=averageValue)
async def backfill_collection_activities(startBlock: int, endBlock: int,
batchSize: int):
databaseConnectionString = Database.create_psql_connection_string(
username=os.environ["DB_USERNAME"],
password=os.environ["DB_PASSWORD"],
host=os.environ["DB_HOST"],
port=os.environ["DB_PORT"],
name=os.environ["DB_NAME"])
database = Database(connectionString=databaseConnectionString)
retriever = Retriever(database=database)
saver = Saver(database=database)
tokenQueue = SqsMessageQueue(
region='eu-west-1',
accessKeyId=os.environ['AWS_KEY'],
accessKeySecret=os.environ['AWS_SECRET'],
queueUrl=
'https://sqs.eu-west-1.amazonaws.com/097520841056/notd-token-queue')
collectionActivityProcessor = CollectionActivityProcessor(
retriever=retriever)
tokenManager = TokenManager(
saver=saver,
retriever=retriever,
tokenQueue=tokenQueue,
collectionProcessor=None,
tokenMetadataProcessor=None,
tokenOwnershipProcessor=None,
collectionActivityProcessor=collectionActivityProcessor)
await database.connect()
await tokenQueue.connect()
currentBlockNumber = startBlock
while currentBlockNumber < endBlock:
endBlockNumber = min(currentBlockNumber + batchSize, endBlock)
logging.info(f'Working on {currentBlockNumber} to {endBlockNumber}...')
tokenTransfers = await retriever.list_token_transfers(
fieldFilters=[
IntegerFieldFilter(BlocksTable.c.blockNumber.key,
gte=currentBlockNumber),
IntegerFieldFilter(BlocksTable.c.blockNumber.key,
lte=endBlockNumber),
],
orders=[
Order(fieldName=BlocksTable.c.blockDate.key,
direction=Direction.ASCENDING)
],
)
if len(tokenTransfers) == 0:
print(
f"Skipping {currentBlockNumber} to {endBlockNumber} with 0 transfers "
)
else:
collectionHourlyActivities = await retriever.list_collections_activity(
fieldFilters=[
DateFieldFilter(CollectionHourlyActivityTable.c.date.key,
gte=date_hour_from_datetime(
tokenTransfers[0].blockDate)),
DateFieldFilter(CollectionHourlyActivityTable.c.date.key,
lte=date_hour_from_datetime(
tokenTransfers[-1].blockDate)),
], )
processedPairs = {
(collectionHourlyActivity.address,
collectionHourlyActivity.date)
for collectionHourlyActivity in collectionHourlyActivities
}
registryDatePairs = {
(tokenTransfer.registryAddress,
date_hour_from_datetime(tokenTransfer.blockDate))
for tokenTransfer in tokenTransfers
if (tokenTransfer.registryAddress,
date_hour_from_datetime(tokenTransfer.blockDate)
) not in processedPairs
}
print(
f'Processing {len(registryDatePairs)} pairs from {len(tokenTransfers)} transfers'
)
# messages = [UpdateActivityForCollectionMessageContent(address=address, startDate=startDate).to_message() for (address, startDate) in registryDatePairs]
# await tokenQueue.send_messages(messages=messages)
for pairChunk in list_util.generate_chunks(
lst=list(registryDatePairs), chunkSize=50):
await asyncio.gather(*[
tokenManager.update_activity_for_collection(
address=registryAddress, startDate=startDate)
for registryAddress, startDate in pairChunk
])
currentBlockNumber = endBlockNumber
await database.disconnect()
await tokenQueue.disconnect()