async def get_unclaimed_submissions(self) -> pd.DataFrame:
"""Get the submissions that are currently unclaimed in the queue."""
# Posts older than 18 hours are archived
queue_start = datetime.now(tz=pytz.utc) - timedelta(hours=18)
results = []
size = 500
page = 1
# Fetch all unclaimed posts from the queue
while True:
queue_response = self.blossom_api.get(
"submission/",
params={
"page_size": size,
"page": page,
"completed_by__isnull": True,
"claimed_by__isnull": True,
"archived": False,
"removed_from_queue": False,
"create_time__gte": queue_start.isoformat(),
},
)
if not queue_response.ok:
raise BlossomException(queue_response)
data = queue_response.json()["results"]
data = [fix_submission_source(entry) for entry in data]
results += data
page += 1
if len(data) < size:
break
data_frame = pd.DataFrame.from_records(data=results, index="id")
return data_frame
def calculate_history_offset(
self,
user: Optional[BlossomUser],
rate_data: pd.DataFrame,
after_time: Optional[datetime],
before_time: Optional[datetime],
) -> int:
"""Calculate the gamma offset for the history graph.
Note: We always need to do this, because it might be the case that some
transcriptions don't have a date set.
"""
gamma = get_user_gamma(user, self.blossom_api)
if before_time is not None:
# We need to get the offset from the API
offset_response = self.blossom_api.get(
"submission/",
params={
"completed_by__isnull": False,
"completed_by": get_user_id(user),
"complete_time__gte": before_time.isoformat(),
"page_size": 1,
},
)
if not offset_response.ok:
raise BlossomException(offset_response)
# We still need to calculate based on the total gamma
# It may be the case that not all transcriptions have a date set
# Then they are not included in the data nor in the API response
return gamma - rate_data.sum() - offset_response.json()["count"]
else:
# We can calculate the offset from the given data
return gamma - rate_data.sum()
async def _get_user_progress(
user: Optional[BlossomUser],
after_time: Optional[datetime],
before_time: Optional[datetime],
blossom_api: BlossomAPI,
) -> int:
"""Get the number of transcriptions made in the given time frame."""
from_str = after_time.isoformat() if after_time else None
until_str = before_time.isoformat() if before_time else None
# We ask for submission completed by the user in the time frame
# The response will contain a count, so we just need 1 result
progress_response = blossom_api.get(
"submission/",
params={
"completed_by": get_user_id(user),
"complete_time__gte": from_str,
"complete_time__lte": until_str,
"page_size": 1,
},
)
if progress_response.status_code != 200:
raise BlossomException(progress_response)
return progress_response.json()["count"]
async def _heatmap(
self,
ctx: SlashContext,
username: Optional[str] = "me",
after: Optional[str] = None,
before: Optional[str] = None,
) -> None:
"""Generate a heatmap for the given user."""
start = datetime.now()
after_time, before_time, time_str = parse_time_constraints(
after, before)
msg = await ctx.send(i18n["heatmap"]["getting_heatmap"].format(
user=get_initial_username(username, ctx), time_str=time_str))
utc_offset = extract_utc_offset(ctx.author.display_name)
from_str = after_time.isoformat() if after_time else None
until_str = before_time.isoformat() if before_time else None
user = get_user(username, ctx, self.blossom_api)
heatmap_response = self.blossom_api.get(
"submission/heatmap/",
params={
"completed_by": get_user_id(user),
"utc_offset": utc_offset,
"complete_time__gte": from_str,
"complete_time__lte": until_str,
},
)
if heatmap_response.status_code != 200:
raise BlossomException(heatmap_response)
data = heatmap_response.json()
day_index = pd.Index(range(1, 8))
hour_index = pd.Index(range(0, 24))
heatmap = (
# Create a data frame from the data
pd.DataFrame.from_records(data, columns=["day", "hour", "count"])
# Convert it into a table with the days as rows and hours as columns
.pivot(index="day", columns="hour", values="count")
# Add the missing days and hours
.reindex(index=day_index, columns=hour_index))
heatmap_table = create_file_from_heatmap(heatmap, user, utc_offset)
await msg.edit(
content=i18n["heatmap"]["response_message"].format(
user=get_username(user),
time_str=time_str,
duration=get_duration_str(start),
),
file=heatmap_table,
)
def get_all_rate_data(
self,
user: Optional[BlossomUser],
time_frame: str,
after_time: Optional[datetime],
before_time: Optional[datetime],
utc_offset: int,
) -> pd.DataFrame:
"""Get all rate data for the given user."""
page_size = 500
rate_data = pd.DataFrame(columns=["date", "count"]).set_index("date")
page = 1
# Placeholder until we get the real value from the response
next_page = "1"
from_str = after_time.isoformat() if after_time else None
until_str = before_time.isoformat() if before_time else None
while next_page is not None:
response = self.blossom_api.get(
"submission/rate",
params={
"completed_by": get_user_id(user),
"page": page,
"page_size": page_size,
"time_frame": time_frame,
"complete_time__gte": from_str,
"complete_time__lte": until_str,
"utc_offset": utc_offset,
},
)
if response.status_code != 200:
raise BlossomException(response)
new_data = response.json()["results"]
next_page = response.json()["next"]
new_frame = pd.DataFrame.from_records(new_data)
# Convert date strings to datetime objects
new_frame["date"] = new_frame["date"].apply(
lambda x: parser.parse(x))
# Add the data to the list
rate_data = rate_data.append(new_frame.set_index("date"))
# Continue with the next page
page += 1
# Add the missing zero entries
rate_data = add_zero_rates(rate_data, time_frame, after_time,
before_time)
return rate_data
def update_user_cache(self) -> None:
"""Fetch the users from their IDs."""
user_cache = {}
for idx, submission in self.claimed.head(5).iterrows():
user_id = extract_user_id(submission["claimed_by"])
if user := self.user_cache.get(user_id):
# Take the user from the old cache, if available
user_cache[user_id] = user
user_response = self.blossom_api.get("volunteer", params={"id": user_id})
if not user_response.ok:
raise BlossomException(user_response)
user = user_response.json()["results"][0]
user_cache[user_id] = user
async def leaderboard(
self,
ctx: SlashContext,
username: str = "me",
after: Optional[str] = None,
before: Optional[str] = None,
) -> None:
"""Get the leaderboard for the given user."""
start = datetime.now(tz=pytz.utc)
after_time, before_time, time_str = parse_time_constraints(after, before)
# Send a first message to show that the bot is responsive.
# We will edit this message later with the actual content.
msg = await ctx.send(
i18n["leaderboard"]["getting_leaderboard"].format(
user=get_initial_username(username, ctx), time_str=time_str
)
)
user = get_user(username, ctx, self.blossom_api)
top_count = 5 if user else 15
context_count = 5
from_str = after_time.isoformat() if after_time else None
until_str = before_time.isoformat() if before_time else None
# Get the leaderboard data
leaderboard_response = self.blossom_api.get(
"submission/leaderboard",
params={
"user_id": get_user_id(user),
"top_count": top_count,
"below_count": context_count,
"above_count": context_count,
"complete_time__gte": from_str,
"complete_time__lte": until_str,
},
)
if leaderboard_response.status_code != 200:
raise BlossomException(leaderboard_response)
leaderboard = leaderboard_response.json()
# Extract needed data
top_users = leaderboard["top"]
above_users = leaderboard["above"]
lb_user = leaderboard["user"]
below_users = leaderboard["below"]
description = ""
# Only show the top users if they are not already included
top_user_limit = (
top_count + 1
if user is None
else above_users[0]["rank"]
if len(above_users) > 0
else lb_user["rank"]
)
# Show top users
for top_user in top_users[: top_user_limit - 1]:
description += format_leaderboard_user(top_user) + "\n"
rank = get_rank(top_users[0]["gamma"])
if user:
# Add separator if necessary
if top_user_limit > top_count + 1:
description += "...\n"
# Show users with more gamma than the current user
for above_user in above_users:
description += format_leaderboard_user(above_user) + "\n"
# Show the current user
description += "**" + format_leaderboard_user(lb_user) + "**\n"
# Show users with less gamma than the current user
for below_user in below_users:
description += format_leaderboard_user(below_user) + "\n"
rank = get_rank(user["gamma"])
time_frame = format_leaderboard_timeframe(after_time, before_time)
await msg.edit(
content=i18n["leaderboard"]["embed_message"].format(
user=get_username(user),
time_str=time_str,
duration=get_duration_str(start),
),
embed=Embed(
title=i18n["leaderboard"]["embed_title"].format(
user=get_username(user), time_frame=time_frame
),
description=description,
color=Colour.from_rgb(*get_rgb_from_hex(rank["color"])),
),
)
async def _search_from_cache(
self,
msg: SlashMessage,
start: datetime,
cache_item: SearchCacheItem,
page_mod: int,
) -> None:
"""Execute the search with the given cache."""
# Clear previous control emojis
await clear_reactions(msg)
discord_page = cache_item["cur_page"] + page_mod
query = cache_item["query"]
user = cache_item["user"]
user_id = user["id"] if user else None
after_time = cache_item["after_time"]
before_time = cache_item["before_time"]
time_str = cache_item["time_str"]
from_str = after_time.isoformat() if after_time else None
until_str = before_time.isoformat() if before_time else None
request_page = (discord_page *
self.discord_page_size) // self.request_page_size
if (not cache_item["response_data"]
or request_page != cache_item["request_page"]):
# A new request has to be made
data = {
"text__icontains": cache_item["query"],
"author": user_id,
"create_time__gte": from_str,
"create_time__lte": until_str,
"url__isnull": False,
"ordering": "-create_time",
"page_size": self.request_page_size,
"page": request_page + 1,
}
response = self.blossom_api.get(path="transcription", params=data)
if response.status_code != 200:
raise BlossomException(response)
response_data = response.json()
else:
response_data = cache_item["response_data"]
if response_data["count"] == 0:
await msg.edit(content=i18n["search"]["no_results"].format(
query=query,
user=get_username(user),
time_str=time_str,
duration_str=get_duration_str(start),
))
return
# Only cache the result if the user can change pages
if response_data["count"] > self.discord_page_size:
# Update the cache
self.cache.set(
msg.id,
{
"query": query,
"user": cache_item["user"],
"after_time": after_time,
"before_time": before_time,
"time_str": time_str,
"cur_page": discord_page,
"discord_user_id": cache_item["discord_user_id"],
"response_data": response_data,
"request_page": request_page,
},
)
# Calculate the offset within the response
# The requested pages are larger than the pages displayed on Discord
request_offset = request_page * self.request_page_size
discord_offset = discord_page * self.discord_page_size
result_offset = discord_offset - request_offset
page_results: List[Dict[
str, Any]] = response_data["results"][result_offset:result_offset +
self.discord_page_size]
description = ""
for i, res in enumerate(page_results):
description += create_result_description(res,
discord_offset + i + 1,
query)
total_discord_pages = math.ceil(response_data["count"] /
self.discord_page_size)
await msg.edit(
content=i18n["search"]["embed_message"].format(
query=query,
user=get_username(user),
time_str=time_str,
duration_str=get_duration_str(start),
),
embed=Embed(
title=i18n["search"]["embed_title"].format(
query=query, user=get_username(user)),
description=description,
).set_footer(text=i18n["search"]["embed_footer"].format(
cur_page=discord_page + 1,
total_pages=total_discord_pages,
total_results=response_data["count"],
), ),
)
emoji_controls = []
# Determine which controls are appropriate
if discord_page > 0:
emoji_controls.append(first_page_emoji)
emoji_controls.append(previous_page_emoji)
if discord_page < total_discord_pages - 1:
emoji_controls.append(next_page_emoji)
emoji_controls.append(last_page_emoji)
# Add control emojis to message
await asyncio.gather(
*[msg.add_reaction(emoji) for emoji in emoji_controls])