def get_entity_week(
entity: str,
use_mapping: bool = False
) -> Optional[List[SitewideEntityStatMessage]]:
""" Get the weekly sitewide top entity """
current_app.logger.debug("Calculating sitewide_{}_week...".format(entity))
date = get_latest_listen_ts()
to_date = get_last_monday(date)
# Set time to 00:00
to_date = datetime(to_date.year, to_date.month, to_date.day)
from_date = offset_days(to_date, 14)
day = from_date
# Genarate a dataframe containing days of last and current week along with start and end time
time_range = []
while day < to_date:
time_range.append([
day.strftime('%A %d %B %Y'),
int(day.timestamp()),
int(get_day_end(day).timestamp())
])
day = offset_days(day, 1, shift_backwards=False)
time_range_df = listenbrainz_spark.session.createDataFrame(
time_range, schema=time_range_schema)
time_range_df.createOrReplaceTempView('time_range')
listens_df = get_listens(from_date, to_date, LISTENBRAINZ_DATA_DIRECTORY)
filtered_df = filter_listens(listens_df, from_date, to_date)
table_name = 'sitewide_{}_week'.format(entity)
filtered_df.createOrReplaceTempView(table_name)
handler = entity_handler_map[entity]
data = handler(table_name, "EEEE dd MMMM yyyy", use_mapping)
message = create_message(data=data,
entity=entity,
stats_range='week',
from_ts=from_date.timestamp(),
to_ts=to_date.timestamp())
current_app.logger.debug("Done!")
return message
def get_entity_week(entity: str) -> Iterator[Optional[UserEntityStatMessage]]:
""" Get the weekly top entity for all users """
logger.debug("Calculating {}_week...".format(entity))
date = get_latest_listen_ts()
to_date = get_last_monday(date)
from_date = offset_days(to_date, 7)
listens_df = get_listens(from_date, to_date, LISTENBRAINZ_DATA_DIRECTORY)
filtered_df = filter_listens(listens_df, from_date, to_date)
table_name = 'user_{}_week'.format(entity)
filtered_df.createOrReplaceTempView(table_name)
handler = entity_handler_map[entity]
data = handler(table_name)
messages = create_messages(data=data, entity=entity, stats_range='week',
from_ts=from_date.timestamp(), to_ts=to_date.timestamp())
logger.debug("Done!")
return messages
def test_filter_listens(self):
from_date = datetime(2020, 5, 1)
to_date = datetime(2020, 5, 31)
df = utils.create_dataframe(
Row(listened_at=offset_months(from_date, 1)), None)
df = df.union(
utils.create_dataframe(
Row(listened_at=offset_months(
to_date, 1, shift_backwards=False)), None))
df = df.union(
utils.create_dataframe(
Row(listened_at=offset_days(
from_date, 5, shift_backwards=False)), None))
df = df.union(
utils.create_dataframe(Row(listened_at=offset_days(to_date, 5)),
None))
result = stats_utils.filter_listens(df, from_date, to_date)
rows = result.collect()
self.assertEqual(rows[0]['listened_at'],
offset_days(from_date, 5, shift_backwards=False))
self.assertEqual(rows[1]['listened_at'], offset_days(to_date, 5))
def _get_listens(from_date: datetime, to_date: datetime):
listens = get_listens(from_date, to_date, LISTENBRAINZ_DATA_DIRECTORY)
filtered_listens = filter_listens(listens, from_date, to_date)
filtered_listens.createOrReplaceTempView('listens')