def get_listening_activity_month() -> Iterator[Optional[UserListeningActivityStatMessage]]:
""" Calculate number of listens for an user on each day of the past month and current month. """
logger.debug("Calculating listening_activity_month")
to_date = get_latest_listen_ts()
from_date = offset_months(replace_days(to_date, 1), 1)
# Set time to 00:00
from_date = datetime(from_date.year, from_date.month, from_date.day)
day = from_date
# Genarate a dataframe containing days of last and current month along with start and end time
time_range = []
while day < to_date:
time_range.append([day.strftime('%d %B %Y'), day, get_day_end(day)])
day = offset_days(day, 1, shift_backwards=False)
time_range_df = listenbrainz_spark.session.createDataFrame(time_range, time_range_schema)
time_range_df.createOrReplaceTempView('time_range')
_get_listens(from_date, to_date)
data = get_listening_activity()
messages = create_messages(data=data, stats_range='month', from_ts=from_date.timestamp(), to_ts=to_date.timestamp())
logger.debug("Done!")
return messages
def test_get_listening_activity_month(self, mock_create_messages,
mock_get_listening_activity,
mock_get_listens,
mock_get_latest_listen_ts):
mock_df = MagicMock()
mock_get_listens.return_value = mock_df
listening_activity_stats.get_listening_activity_month()
to_date = datetime(2020, 6, 19)
from_date = day = datetime(2020, 5, 1)
time_range = []
while day < to_date:
time_range.append(
[day.strftime('%d %B %Y'), day,
get_day_end(day)])
day = offset_days(day, 1, shift_backwards=False)
time_range_df = run_query("SELECT * FROM time_range")
time_range_result = time_range_df.rdd.map(list).collect()
self.assertListEqual(time_range_result, time_range)
mock_get_latest_listen_ts.assert_called_once()
mock_get_listens.assert_called_with(from_date, to_date,
LISTENBRAINZ_DATA_DIRECTORY)
mock_df.createOrReplaceTempView.assert_called_with('listens')
mock_create_messages.assert_called_with(
data='listening_activity_table',
stats_range='month',
from_ts=from_date.timestamp(),
to_ts=to_date.timestamp())
def test_get_entity_month(self, mock_create_message, mock_get_listens,
mock_get_latest_listen_ts):
mock_df = MagicMock()
mock_get_listens.return_value = mock_df
entity_stats.get_entity_month('test', False)
from_date = datetime(2020, 7, 1)
to_date = datetime(2020, 8, 21)
day = from_date
time_range = []
while day < to_date:
time_range.append([
day.strftime('%d %B %Y'),
int(day.timestamp()),
int(get_day_end(day).timestamp())
])
day = offset_days(day, 1, shift_backwards=False)
time_range_df = run_query("SELECT * FROM time_range")
time_range_result = time_range_df.rdd.map(list).collect()
self.assertListEqual(time_range_result, time_range)
mock_get_latest_listen_ts.assert_called_once()
mock_get_listens.assert_called_with(from_date, to_date,
LISTENBRAINZ_DATA_DIRECTORY)
mock_df.createOrReplaceTempView.assert_called_with(
'sitewide_test_month')
mock_create_message.assert_called_with(data='sitewide_test_month_data',
entity='test',
stats_range='month',
from_ts=from_date.timestamp(),
to_ts=to_date.timestamp())
def get_entity_week(
entity: str,
use_mapping: bool = False
) -> Optional[List[SitewideEntityStatMessage]]:
""" Get the weekly sitewide top entity """
current_app.logger.debug("Calculating sitewide_{}_week...".format(entity))
date = get_latest_listen_ts()
to_date = get_last_monday(date)
# Set time to 00:00
to_date = datetime(to_date.year, to_date.month, to_date.day)
from_date = offset_days(to_date, 14)
day = from_date
# Genarate a dataframe containing days of last and current week along with start and end time
time_range = []
while day < to_date:
time_range.append([
day.strftime('%A %d %B %Y'),
int(day.timestamp()),
int(get_day_end(day).timestamp())
])
day = offset_days(day, 1, shift_backwards=False)
time_range_df = listenbrainz_spark.session.createDataFrame(
time_range, schema=time_range_schema)
time_range_df.createOrReplaceTempView('time_range')
listens_df = get_listens(from_date, to_date, LISTENBRAINZ_DATA_DIRECTORY)
filtered_df = filter_listens(listens_df, from_date, to_date)
table_name = 'sitewide_{}_week'.format(entity)
filtered_df.createOrReplaceTempView(table_name)
handler = entity_handler_map[entity]
data = handler(table_name, "EEEE dd MMMM yyyy", use_mapping)
message = create_message(data=data,
entity=entity,
stats_range='week',
from_ts=from_date.timestamp(),
to_ts=to_date.timestamp())
current_app.logger.debug("Done!")
return message
def test_get_listening_activity_month(self, mock_create_messages, _,
mock_get_listens):
listening_activity_stats.get_listening_activity('month')
from_date = day = datetime(2021, 6, 1)
to_date = datetime(2021, 8, 1)
time_range = []
while day < to_date:
time_range.append(
[day.strftime('%d %B %Y'), day,
get_day_end(day)])
day = offset_days(day, 1, shift_backwards=False)
time_range_df = run_query("SELECT * FROM time_range")
time_range_result = time_range_df.rdd.map(list).collect()
self.assertListEqual(time_range_result, time_range)
mock_get_listens.assert_called_with(from_date, to_date)
mock_create_messages.assert_called_with(data='activity_table',
stats_range='month',
from_date=from_date,
to_date=to_date)
def test_get_day_end(self):
day = datetime.datetime(2020, 6, 19)
self.assertEqual(datetime.datetime(2020, 6, 19, 23, 59, 59, 999999), stats.get_day_end(day))
