def get_listening_activity(stats_range: str):
""" Calculate the number of listens of users for the given stats_range """
logger.debug(f"Calculating listening_activity_{stats_range}")
from_date, to_date, step, date_format = get_time_range(stats_range)
time_range = []
period_start = from_date
while period_start < to_date:
period_formatted = period_start.strftime(date_format)
# calculate the time at which this period ends i.e. 1 microsecond before the next period's start
# here, period_start + step is next period's start
period_end = period_start + step + relativedelta(microseconds=-1)
time_range.append([period_formatted, period_start, period_end])
period_start = period_start + step
time_range_df = listenbrainz_spark.session.createDataFrame(
time_range, time_range_schema)
time_range_df.createOrReplaceTempView("time_range")
get_listens_from_new_dump(from_date,
to_date).createOrReplaceTempView("listens")
data = calculate_listening_activity()
messages = create_messages(data=data,
stats_range=stats_range,
from_date=from_date,
to_date=to_date)
logger.debug("Done!")
return messages
def get_daily_activity(stats_range: str) -> Iterator[Optional[UserDailyActivityStatMessage]]:
""" Calculate number of listens for an user for the specified time range """
logger.debug(f"Calculating daily_activity_{stats_range}")
from_date, to_date = get_dates_for_stats_range(stats_range)
get_listens_from_new_dump(from_date, to_date).createOrReplaceTempView("listens")
data = calculate_daily_activity()
messages = create_messages(data=data, stats_range=stats_range, from_date=from_date, to_date=to_date)
logger.debug("Done!")
return messages
def calculate_listens_per_day(year):
from_date = datetime(year, 1, 1)
to_date = datetime.combine(date(year, 12, 31), time.max)
step = relativedelta(days=+1)
date_format = "%d %B %Y"
time_range = []
segment_start = from_date
while segment_start < to_date:
segment_formatted = segment_start.strftime(date_format)
# calculate the time at which this period ends i.e. 1 microsecond before the next period's start
# here, segment_start + step is next segment's start
segment_end = segment_start + step + relativedelta(microseconds=-1)
time_range.append([segment_formatted, segment_start, segment_end])
segment_start = segment_start + step
time_range_df = listenbrainz_spark.session.createDataFrame(
time_range, time_range_schema)
time_range_df.createOrReplaceTempView("time_range")
listens = get_listens_from_new_dump(from_date, to_date)
listens.createOrReplaceTempView("listens")
data = calculate_listening_activity()
return create_messages(data=data,
stats_range="year_in_music",
from_date=from_date,
to_date=to_date,
message_type="year_in_music_listens_per_day")
def get_listening_activity(stats_range: str) -> Iterator[Optional[Dict]]:
""" Compute the number of listens for a time range compared to the previous range
Given a time range, this computes a histogram of all listens for that range
and the previous range of the same duration, so that they can be compared. The
bin size of the histogram depends on the size of the range (e.g.
year -> 12 months, month -> ~30 days, week -> ~7 days, see get_time_range for
details). These values are used on the listening activity reports.
"""
logger.debug(f"Calculating listening_activity_{stats_range}")
from_date, to_date, _, _, spark_date_format = setup_time_range(stats_range)
get_listens_from_new_dump(from_date,
to_date).createOrReplaceTempView("listens")
data = calculate_listening_activity(spark_date_format)
messages = create_messages(data=data,
stats_range=stats_range,
from_date=from_date,
to_date=to_date)
logger.debug("Done!")
return messages
def calculate_top_entity_stats(year):
from_date = datetime(year, 1, 1)
to_date = datetime.combine(date(year, 12, 31), time.max)
table = "listens_of_year"
listens = get_listens_from_new_dump(from_date, to_date)
listens.createOrReplaceTempView(table)
for entity in ["artists", "recordings", "releases"]:
stats = calculate_entity_stats(from_date, to_date, table, entity,
"this_year", "year_in_music_top_stats")
for message in stats:
yield message
def get_listening_activity(stats_range: str):
""" Compute the number of listens for a time range compared to the previous range
Given a time range, this computes a histogram of a users' listens for that range
and the previous range of the same duration, so that they can be compared. The
bin size of the histogram depends on the size of the range (e.g.
year -> 12 months, month -> ~30 days, week -> ~7 days, see get_time_range for
details). These values are used on the listening activity reports.
"""
logger.debug(f"Calculating listening_activity_{stats_range}")
from_date, to_date, step, date_format = get_time_range(stats_range)
time_range = []
period_start = from_date
while period_start < to_date:
period_formatted = period_start.strftime(date_format)
# calculate the time at which this period ends i.e. 1 microsecond before the next period's start
# here, period_start + step is next period's start
period_end = period_start + step + relativedelta(microseconds=-1)
time_range.append([period_formatted, period_start, period_end])
period_start = period_start + step
time_range_df = listenbrainz_spark.session.createDataFrame(
time_range, time_range_schema)
time_range_df.createOrReplaceTempView("time_range")
get_listens_from_new_dump(from_date,
to_date).createOrReplaceTempView("listens")
data = calculate_listening_activity()
messages = create_messages(data=data,
stats_range=stats_range,
from_date=from_date,
to_date=to_date)
logger.debug("Done!")
return messages
def get_entity_stats(entity: str, stats_range: str, message_type="user_entity")\
-> Iterator[Optional[Dict]]:
""" Get the top entity for all users for specified stats_range """
logger.debug(f"Calculating user_{entity}_{stats_range}...")
from_date, to_date = get_dates_for_stats_range(stats_range)
listens_df = get_listens_from_new_dump(from_date, to_date)
table = f"user_{entity}_{stats_range}"
listens_df.createOrReplaceTempView(table)
messages = calculate_entity_stats(from_date, to_date, table, entity,
stats_range, message_type)
logger.debug("Done!")
return messages
def get_entity_stats(entity: str, stats_range: str) -> Optional[List[SitewideEntityStatMessage]]:
""" Returns top entity stats for given time period """
logger.debug(f"Calculating sitewide_{entity}_{stats_range}...")
from_date, to_date = get_dates_for_stats_range(stats_range)
listens_df = get_listens_from_new_dump(from_date, to_date)
table_name = f"sitewide_{entity}_{stats_range}"
listens_df.createOrReplaceTempView(table_name)
listen_count_limit = get_listen_count_limit(stats_range)
handler = entity_handler_map[entity]
data = handler(table_name, listen_count_limit)
messages = create_messages(data=data, entity=entity, stats_range=stats_range,
from_date=from_date, to_date=to_date)
logger.debug("Done!")
return messages
def get_entity_stats(
entity: str,
stats_range: str) -> Iterator[Optional[UserEntityStatMessage]]:
""" Get the top entity for all users for specified stats_range """
logger.debug(f"Calculating user_{entity}_{stats_range}...")
from_date, to_date = get_dates_for_stats_range(stats_range)
listens_df = get_listens_from_new_dump(from_date, to_date)
table_name = f"user_{entity}_{stats_range}"
listens_df.createOrReplaceTempView(table_name)
handler = entity_handler_map[entity]
data = handler(table_name)
messages = create_messages(data=data,
entity=entity,
stats_range=stats_range,
from_date=from_date,
to_date=to_date)
logger.debug("Done!")
return messages
def setup_listens_for_year(year):
start = datetime(year, 1, 1)
end = datetime.combine(date(year, 12, 31), time.max)
listens = get_listens_from_new_dump(start, end)
listens.createOrReplaceTempView("listens_of_year")
def calculate_dataframes(from_date, to_date, job_type,
minimum_listens_threshold):
if job_type == "recommendation_recording":
paths = {
"mapped_listens": path.RECOMMENDATION_RECORDING_MAPPED_LISTENS,
"playcounts": path.RECOMMENDATION_RECORDING_PLAYCOUNTS_DATAFRAME,
"recordings": path.RECOMMENDATION_RECORDINGS_DATAFRAME,
"users": path.RECOMMENDATION_RECORDING_USERS_DATAFRAME,
"metadata": path.RECOMMENDATION_RECORDING_DATAFRAME_METADATA,
"prefix": "listenbrainz-dataframe-recording-recommendations"
}
elif job_type == "similar_users":
paths = {
"mapped_listens": path.USER_SIMILARITY_MAPPED_LISTENS,
"playcounts": path.USER_SIMILARITY_PLAYCOUNTS_DATAFRAME,
"recordings": path.USER_SIMILARITY_RECORDINGS_DATAFRAME,
"users": path.USER_SIMILARITY_USERS_DATAFRAME,
"metadata": path.USER_SIMILARITY_METADATA_DATAFRAME,
"prefix": "listenbrainz-dataframe-user-similarity"
}
else:
raise SparkException(
"Invalid job_type parameter received for creating dataframes: " +
job_type)
# dict to save dataframe metadata which would be later merged in model_metadata dataframe.
metadata = {}
# "updated" should always be set to False in this script.
metadata['updated'] = False
try:
listenbrainz_spark.init_spark_session('Create Dataframes')
except SparkSessionNotInitializedException as err:
logger.error(str(err), exc_info=True)
raise
metadata['to_date'] = to_date
metadata['from_date'] = from_date
complete_listens_df = get_listens_from_new_dump(from_date, to_date)
logger.info(
f'Listen count from {from_date} to {to_date}: {complete_listens_df.count()}'
)
logger.info('Discarding listens without mbids...')
partial_listens_df = complete_listens_df.where(
col('recording_mbid').isNotNull())
logger.info(f'Listen count after discarding: {partial_listens_df.count()}')
logger.info('Thresholding listens...')
threshold_listens_df = get_threshold_listens_df(partial_listens_df,
paths["mapped_listens"],
minimum_listens_threshold)
logger.info(
f'Listen count after thresholding: {threshold_listens_df.count()}')
logger.info('Preparing users data and saving to HDFS...')
users_df = get_users_dataframe(threshold_listens_df, metadata,
paths["users"])
logger.info('Preparing recordings data and saving to HDFS...')
recordings_df = get_recordings_df(threshold_listens_df, metadata,
paths["recordings"])
logger.info(
'Preparing listen data dump and playcounts, saving playcounts to HDFS...'
)
listens_df = get_listens_df(threshold_listens_df, metadata)
save_playcounts_df(listens_df, recordings_df, users_df, metadata,
paths["playcounts"])
metadata['dataframe_id'] = get_dataframe_id(paths["prefix"])
save_dataframe_metadata_to_hdfs(metadata, paths["metadata"])
return complete_listens_df
def get_all_test_listens(cls):
return get_listens_from_new_dump(cls.begin_date, cls.end_date)