def download(self):
"""
The function to download songs on the playlist
:return str is_downloaded: Flag that indicates songs are downloaded
"""
""" Generate youtube-dl option """
logger.info('Generating youtube-dl option...')
ydl_opts = None
if self.is_playlist and len(self.scheduled_queue_indices) > 0:
ydl_opts = self.ydl_helper.get_download_option(
download_dir=self.download_dir,
hook=self.__download_hook,
audio_codec=self.audio_codec,
audio_bitrate=self.audio_bitrate,
queue_indices=self.scheduled_queue_indices,
verbose=self.verbose)
logger.debug(pformat(ydl_opts))
elif not self.is_playlist:
ydl_opts = self.ydl_helper.get_download_option(
download_dir=self.download_dir,
hook=self.__download_hook,
audio_codec=self.audio_codec,
audio_bitrate=self.audio_bitrate,
verbose=self.verbose)
logger.debug(pformat(ydl_opts))
logger.info('Done.')
""" Download songs """
if ydl_opts:
logger.info('Downloading songs...')
try:
# Download playlist
self.ydl.__init__(params=ydl_opts)
self.downloaded_playlist_data = self.ydl.extract_info(
self.download_url, download=True)
# Save playlist data
with open(self.downloaded_playlist_file, 'w') as f:
json.dump(self.downloaded_playlist_data,
f,
indent=4,
ensure_ascii=False)
except Exception:
raise PlaylistDownloadException('Failed to download playlist.',
None)
if self.downloaded_playlist_data is None:
raise PlaylistDownloadException('Failed to download playlist.',
None)
logger.info('Done.')
return True
else:
logger.warning(
'All songs on the playlist are already downloaded. There is nothing to process.'
)
return False
def __merge_playlist(self, pl_data):
"""
The function that merges remote (head_playlist) and local playlist (base_playlist) and generate scheduled queue indices
:param dict pl_data: Playlist data contains downloaded songs
:rtype: (dict, list)
:return: (playlist, indices): (Merged playlist, Queue indices to download)
"""
head_playlist_data = pl_data # Playlist data downloaded from url
base_playlist_data = None # Playlist data previously saved on download directory
""" Load playlist previously saved """
if os.path.exists(self.playlist_file):
with open(self.playlist_file) as f:
# base_playlist_data = json.load(f, object_pairs_hook=OrderedDict)
base_playlist_data = json.load(f)
""" Merge Playlist """
candidate_queue_indices = []
candidate_queue_index = 1
# Playlist
if base_playlist_data:
# Copy list to avoid index shifting when elements are removed while iterating.
# https://stackoverflow.com/questions/1207406/how-to-remove-items-from-a-list-while-iterating
head_index = 0
head_entries = head_playlist_data['entries'][:]
for head_entry in head_entries:
# Delete entry if invalid.
if head_entry is None or head_entry.get(
'title', 'N/A').lower() in [
'[private video]', '[deleted video]'
]:
song_title = head_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
logger.error('[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1, len(head_playlist_data['entries']),
head_entry.get('id', 'N/A'), song_title,
'The video is private or deleted. Removed from the playlist.'
))
del head_playlist_data['entries'][head_index]
candidate_queue_index += 1
else:
# Copy list to avoid index shifting when elements are removed while iterating.
# https://stackoverflow.com/questions/1207406/how-to-remove-items-from-a-list-while-iterating
base_index = 0
base_entries = base_playlist_data['entries'][:]
for base_entry in base_entries:
# Delete entry if invalid.
if base_entry is None or base_entry.get(
'title', 'N/A').lower() in [
'[private video]', '[deleted video]'
]:
song_title = base_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
logger.error('[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1,
len(head_playlist_data['entries']),
base_entry.get('id', 'N/A'),
song_title,
'The video is private or deleted. Removed from the playlist.',
))
del base_playlist_data['entries'][base_index]
else:
# If same entry is found, update status
if head_entry['id'] == base_entry['id']:
# Merge base status into head status
base_entry_status = base_entry.get(
'status', YDLQueueStatus.ready.value)
head_playlist_data['entries'][head_index][
'status'] = base_entry_status
# Queue index is out of range requested
if not self.__is_queue_in_range(head_index):
song_title = base_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
logger.debug(
'[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1,
len(head_playlist_data['entries']),
base_entry.get('id', 'N/A'),
song_title,
'This queue is out of range requested. Skipped.',
))
# Song is already downloaded
elif base_entry_status == YDLQueueStatus.finished.value:
song_title = base_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
logger.warning(
'[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1,
len(head_playlist_data['entries']),
base_entry.get('id', 'N/A'),
song_title,
'This queue is already finished. Skipped.',
))
# Song is not downloaded yet
else:
song_title = base_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
logger.info(
'[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1,
len(head_playlist_data['entries']),
base_entry.get('id', 'N/A'),
song_title,
'This queue is not finished yet. Added to scheduled queues.',
))
# Delete entry to make iteration faster
del base_playlist_data['entries'][base_index]
break
base_index += 1
# Update track number
head_entry['track_number'] = head_index + 1
# Add queue
is_not_finished = head_entry.get(
'status', YDLQueueStatus.ready.value
) != YDLQueueStatus.finished.value
if self.__is_queue_in_range(
head_index) and is_not_finished:
candidate_queue_indices.append(candidate_queue_index)
# Add element to dictionary that maps index and entry_id
self.playlist_data_map[head_entry['id']] = head_index
candidate_queue_index += 1
head_index += 1
# Single song
else:
# Copy list to avoid index shifting when elements are removed while iterating.
# https://stackoverflow.com/questions/1207406/how-to-remove-items-from-a-list-while-iterating
head_index = 0
head_entries = head_playlist_data['entries'][:]
for head_entry in head_entries:
# Delete entry if invalid.
if head_entry is None or head_entry.get(
'title', 'N/A').lower() in [
'[private video]', '[deleted video]'
]:
song_title = head_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
logger.error('[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1, len(head_playlist_data['entries']),
head_entry.get('id', 'N/A'), song_title,
'The video is private or deleted. Removed from the playlist.'
))
del head_playlist_data['entries'][head_index]
candidate_queue_index += 1
else:
song_title = head_entry.get('title', None)
song_title = '[title:{}]'.format(
song_title) if song_title else ''
# Add queue
if self.__is_queue_in_range(head_index):
candidate_queue_indices.append(candidate_queue_index)
logger.info('[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1, len(head_playlist_data['entries']),
head_entry.get('id', 'N/A'), song_title,
'This queue is not finished yet. Added to scheduled queues.'
))
else:
logger.debug('[Playlist:{}/{}][ID:{}]{} {}'.format(
head_index + 1, len(head_playlist_data['entries']),
head_entry.get('id', 'N/A'), song_title,
'This queue is out of range requested. Skipped.'))
# Update value
head_entry['status'] = YDLQueueStatus.ready.value
# Update track number
head_entry['track_number'] = head_index + 1
# Add element to dictionary that maps index and entry_id
self.playlist_data_map[head_entry['id']] = head_index
candidate_queue_index += 1
head_index += 1
# Save playlist
with open(self.playlist_file, 'w') as file:
json.dump(head_playlist_data, file, indent=4, ensure_ascii=False)
self.playlist_data = head_playlist_data
return head_playlist_data, candidate_queue_indices
def preprocess(self, download_url, working_dir):
"""
:param str download_url: URL to download
:param str working_dir: Path to root directory
"""
self.download_url = download_url
""" Retrieve playlist """
logger.info('Retrieving playlist...')
logger.info('Download URL: {}'.format(self.download_url))
try:
ydl_opts = self.ydl_helper.get_preprocess_option(
download_url=self.download_url,
audio_codec=self.audio_codec,
audio_bitrate=self.audio_bitrate,
playlist_start=self.playlist_start,
playlist_end=self.playlist_end,
verbose=self.verbose,
)
logger.debug(pformat(ydl_opts))
self.ydl.__init__(params=ydl_opts)
# TODO: What is extra_info? Need investigation.
# self.playlist_data = self.ydl.extract_info(download_url, download=False, process=False, extra_info={})
self.playlist_data = self.ydl.extract_info(self.download_url,
download=False,
process=False)
except:
raise PlaylistPreprocessException('Could not retrieve playlist.',
None)
if self.playlist_data is None or self.ydl is None:
raise PlaylistPreprocessException('Could not retrieve playlist.',
None)
logger.info('Done.')
""" Validate playlist """
logger.info('Validating playlist...')
# Determines playlist type
playlist_extractor = self.playlist_data['extractor'].lower()
if playlist_extractor == 'youtube:playlist' or playlist_extractor == 'soundcloud:set':
self.is_playlist = True
# Define download folder name
if self.test_id is not None:
download_folder = self.test_id
else:
playlist_title = sanitize_filename(self.playlist_data['title'])
download_folder = '[{}] {}'.format(self.playlist_data['id'],
playlist_title)
self.download_dir = os.path.join(working_dir, download_folder)
elif playlist_extractor == 'youtube' or playlist_extractor == 'soundcloud':
self.is_playlist = False
# Define download folder name
if self.test_id is not None:
download_folder = self.test_id
else:
download_folder = self.folder_name
self.download_dir = os.path.join(working_dir, download_folder)
else:
raise PlaylistPreprocessException(
'This playlist is not supported.', self.playlist_data)
self.playlist_file = os.path.join(self.download_dir, '.queued.json')
self.downloaded_playlist_file = os.path.join(self.download_dir,
'.downloaded.json')
logger.debug(pformat(self.playlist_data))
logger.info('Done.')
""" Create directories """
logger.info('Creating download directory...')
# Download directory
os.makedirs(self.download_dir, exist_ok=True)
# Playlist
if self.clear_cache and os.path.exists(self.playlist_file):
os.remove(self.playlist_file)
if os.path.exists(self.downloaded_playlist_file):
os.remove(self.downloaded_playlist_file)
logger.info('Done.')
""" Process playlist """
logger.info('Processing playlist...')
if self.is_playlist:
# Convert generator object to list
self.playlist_data['entries'] = list(self.playlist_data['entries'])
self.playlist_entry_total = len(self.playlist_data['entries'])
# Merge playlist
merged_playlist_data, queue_indices = self.__merge_playlist(
self.playlist_data)
with open(self.playlist_file, 'w') as f:
json.dump(merged_playlist_data,
f,
indent=4,
ensure_ascii=False)
self.playlist_data = merged_playlist_data
self.scheduled_queue_indices = queue_indices
logger.debug(pformat(self.playlist_data))
logger.info('Done.')
return self.download_dir