def save_to_cache(search_term, video_id):
"""
Saves the search term and video id to the database cache so it can be looked up later.
:param search_term: Search term to be saved to in the cache
:param video_id: Video id to be saved to in the cache
:return Video id saved in the cache
"""
song_info, saved = Song.get_or_create(search_term=search_term, video_id=video_id)
log.debug(f"Saved: {saved} video id {song_info.video_id} in cache")
return song_info.video_id
def fetch_youtube_url(search_term):
"""For each song name/artist name combo, fetch the YouTube URL and return the list of URLs"""
YOUTUBE_DEV_KEY = getenv('YOUTUBE_DEV_KEY')
youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=YOUTUBE_DEV_KEY)
log.info(u"Searching for {}".format(search_term))
search_response = youtube.search().list(q=search_term, part='id, snippet').execute()
for v in search_response['items']:
if v['id']['kind'] == VIDEO:
log.debug("Adding Video id {}".format(v['id']['videoId']))
return YOUTUBE_VIDEO_URL + v['id']['videoId']
def check_if_in_cache(search_term):
"""
Checks if the specified search term is in the local database cache.
and returns the video id if it exists.
:param search_term: String to be searched for in the cache
:return A tuple with Boolean and video id if it exists
"""
try:
song = Song.get(search_term=search_term)
log.debug(f"Found id {song.video_id} for {search_term} in cache")
return True, song.video_id
except DoesNotExist:
log.debug(f"Couldn't find id for {search_term} in cache")
return False, None
def fetch_youtube_url(search_term):
"""For each song name/artist name combo, fetch the YouTube URL
and return the list of URLs"""
YOUTUBE_DEV_KEY = getenv('YOUTUBE_DEV_KEY')
youtube = build(YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=YOUTUBE_DEV_KEY)
log.info(u"Searching for {}".format(search_term))
search_response = youtube.search().list(q=search_term,
part='id, snippet').execute()
for v in search_response['items']:
if v['id']['kind'] == VIDEO:
log.debug("Adding Video id {}".format(v['id']['videoId']))
return YOUTUBE_VIDEO_URL + v['id']['videoId']
def validate_spotify_url(url):
"""
Validate the URL and determine if the item type is supported.
:return Boolean indicating whether or not item is supported
"""
item_type, item_id = parse_spotify_url(url)
log.debug(f"Got item type {item_type} and item_id {item_id}")
if item_type not in ['album', 'track', 'playlist']:
log.error("Only albums/tracks/playlists are supported")
return False
if item_id is None:
log.error("Couldn't get a valid id")
return False
return True
def validate_spotify_url(url):
"""
Validate the URL to determine if the item type is supported.
:return Boolean .
"""
type, id = parse_spotify_url(url)
log.debug(f" item type :{type} ; item_id: {id}")
if type not in ['album', 'track', 'playlist']:
log.error("Only albums/tracks/playlists are supported")
return False
if id is None:
log.error("Couldn't get a valid id")
return False
return True
def download_songs(songs, download_directory, format_string, skip_mp3):
"""
Downloads songs from the YouTube URL passed to either current directory or download_directory, is it is passed.
:param songs: Dictionary of songs and associated artist
:param download_directory: Location where to save
:param format_string: format string for the file conversion
:param skip_mp3: Whether to skip conversion to MP3
"""
log.debug(f"Downloading to {download_directory}")
for song, artist in songs.items():
query = f"{artist} - {song}".replace(":", "").replace("\"", "")
download_archive = path.join(download_directory,
'downloaded_songs.txt')
outtmpl = path.join(download_directory, '%(title)s.%(ext)s')
ydl_opts = {
'format':
format_string,
'download_archive':
download_archive,
'outtmpl':
outtmpl,
'default_search':
'ytsearch',
'noplaylist':
True,
'postprocessor_args':
['-metadata', 'title=' + song, '-metadata', 'artist=' + artist]
}
if not skip_mp3:
mp3_postprocess_opts = {
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}
ydl_opts['postprocessors'] = [mp3_postprocess_opts.copy()]
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
try:
ydl.download([query])
except Exception as e:
log.debug(e)
print(
'Failed to download: {}, please ensure YouTubeDL is up-to-date. '
.format(query))
continue
def fetch_youtube_url(search_term, dev_key):
"""For each song name/artist name combo, fetch the YouTube URL
and return the list of URLs"""
YOUTUBE_DEV_KEY = dev_key
youtube = build(YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=YOUTUBE_DEV_KEY,
cache_discovery=False)
log.info(u"Searching for {}".format(search_term))
try:
search_response = youtube.search().list(q=search_term,
part='id, snippet').execute()
for v in search_response['items']:
if v['id']['kind'] == VIDEO:
log.debug("Adding Video id {}".format(v['id']['videoId']))
return YOUTUBE_VIDEO_URL + v['id']['videoId']
except HttpError as err:
err_details = loads(
err.content.decode('utf-8')).get('error').get('errors')
secho("Couldn't complete search due to following errors: ", fg='red')
for e in err_details:
error_reason = e.get('reason')
error_domain = e.get('domain')
error_message = e.get('message')
if error_reason == 'quotaExceeded' or error_reason == 'dailyLimitExceeded':
secho(
f"\tYou're over daily allowed quota. Unfortunately, YouTube restricts API keys to a max of 10,000 requests per day which translates to a maximum of 100 searches.",
fg='red')
secho(
f"\tThe quota will be reset at midnight Pacific Time (PT).",
fg='red')
secho(
f"\tYou can request for Quota increase from https://console.developers.google.com/apis/api/youtube.googleapis.com/quotas.",
fg='red')
else:
secho(
f"\t Search failed due to {error_domain}:{error_reason}, message: {error_message}"
)
return None
def spotify_dl():
"""Main entry point of the script."""
parser = argparse.ArgumentParser(prog='spotify_dl')
parser.add_argument('-d',
'--download',
action='store_true',
help='Download using youtube-dl',
default=True)
parser.add_argument('-p',
'--playlist',
action='store',
help='Download from playlist id instead of'
' saved tracks')
parser.add_argument('-V',
'--verbose',
action='store_true',
help='Show more information on what'
's happening.')
parser.add_argument('-v',
'--version',
action='store_true',
help='Shows current version of the program')
parser.add_argument('-o',
'--output',
type=str,
action='store',
help='Specify download directory.')
parser.add_argument('-u',
'--user_id',
action='store',
help='Specify the playlist owner\'s userid when it'
' is different than your spotify userid')
parser.add_argument('-i',
'--uri',
type=str,
action='store',
nargs='*',
help='Given a URI, download it.')
parser.add_argument('-f',
'--format_str',
type=str,
action='store',
help='Specify youtube-dl format string.',
default='bestaudio/best')
parser.add_argument('-m',
'--skip_mp3',
action='store_true',
help='Don\'t convert downloaded songs to mp3')
parser.add_argument('-l',
'--url',
action="store",
help="Spotify Playlist link URL")
parser.add_argument('-s',
'--scrape',
action="store",
help="Use HTML Scraper for YouTube Search",
default=True)
args = parser.parse_args()
playlist_url_pattern = re.compile(r'^https://open.spotify.com/(.+)$')
if args.version:
print("spotify_dl v{}".format(VERSION))
exit(0)
db.connect()
db.create_tables([Song])
if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')):
with open(os.path.expanduser('~/.spotify_dl_settings')) as file:
config = json.loads(file.read())
for key, value in config.items():
if value and (value.lower() == 'true' or value.lower() == 't'):
setattr(args, key, True)
else:
setattr(args, key, value)
if args.verbose:
log.setLevel(DEBUG)
log.info('Starting spotify_dl')
log.debug('Setting debug mode on spotify_dl')
if not check_for_tokens():
exit(1)
token = authenticate()
sp = spotipy.Spotify(auth=token)
log.debug('Arguments: {}'.format(args))
if args.url:
url_match = playlist_url_pattern.match(args.url)
if url_match and len(url_match.groups()) > 0:
uri = "spotify:" + url_match.groups()[0].replace('/', ':')
args.uri = [uri]
else:
raise Exception('Invalid playlist URL ')
if args.uri:
current_user_id, playlist_id = extract_user_and_playlist_from_uri(
args.uri[0], sp)
else:
if args.user_id is None:
current_user_id = sp.current_user()['id']
else:
current_user_id = args.user_id
if args.output:
if args.uri:
uri = args.uri[0]
playlist = playlist_name(uri, sp)
else:
playlist = get_playlist_name_from_id(args.playlist,
current_user_id, sp)
log.info("Saving songs to: {}".format(playlist))
download_directory = args.output + '/' + playlist
if len(download_directory) >= 0 and download_directory[-1] != '/':
download_directory += '/'
if not os.path.exists(download_directory):
os.makedirs(download_directory)
else:
download_directory = ''
if args.uri:
songs = fetch_tracks(sp, playlist_id, current_user_id)
else:
songs = fetch_tracks(sp, args.playlist, current_user_id)
url = []
for song, artist in songs.items():
link = fetch_youtube_url(song + ' - ' + artist, get_youtube_dev_key())
if link:
url.append((link, song, artist))
save_songs_to_file(url, download_directory)
if args.download is True:
download_songs(url, download_directory, args.format_str, args.skip_mp3)
def fetch_youtube_url(search_term, dev_key=None):
"""
For each song name/artist name combo, fetch the YouTube URL and return the list of URLs.
:param search_term: Search term to be looked up on YouTube
:param dev_key: Youtube API key
"""
in_cache, video_id = check_if_in_cache(search_term)
if in_cache:
return YOUTUBE_VIDEO_URL + video_id
if not dev_key:
YOUTUBE_SEARCH_BASE = "https://www.youtube.com/results?search_query="
try:
response = requests.get(YOUTUBE_SEARCH_BASE + search_term).content
html_response = html.fromstring(response)
video = html_response.xpath(
"//a[contains(@class, 'yt-uix-tile-link')]/@href")
video_id = re.search("((\?v=)[a-zA-Z0-9_-]{4,15})",
video[0]).group(0)[3:]
log.debug(
f"Found video id {video_id} for search term {search_term}")
_ = save_to_cache(search_term=search_term, video_id=video_id)
return YOUTUBE_VIDEO_URL + video_id
except AttributeError as e:
log.warning(f"Could not find scrape details for {search_term}")
capture_exception(e)
return None
except IndexError as e:
log.warning(
f"Could not perform scrape search for {search_term}, got a different HTML"
)
capture_exception(e)
return None
else:
youtube = build(YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=dev_key,
cache_discovery=False)
try:
in_cache, video_id = check_if_in_cache(search_term)
if not in_cache:
search_response = youtube.search().list(
q=search_term, part='id, snippet').execute()
for v in search_response['items']:
if v['id']['kind'] == VIDEO:
video_id = v['id']['videoId']
log.debug(f"Adding Video id {video_id}")
_ = save_to_cache(search_term=search_term,
video_id=video_id)
return YOUTUBE_VIDEO_URL + video_id
except HttpError as err:
err_details = loads(
err.content.decode('utf-8')).get('error').get('errors')
secho("Couldn't complete search due to following errors: ",
fg='red')
for e in err_details:
error_reason = e.get('reason')
error_domain = e.get('domain')
error_message = e.get('message')
if error_reason == 'quotaExceeded' or error_reason == 'dailyLimitExceeded':
secho(
f"\tYou're over daily allowed quota. Unfortunately, YouTube restricts API keys to a max of 10,000 requests per day which translates to a maximum of 100 searches.",
fg='red')
secho(
f"\tThe quota will be reset at midnight Pacific Time (PT).",
fg='red')
secho(
f"\tYou can request for Quota increase from https://console.developers.google.com/apis/api/youtube.googleapis.com/quotas.",
fg='red')
else:
secho(
f"\t Search failed due to {error_domain}:{error_reason}, message: {error_message}"
)
return None
def download_songs(songs,
download_directory,
format_string,
skip_mp3,
keep_playlist_order=False):
"""
Downloads songs from the YouTube URL passed to either current directory or download_directory, is it is passed.
:param songs: Dictionary of songs and associated artist
:param download_directory: Location where to save
:param format_string: format string for the file conversion
:param skip_mp3: Whether to skip conversion to MP3
:param keep_playlist_order: Whether to keep original playlist ordering. Also, prefixes songs files with playlist num
"""
log.debug(f"Downloading to {download_directory}")
for song in songs:
query = f"{song.get('artist')} - {song.get('name')} Lyrics".replace(
":", "").replace("\"", "")
download_archive = path.join(download_directory,
'downloaded_songs.txt')
file_name = sanitize(f"{song.get('artist')} - {song.get('name')}",
'#') # youtube-dl automatically replaces with #
if keep_playlist_order:
# add song number prefix
file_name = f"{song.get('playlist_num')} - {file_name}"
file_path = path.join(download_directory, file_name)
outtmpl = f"{file_path}.%(ext)s"
ydl_opts = {
'format':
format_string,
'download_archive':
download_archive,
'outtmpl':
outtmpl,
'default_search':
'ytsearch',
'noplaylist':
True,
'postprocessor_args': [
'-metadata', 'title=' + song.get('name'), '-metadata',
'artist=' + song.get('artist'), '-metadata',
'album=' + song.get('album')
]
}
if not skip_mp3:
mp3_postprocess_opts = {
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}
ydl_opts['postprocessors'] = [mp3_postprocess_opts.copy()]
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
try:
ydl.download([query])
except Exception as e:
log.debug(e)
print(
'Failed to download: {}, please ensure YouTubeDL is up-to-date. '
.format(query))
continue
if not skip_mp3:
song_file = MP3(path.join(f"{file_path}.mp3"), ID3=EasyID3)
song_file['date'] = song.get('year')
if keep_playlist_order:
song_file['tracknumber'] = str(song.get('playlist_num'))
else:
song_file['tracknumber'] = str(song.get('num')) + '/' + str(
song.get('num_tracks'))
song_file['genre'] = song.get('genre')
song_file.save()
song_file = MP3(path.join(
download_directory,
f"{song.get('artist')} - {song.get('name')}.mp3"),
ID3=ID3)
if song.get('cover') is not None:
song_file.tags['APIC'] = APIC(encoding=3,
mime='image/jpeg',
type=3,
desc=u'Cover',
data=urllib.request.urlopen(
song.get('cover')).read())
song_file.save()
def spotify_dl():
"""Main entry point of the script."""
parser = argparse.ArgumentParser(prog='spotify_dl')
parser.add_argument('-l',
'--url',
action="store",
help="Spotify Playlist link URL",
type=str,
required=True)
parser.add_argument('-o',
'--output',
type=str,
action='store',
help='Specify download directory.',
required=True)
parser.add_argument('-d',
'--download',
action='store_true',
help='Download using youtube-dl',
default=True)
parser.add_argument('-f',
'--format_str',
type=str,
action='store',
help='Specify youtube-dl format string.',
default='bestaudio/best')
parser.add_argument(
'-k',
'--keep_playlist_order',
type=bool,
default=False,
action=argparse.BooleanOptionalAction,
help='Whether to keep original playlist ordering or not.')
parser.add_argument('-m',
'--skip_mp3',
action='store_true',
help='Don\'t convert downloaded songs to mp3')
parser.add_argument('-s',
'--scrape',
action="store",
help="Use HTML Scraper for YouTube Search",
default=True)
parser.add_argument('-V',
'--verbose',
action='store_true',
help='Show more information on what'
's happening.')
parser.add_argument('-v',
'--version',
action='store_true',
help='Shows current version of the program')
parser.add_argument('-c',
'--createdir',
action='store_true',
help='Create a subdirectory')
args = parser.parse_args()
if args.version:
print("spotify_dl v{}".format(VERSION))
exit(0)
db.connect()
db.create_tables([Song])
if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')):
with open(os.path.expanduser('~/.spotify_dl_settings')) as file:
config = json.loads(file.read())
for key, value in config.items():
if value and (value.lower() == 'true' or value.lower() == 't'):
setattr(args, key, True)
else:
setattr(args, key, value)
if args.verbose:
log.setLevel(DEBUG)
log.info('Starting spotify_dl')
log.debug('Setting debug mode on spotify_dl')
if not check_for_tokens():
exit(1)
sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials())
log.debug('Arguments: {}'.format(args))
# TODO: make the logic less dumb
if args.url:
valid_item = validate_spotify_url(args.url)
valid_yt = validate_youtube_url(args.url)
if valid_item:
download_spotify(sp, args)
elif valid_yt:
download_youtube(sp, args)
exit(1)
def spotify_dl():
"""Main entry point of the script."""
parser = argparse.ArgumentParser(prog='spotify_dl')
parser.add_argument('-l',
'--url',
action="store",
help="Spotify Playlist link URL",
type=str,
required=True)
parser.add_argument('-o',
'--output',
type=str,
action='store',
help='Specify download directory.',
required=True)
parser.add_argument('-d',
'--download',
action='store_true',
help='Download using youtube-dl',
default=True)
parser.add_argument('-f',
'--format_str',
type=str,
action='store',
help='Specify youtube-dl format string.',
default='bestaudio/best')
parser.add_argument(
'-k',
'--keep_playlist_order',
default=False,
action='store_true',
help='Whether to keep original playlist ordering or not.')
parser.add_argument('-m',
'--skip_mp3',
action='store_true',
help='Don\'t convert downloaded songs to mp3')
parser.add_argument('-s',
'--scrape',
action="store",
help="Use HTML Scraper for YouTube Search",
default=True)
parser.add_argument('-V',
'--verbose',
action='store_true',
help='Show more information on what'
's happening.')
parser.add_argument('-v',
'--version',
action='store_true',
help='Shows current version of the program')
args = parser.parse_args()
if args.version:
print("spotify_dl v{}".format(VERSION))
exit(0)
db.connect()
db.create_tables([Song])
if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')):
with open(os.path.expanduser('~/.spotify_dl_settings')) as file:
config = json.loads(file.read())
for key, value in config.items():
if value and (value.lower() == 'true' or value.lower() == 't'):
setattr(args, key, True)
else:
setattr(args, key, value)
if args.verbose:
log.setLevel(DEBUG)
log.info('Starting spotify_dl')
log.debug('Setting debug mode on spotify_dl')
if not check_for_tokens():
exit(1)
sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials())
log.debug('Arguments: {}'.format(args))
if args.url:
valid_item = validate_spotify_url(args.url)
if not valid_item:
sys.exit(1)
if args.output:
item_type, item_id = parse_spotify_url(args.url)
directory_name = get_item_name(sp, item_type, item_id)
save_path = Path(
PurePath.joinpath(Path(args.output), Path(directory_name)))
save_path.mkdir(parents=True, exist_ok=True)
log.info("Saving songs to: {}".format(directory_name))
songs = fetch_tracks(sp, item_type, args.url)
if args.download is True:
file_name_f = default_filename
if args.keep_playlist_order:
file_name_f = playlist_num_filename
download_songs(songs, save_path, args.format_str, args.skip_mp3,
args.keep_playlist_order, file_name_f)
