def save_to_cache(search_term, video_id): """ Saves the search term and video id to the database cache so it can be looked up later. :param search_term: Search term to be saved to in the cache :param video_id: Video id to be saved to in the cache :return Video id saved in the cache """ song_info, saved = Song.get_or_create(search_term=search_term, video_id=video_id) log.debug(f"Saved: {saved} video id {song_info.video_id} in cache") return song_info.video_id
def fetch_youtube_url(search_term): """For each song name/artist name combo, fetch the YouTube URL and return the list of URLs""" YOUTUBE_DEV_KEY = getenv('YOUTUBE_DEV_KEY') youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=YOUTUBE_DEV_KEY) log.info(u"Searching for {}".format(search_term)) search_response = youtube.search().list(q=search_term, part='id, snippet').execute() for v in search_response['items']: if v['id']['kind'] == VIDEO: log.debug("Adding Video id {}".format(v['id']['videoId'])) return YOUTUBE_VIDEO_URL + v['id']['videoId']
def check_if_in_cache(search_term): """ Checks if the specified search term is in the local database cache. and returns the video id if it exists. :param search_term: String to be searched for in the cache :return A tuple with Boolean and video id if it exists """ try: song = Song.get(search_term=search_term) log.debug(f"Found id {song.video_id} for {search_term} in cache") return True, song.video_id except DoesNotExist: log.debug(f"Couldn't find id for {search_term} in cache") return False, None
def validate_spotify_url(url): """ Validate the URL and determine if the item type is supported. :return Boolean indicating whether or not item is supported """ item_type, item_id = parse_spotify_url(url) log.debug(f"Got item type {item_type} and item_id {item_id}") if item_type not in ['album', 'track', 'playlist']: log.error("Only albums/tracks/playlists are supported") return False if item_id is None: log.error("Couldn't get a valid id") return False return True
def validate_spotify_url(url): """ Validate the URL to determine if the item type is supported. :return Boolean . """ type, id = parse_spotify_url(url) log.debug(f" item type :{type} ; item_id: {id}") if type not in ['album', 'track', 'playlist']: log.error("Only albums/tracks/playlists are supported") return False if id is None: log.error("Couldn't get a valid id") return False return True
def download_songs(songs, download_directory, format_string, skip_mp3): """ Downloads songs from the YouTube URL passed to either current directory or download_directory, is it is passed. :param songs: Dictionary of songs and associated artist :param download_directory: Location where to save :param format_string: format string for the file conversion :param skip_mp3: Whether to skip conversion to MP3 """ log.debug(f"Downloading to {download_directory}") for song, artist in songs.items(): query = f"{artist} - {song}".replace(":", "").replace("\"", "") download_archive = path.join(download_directory, 'downloaded_songs.txt') outtmpl = path.join(download_directory, '%(title)s.%(ext)s') ydl_opts = { 'format': format_string, 'download_archive': download_archive, 'outtmpl': outtmpl, 'default_search': 'ytsearch', 'noplaylist': True, 'postprocessor_args': ['-metadata', 'title=' + song, '-metadata', 'artist=' + artist] } if not skip_mp3: mp3_postprocess_opts = { 'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192', } ydl_opts['postprocessors'] = [mp3_postprocess_opts.copy()] with youtube_dl.YoutubeDL(ydl_opts) as ydl: try: ydl.download([query]) except Exception as e: log.debug(e) print( 'Failed to download: {}, please ensure YouTubeDL is up-to-date. ' .format(query)) continue
def fetch_youtube_url(search_term, dev_key): """For each song name/artist name combo, fetch the YouTube URL and return the list of URLs""" YOUTUBE_DEV_KEY = dev_key youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=YOUTUBE_DEV_KEY, cache_discovery=False) log.info(u"Searching for {}".format(search_term)) try: search_response = youtube.search().list(q=search_term, part='id, snippet').execute() for v in search_response['items']: if v['id']['kind'] == VIDEO: log.debug("Adding Video id {}".format(v['id']['videoId'])) return YOUTUBE_VIDEO_URL + v['id']['videoId'] except HttpError as err: err_details = loads( err.content.decode('utf-8')).get('error').get('errors') secho("Couldn't complete search due to following errors: ", fg='red') for e in err_details: error_reason = e.get('reason') error_domain = e.get('domain') error_message = e.get('message') if error_reason == 'quotaExceeded' or error_reason == 'dailyLimitExceeded': secho( f"\tYou're over daily allowed quota. Unfortunately, YouTube restricts API keys to a max of 10,000 requests per day which translates to a maximum of 100 searches.", fg='red') secho( f"\tThe quota will be reset at midnight Pacific Time (PT).", fg='red') secho( f"\tYou can request for Quota increase from https://console.developers.google.com/apis/api/youtube.googleapis.com/quotas.", fg='red') else: secho( f"\t Search failed due to {error_domain}:{error_reason}, message: {error_message}" ) return None
def spotify_dl(): """Main entry point of the script.""" parser = argparse.ArgumentParser(prog='spotify_dl') parser.add_argument('-d', '--download', action='store_true', help='Download using youtube-dl', default=True) parser.add_argument('-p', '--playlist', action='store', help='Download from playlist id instead of' ' saved tracks') parser.add_argument('-V', '--verbose', action='store_true', help='Show more information on what' 's happening.') parser.add_argument('-v', '--version', action='store_true', help='Shows current version of the program') parser.add_argument('-o', '--output', type=str, action='store', help='Specify download directory.') parser.add_argument('-u', '--user_id', action='store', help='Specify the playlist owner\'s userid when it' ' is different than your spotify userid') parser.add_argument('-i', '--uri', type=str, action='store', nargs='*', help='Given a URI, download it.') parser.add_argument('-f', '--format_str', type=str, action='store', help='Specify youtube-dl format string.', default='bestaudio/best') parser.add_argument('-m', '--skip_mp3', action='store_true', help='Don\'t convert downloaded songs to mp3') parser.add_argument('-l', '--url', action="store", help="Spotify Playlist link URL") parser.add_argument('-s', '--scrape', action="store", help="Use HTML Scraper for YouTube Search", default=True) args = parser.parse_args() playlist_url_pattern = re.compile(r'^https://open.spotify.com/(.+)$') if args.version: print("spotify_dl v{}".format(VERSION)) exit(0) db.connect() db.create_tables([Song]) if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')): with open(os.path.expanduser('~/.spotify_dl_settings')) as file: config = json.loads(file.read()) for key, value in config.items(): if value and (value.lower() == 'true' or value.lower() == 't'): setattr(args, key, True) else: setattr(args, key, value) if args.verbose: log.setLevel(DEBUG) log.info('Starting spotify_dl') log.debug('Setting debug mode on spotify_dl') if not check_for_tokens(): exit(1) token = authenticate() sp = spotipy.Spotify(auth=token) log.debug('Arguments: {}'.format(args)) if args.url: url_match = playlist_url_pattern.match(args.url) if url_match and len(url_match.groups()) > 0: uri = "spotify:" + url_match.groups()[0].replace('/', ':') args.uri = [uri] else: raise Exception('Invalid playlist URL ') if args.uri: current_user_id, playlist_id = extract_user_and_playlist_from_uri( args.uri[0], sp) else: if args.user_id is None: current_user_id = sp.current_user()['id'] else: current_user_id = args.user_id if args.output: if args.uri: uri = args.uri[0] playlist = playlist_name(uri, sp) else: playlist = get_playlist_name_from_id(args.playlist, current_user_id, sp) log.info("Saving songs to: {}".format(playlist)) download_directory = args.output + '/' + playlist if len(download_directory) >= 0 and download_directory[-1] != '/': download_directory += '/' if not os.path.exists(download_directory): os.makedirs(download_directory) else: download_directory = '' if args.uri: songs = fetch_tracks(sp, playlist_id, current_user_id) else: songs = fetch_tracks(sp, args.playlist, current_user_id) url = [] for song, artist in songs.items(): link = fetch_youtube_url(song + ' - ' + artist, get_youtube_dev_key()) if link: url.append((link, song, artist)) save_songs_to_file(url, download_directory) if args.download is True: download_songs(url, download_directory, args.format_str, args.skip_mp3)
def fetch_youtube_url(search_term, dev_key=None): """ For each song name/artist name combo, fetch the YouTube URL and return the list of URLs. :param search_term: Search term to be looked up on YouTube :param dev_key: Youtube API key """ in_cache, video_id = check_if_in_cache(search_term) if in_cache: return YOUTUBE_VIDEO_URL + video_id if not dev_key: YOUTUBE_SEARCH_BASE = "https://www.youtube.com/results?search_query=" try: response = requests.get(YOUTUBE_SEARCH_BASE + search_term).content html_response = html.fromstring(response) video = html_response.xpath( "//a[contains(@class, 'yt-uix-tile-link')]/@href") video_id = re.search("((\?v=)[a-zA-Z0-9_-]{4,15})", video[0]).group(0)[3:] log.debug( f"Found video id {video_id} for search term {search_term}") _ = save_to_cache(search_term=search_term, video_id=video_id) return YOUTUBE_VIDEO_URL + video_id except AttributeError as e: log.warning(f"Could not find scrape details for {search_term}") capture_exception(e) return None except IndexError as e: log.warning( f"Could not perform scrape search for {search_term}, got a different HTML" ) capture_exception(e) return None else: youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=dev_key, cache_discovery=False) try: in_cache, video_id = check_if_in_cache(search_term) if not in_cache: search_response = youtube.search().list( q=search_term, part='id, snippet').execute() for v in search_response['items']: if v['id']['kind'] == VIDEO: video_id = v['id']['videoId'] log.debug(f"Adding Video id {video_id}") _ = save_to_cache(search_term=search_term, video_id=video_id) return YOUTUBE_VIDEO_URL + video_id except HttpError as err: err_details = loads( err.content.decode('utf-8')).get('error').get('errors') secho("Couldn't complete search due to following errors: ", fg='red') for e in err_details: error_reason = e.get('reason') error_domain = e.get('domain') error_message = e.get('message') if error_reason == 'quotaExceeded' or error_reason == 'dailyLimitExceeded': secho( f"\tYou're over daily allowed quota. Unfortunately, YouTube restricts API keys to a max of 10,000 requests per day which translates to a maximum of 100 searches.", fg='red') secho( f"\tThe quota will be reset at midnight Pacific Time (PT).", fg='red') secho( f"\tYou can request for Quota increase from https://console.developers.google.com/apis/api/youtube.googleapis.com/quotas.", fg='red') else: secho( f"\t Search failed due to {error_domain}:{error_reason}, message: {error_message}" ) return None
def download_songs(songs, download_directory, format_string, skip_mp3, keep_playlist_order=False): """ Downloads songs from the YouTube URL passed to either current directory or download_directory, is it is passed. :param songs: Dictionary of songs and associated artist :param download_directory: Location where to save :param format_string: format string for the file conversion :param skip_mp3: Whether to skip conversion to MP3 :param keep_playlist_order: Whether to keep original playlist ordering. Also, prefixes songs files with playlist num """ log.debug(f"Downloading to {download_directory}") for song in songs: query = f"{song.get('artist')} - {song.get('name')} Lyrics".replace( ":", "").replace("\"", "") download_archive = path.join(download_directory, 'downloaded_songs.txt') file_name = sanitize(f"{song.get('artist')} - {song.get('name')}", '#') # youtube-dl automatically replaces with # if keep_playlist_order: # add song number prefix file_name = f"{song.get('playlist_num')} - {file_name}" file_path = path.join(download_directory, file_name) outtmpl = f"{file_path}.%(ext)s" ydl_opts = { 'format': format_string, 'download_archive': download_archive, 'outtmpl': outtmpl, 'default_search': 'ytsearch', 'noplaylist': True, 'postprocessor_args': [ '-metadata', 'title=' + song.get('name'), '-metadata', 'artist=' + song.get('artist'), '-metadata', 'album=' + song.get('album') ] } if not skip_mp3: mp3_postprocess_opts = { 'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192', } ydl_opts['postprocessors'] = [mp3_postprocess_opts.copy()] with youtube_dl.YoutubeDL(ydl_opts) as ydl: try: ydl.download([query]) except Exception as e: log.debug(e) print( 'Failed to download: {}, please ensure YouTubeDL is up-to-date. ' .format(query)) continue if not skip_mp3: song_file = MP3(path.join(f"{file_path}.mp3"), ID3=EasyID3) song_file['date'] = song.get('year') if keep_playlist_order: song_file['tracknumber'] = str(song.get('playlist_num')) else: song_file['tracknumber'] = str(song.get('num')) + '/' + str( song.get('num_tracks')) song_file['genre'] = song.get('genre') song_file.save() song_file = MP3(path.join( download_directory, f"{song.get('artist')} - {song.get('name')}.mp3"), ID3=ID3) if song.get('cover') is not None: song_file.tags['APIC'] = APIC(encoding=3, mime='image/jpeg', type=3, desc=u'Cover', data=urllib.request.urlopen( song.get('cover')).read()) song_file.save()
def spotify_dl(): """Main entry point of the script.""" parser = argparse.ArgumentParser(prog='spotify_dl') parser.add_argument('-l', '--url', action="store", help="Spotify Playlist link URL", type=str, required=True) parser.add_argument('-o', '--output', type=str, action='store', help='Specify download directory.', required=True) parser.add_argument('-d', '--download', action='store_true', help='Download using youtube-dl', default=True) parser.add_argument('-f', '--format_str', type=str, action='store', help='Specify youtube-dl format string.', default='bestaudio/best') parser.add_argument( '-k', '--keep_playlist_order', type=bool, default=False, action=argparse.BooleanOptionalAction, help='Whether to keep original playlist ordering or not.') parser.add_argument('-m', '--skip_mp3', action='store_true', help='Don\'t convert downloaded songs to mp3') parser.add_argument('-s', '--scrape', action="store", help="Use HTML Scraper for YouTube Search", default=True) parser.add_argument('-V', '--verbose', action='store_true', help='Show more information on what' 's happening.') parser.add_argument('-v', '--version', action='store_true', help='Shows current version of the program') parser.add_argument('-c', '--createdir', action='store_true', help='Create a subdirectory') args = parser.parse_args() if args.version: print("spotify_dl v{}".format(VERSION)) exit(0) db.connect() db.create_tables([Song]) if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')): with open(os.path.expanduser('~/.spotify_dl_settings')) as file: config = json.loads(file.read()) for key, value in config.items(): if value and (value.lower() == 'true' or value.lower() == 't'): setattr(args, key, True) else: setattr(args, key, value) if args.verbose: log.setLevel(DEBUG) log.info('Starting spotify_dl') log.debug('Setting debug mode on spotify_dl') if not check_for_tokens(): exit(1) sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials()) log.debug('Arguments: {}'.format(args)) # TODO: make the logic less dumb if args.url: valid_item = validate_spotify_url(args.url) valid_yt = validate_youtube_url(args.url) if valid_item: download_spotify(sp, args) elif valid_yt: download_youtube(sp, args) exit(1)
def spotify_dl(): """Main entry point of the script.""" parser = argparse.ArgumentParser(prog='spotify_dl') parser.add_argument('-l', '--url', action="store", help="Spotify Playlist link URL", type=str, required=True) parser.add_argument('-o', '--output', type=str, action='store', help='Specify download directory.', required=True) parser.add_argument('-d', '--download', action='store_true', help='Download using youtube-dl', default=True) parser.add_argument('-f', '--format_str', type=str, action='store', help='Specify youtube-dl format string.', default='bestaudio/best') parser.add_argument( '-k', '--keep_playlist_order', default=False, action='store_true', help='Whether to keep original playlist ordering or not.') parser.add_argument('-m', '--skip_mp3', action='store_true', help='Don\'t convert downloaded songs to mp3') parser.add_argument('-s', '--scrape', action="store", help="Use HTML Scraper for YouTube Search", default=True) parser.add_argument('-V', '--verbose', action='store_true', help='Show more information on what' 's happening.') parser.add_argument('-v', '--version', action='store_true', help='Shows current version of the program') args = parser.parse_args() if args.version: print("spotify_dl v{}".format(VERSION)) exit(0) db.connect() db.create_tables([Song]) if os.path.isfile(os.path.expanduser('~/.spotify_dl_settings')): with open(os.path.expanduser('~/.spotify_dl_settings')) as file: config = json.loads(file.read()) for key, value in config.items(): if value and (value.lower() == 'true' or value.lower() == 't'): setattr(args, key, True) else: setattr(args, key, value) if args.verbose: log.setLevel(DEBUG) log.info('Starting spotify_dl') log.debug('Setting debug mode on spotify_dl') if not check_for_tokens(): exit(1) sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials()) log.debug('Arguments: {}'.format(args)) if args.url: valid_item = validate_spotify_url(args.url) if not valid_item: sys.exit(1) if args.output: item_type, item_id = parse_spotify_url(args.url) directory_name = get_item_name(sp, item_type, item_id) save_path = Path( PurePath.joinpath(Path(args.output), Path(directory_name))) save_path.mkdir(parents=True, exist_ok=True) log.info("Saving songs to: {}".format(directory_name)) songs = fetch_tracks(sp, item_type, args.url) if args.download is True: file_name_f = default_filename if args.keep_playlist_order: file_name_f = playlist_num_filename download_songs(songs, save_path, args.format_str, args.skip_mp3, args.keep_playlist_order, file_name_f)