def create_playlist_file (name, songs, outpath):
filename = os.path.join(outpath, name + '.m3u')
if not cli['dry-run']:
m3u = [u'#EXTM3U']
for track in songs:
id = songid(track)
song = songs_dict[id]
artist = song['artist']
title = song['title']
duration = str(int(int(song['durationMillis']) / 1000)) if 'durationMillis' in song else '0'
metadata = metadata_from_mobile_client_song(song)
songpath = template_to_filepath(cli['output'], metadata) + '.mp3'
m3u.append(u'#EXTINF,' + duration + ',' + song['artist'] + ' - ' + song['title'])
m3u.append(os.path.relpath(songpath, outpath))
# track this song as a local song to keep
add_to_local_songs(songpath)
# write m3u file
contentstr = u'\n'.join(m3u)
# write to temp file
with tempfile.NamedTemporaryFile(suffix='.m3u', delete=False) as temp:
temp.write(contentstr.encode('UTF-8-SIG'))
# move tempfile into place
utils.make_sure_path_exists(os.path.dirname(filename), 0o700)
shutil.move(temp.name, filename)
logger.log(QUIET, "Playlist ({0} tracks): {1}".format(len(songs), filename))
return filename
def _oauth_login(self, oauth_credentials):
"""Auth ourselves to the MM oauth endpoint.
Return True on success; see :py:func:`login` for params.
"""
if isinstance(oauth_credentials, basestring):
oauth_file = oauth_credentials
if oauth_file == OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH),
0o700)
storage = oauth2client.file.Storage(oauth_file)
oauth_credentials = storage.get()
if oauth_credentials is None:
self.logger.warning(
"could not retrieve oauth credentials from '%s'",
oauth_file)
return False
if not self.session.login(oauth_credentials):
self.logger.warning("failed to authenticate")
return False
self.logger.info("oauth successful")
return True
def login_mobile_client_from_cache(mcw, oauth_filename='oauth'):
# mcw is MobileClientWrapper
creds_filepath = os.path.join(os.path.dirname(OAUTH_FILEPATH), 'mc' + oauth_filename + '.cred')
# fetch credentials from file
try:
logger.info("Trying stored credentials to log in to Mobile Client")
with open(creds_filepath) as creds_file:
creds = json.load(creds_file)
logger.info(creds);
# fake login
mcw.api.session._master_token = creds['masterToken']
mcw.api.session._authtoken = creds['authToken']
mcw.api.android_id = creds['androidId']
mcw.api.session.is_authenticated = True
except:
logger.info("Unable to load credentials file...")
try:
# test fake login with get_devices call
devices = mcw.api.get_registered_devices()
# logger.info(devices);
except:
creds = login_mobile_client(mcw);
utils.make_sure_path_exists(os.path.dirname(creds_filepath), 0o700)
with open(creds_filepath, 'w') as outfile:
json.dump(creds, outfile)
logger.info("Stored Mobile Client credentials")
def set_oauth_code(self, oauth_code, storage_filepath):
credentials = Musicmanager.flow.step2_exchange(oauth_code)
if storage_filepath is not None:
if storage_filepath == OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700)
storage = oauth2client.file.Storage(storage_filepath)
storage.put(credentials)
def perform_oauth(storage_filepath=OAUTH_FILEPATH, open_browser=False):
"""Provides a series of prompts for a user to follow to authenticate.
Returns ``oauth2client.client.OAuth2Credentials`` when successful.
In most cases, this should only be run once per machine to store
credentials to disk, then never be needed again.
If the user refuses to give access,
``oauth2client.client.FlowExchangeError`` is raised.
:param storage_filepath: a filepath to write the credentials to,
or ``None``
to not write the credentials to disk (which is not recommended).
`Appdirs <https://pypi.python.org/pypi/appdirs/1.2.0>`__
``user_data_dir`` is used by default. Users can run::
import gmusicapi.clients
print gmusicapi.clients.OAUTH_FILEPATH
to see the exact location on their system.
:param open_browser: if True, attempt to open the auth url
in the system default web browser. The url will be printed
regardless of this param's setting.
This flow is intentionally very simple.
For complete control over the OAuth flow, pass an
``oauth2client.client.OAuth2Credentials``
to :func:`login` instead.
"""
flow = OAuth2WebServerFlow(*musicmanager.oauth)
auth_uri = flow.step1_get_authorize_url()
print
print "Visit the following url:\n %s" % auth_uri
if open_browser:
print
print 'Opening your browser to it now...',
webbrowser.open(auth_uri)
print 'done.'
print "If you don't see your browser, you can just copy and paste the url."
print
code = raw_input("Follow the prompts,"
" then paste the auth code here and hit enter: ")
credentials = flow.step2_exchange(code)
if storage_filepath is not None:
if storage_filepath == OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH),
0o700)
storage = oauth2client.file.Storage(storage_filepath)
storage.put(credentials)
return credentials
def perform_oauth(storage_filepath=OAUTH_FILEPATH, open_browser=False):
"""Provides a series of prompts for a user to follow to authenticate.
Returns ``oauth2client.client.OAuth2Credentials`` when successful.
In most cases, this should only be run once per machine to store
credentials to disk, then never be needed again.
If the user refuses to give access,
``oauth2client.client.FlowExchangeError`` is raised.
:param storage_filepath: a filepath to write the credentials to,
or ``None``
to not write the credentials to disk (which is not recommended).
`Appdirs <https://pypi.python.org/pypi/appdirs>`__
``user_data_dir`` is used by default. Users can run::
import gmusicapi.clients
print gmusicapi.clients.OAUTH_FILEPATH
to see the exact location on their system.
:param open_browser: if True, attempt to open the auth url
in the system default web browser. The url will be printed
regardless of this param's setting.
This flow is intentionally very simple.
For complete control over the OAuth flow, pass an
``oauth2client.client.OAuth2Credentials``
to :func:`login` instead.
"""
flow = OAuth2WebServerFlow(*musicmanager.oauth)
auth_uri = flow.step1_get_authorize_url()
print
print "Visit the following url:\n %s" % auth_uri
if open_browser:
print
print 'Opening your browser to it now...',
webbrowser.open(auth_uri)
print 'done.'
print "If you don't see your browser, you can just copy and paste the url."
print
code = raw_input("Follow the prompts,"
" then paste the auth code here and hit enter: ")
credentials = flow.step2_exchange(code)
if storage_filepath is not None:
if storage_filepath == OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700)
storage = oauth2client.file.Storage(storage_filepath)
storage.put(credentials)
return credentials
def get_auth_token (self, code, storage_filepath=OAUTH_FILEPATH):
credentials = self.__flow.step2_exchange(code)
if storage_filepath is not None:
if storage_filepath == OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700)
storage = oauth2client.file.Storage(storage_filepath)
storage.put(credentials)
return credentials
def _oauth_login(self, oauth_credentials):
"""Return True on success."""
if isinstance(oauth_credentials, basestring):
oauth_file = oauth_credentials
if oauth_file == self.OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(self.OAUTH_FILEPATH), 0o700)
storage = oauth2client.file.Storage(oauth_file)
oauth_credentials = storage.get()
if oauth_credentials is None:
self.logger.warning("could not retrieve oauth credentials from '%r'", oauth_file)
return False
if not self.session.login(oauth_credentials):
self.logger.warning("failed to authenticate")
return False
self.logger.info("oauth successful")
return True
def _oauth_login(self, oauth_credentials):
"""Auth ourselves to the MM oauth endpoint.
Return True on success; see :py:func:`login` for params.
"""
if isinstance(oauth_credentials, str):
oauth_file = oauth_credentials
if oauth_file == OAUTH_FILEPATH:
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700)
storage = oauth2client.file.Storage(oauth_file)
oauth_credentials = storage.get()
if oauth_credentials is None:
self.logger.warning("could not retrieve oauth credentials from '%s'", oauth_file)
return False
if not self.session.login(oauth_credentials):
self.logger.warning("failed to authenticate")
return False
self.logger.info("oauth successful")
return True
def _oauth_login(self, oauth_credentials):
"""Return True on success."""
if isinstance(oauth_credentials, str):
oauth_file = oauth_credentials
if oauth_file == self.OAUTH_FILEPATH:
utils.make_sure_path_exists(
os.path.dirname(self.OAUTH_FILEPATH), 0o700)
storage = oauth2client.file.Storage(oauth_file)
oauth_credentials = storage.get()
if oauth_credentials is None:
self.logger.warning(
"could not retrieve oauth credentials from '%r'",
oauth_file)
return False
if not self.session.login(oauth_credentials):
self.logger.warning("failed to authenticate")
return False
self.logger.info("oauth successful")
return True
import httplib2 # included with oauth2client
from oauth2client.client import OAuth2WebServerFlow, TokenRevokeError
import oauth2client.file
import gmusicapi
from gmusicapi.gmtools import tools
from gmusicapi.exceptions import CallFailure, NotLoggedIn
from gmusicapi.protocol import webclient, musicmanager, upload_pb2, locker_pb2
from gmusicapi.utils import utils
import gmusicapi.session
OAUTH_FILEPATH = os.path.join(utils.my_appdirs.user_data_dir, 'oauth.cred')
# oauth client breaks if the dir doesn't exist
utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700)
class _Base(object):
"""Factors out common client setup."""
__metaclass__ = utils.DocstringInheritMeta
num_clients = 0 # used to disambiguate loggers
def __init__(self, logger_basename, debug_logging):
"""
:param debug_logging: each Client has a ``logger`` member.
The logger is named ``gmusicapi.<client class><client number>`` and
will propogate to the ``gmusicapi`` root logger.
def main():
cli = dict((key.lstrip("-<").rstrip(">"), value) for key, value in docopt(__doc__).items())
if cli['no-recursion']:
cli['max-depth'] = 0
else:
cli['max-depth'] = int(cli['max-depth']) if cli['max-depth'] else float('inf')
if cli['quiet']:
logger.setLevel(QUIET)
else:
logger.setLevel(logging.INFO)
if not cli['input']:
cli['input'] = [os.getcwd()]
if not cli['output']:
cli['output'] = os.getcwd()
include_filters = [tuple(filt.split(':', 1)) for filt in cli['include-filter']]
exclude_filters = [tuple(filt.split(':', 1)) for filt in cli['exclude-filter']]
mmw = MusicManagerWrapper(enable_logging=cli['log'])
mmw.login(oauth_filename=cli['cred'], uploader_id=cli['uploader-id'])
if not mmw.is_authenticated:
sys.exit()
mcw = MobileClientWrapper(enable_logging=cli['log'])
if cli['playlists']:
login_mobile_client_from_cache(mcw, oauth_filename=cli['cred'])
if not mcw.is_authenticated:
sys.exit()
if cli['down']:
matched_google_songs, filtered_google_songs = mmw.get_google_songs(
include_filters=include_filters, exclude_filters=exclude_filters,
all_includes=cli['all-includes'], all_excludes=cli['all-excludes']
)
cli['input'] = [template_to_base_path(cli['output'], matched_google_songs)]
matched_local_songs, filtered_local_songs, excluded_local_songs = mmw.get_local_songs(cli['input'], exclude_patterns=cli['exclude'])
# keep track of all local songs
all_local_songs_dict = dict()
def add_to_local_songs (filepath):
all_local_songs_dict[os.path.abspath(filepath)] = True
def all_local_songs ():
return list(all_local_songs_dict.keys())
for filepath in matched_local_songs + filtered_local_songs + excluded_local_songs:
metadata = _get_mutagen_metadata(filepath);
songpath = template_to_filepath(cli['output'], metadata) + '.mp3'
if filepath != songpath:
utils.make_sure_path_exists(os.path.dirname(songpath), 0o700)
print("{0} ~> {1}".format(filepath, songpath))
shutil.move(filepath, songpath)
add_to_local_songs(songpath)
# add_to_local_songs(filepath)
# for filepath in matched_local_songs + filtered_local_songs + excluded_local_songs:
# add_to_local_songs(filepath)
def download_songs(songs_to_download):
if songs_to_download:
if cli['dry-run']:
logger.info("\nFound {0} song(s) to download".format(len(songs_to_download)))
for song in songs_to_download:
title = song.get('title', "<title>")
artist = song.get('artist', "<artist>")
album = song.get('album', "<album>")
song_id = song['id']
metadata = metadata_from_mobile_client_song(song)
songpath = template_to_filepath(cli['output'], metadata) + '.mp3'
logger.log(QUIET, "{0} -- {1} -- {2} ({3})".format(title, artist, album, song_id))
add_to_local_songs(songpath)
else:
logger.info("\nDownloading {0} song(s) from Google Music\n".format(len(songs_to_download)))
results = mmw.download(songs_to_download, template=cli['output'])
# keep track of new filepaths
for res in results:
if res['result'] == 'downloaded':
add_to_local_songs(res['filepath'])
else:
logger.info("\nNo songs to download")
def download_missing_google_songs(mmw_songs):
# recheck the local songs after any previous sync
logger.info("\nFinding missing songs...")
songs_to_download = compare_song_collections(mmw_songs, all_local_songs())
songs_to_download.sort(key=lambda song: (song.get('artist'), song.get('album'), song.get('track_number')))
return download_songs(songs_to_download)
logger.info("\nFetching Library songs...")
download_missing_google_songs(matched_google_songs)
if cli['playlists']:
logger.info("Syncing playlists...")
# get all songs from mobileClient api (to include ratings)
all_songs = mcw.api.get_all_songs()
# get id, prioritize trackId over id
def songid (track):
return track['trackId'] if 'trackId' in track else track['id']
# create a dictionary of all fetched google songs, indexed by id
def songs_to_dict (songs, song):
id = songid(song)
songs[id] = song
return songs
# create lookup dicts for tracks (mmw) and songs (mcw)
songs_dict = reduce(songs_to_dict, all_songs, {})
tracks_dict = reduce(songs_to_dict, matched_google_songs + filtered_google_songs, {})
# returns music manager wrapper tracks for list of objects with song id or trackId
# also removes duplicates
def get_mmw_tracks (songs):
tracks = []
seen = {}
for song in songs:
id = songid(song)
if id not in seen:
tracks.append(tracks_dict[id])
seen[id] = True
return tracks;
# path to save playlists
playlists_dir = os.path.abspath(cli['playlists'])
# ensure directory is there
utils.make_sure_path_exists(playlists_dir, 0o700)
def create_playlist_file (name, songs, outpath):
filename = os.path.join(outpath, name + '.m3u')
if not cli['dry-run']:
m3u = [u'#EXTM3U']
for track in songs:
id = songid(track)
song = songs_dict[id]
artist = song['artist']
title = song['title']
duration = str(int(int(song['durationMillis']) / 1000)) if 'durationMillis' in song else '0'
metadata = metadata_from_mobile_client_song(song)
songpath = template_to_filepath(cli['output'], metadata) + '.mp3'
m3u.append(u'#EXTINF,' + duration + ',' + song['artist'] + ' - ' + song['title'])
m3u.append(os.path.relpath(songpath, outpath))
# track this song as a local song to keep
add_to_local_songs(songpath)
# write m3u file
contentstr = u'\n'.join(m3u)
# write to temp file
with tempfile.NamedTemporaryFile(suffix='.m3u', delete=False) as temp:
temp.write(contentstr.encode('UTF-8-SIG'))
# move tempfile into place
utils.make_sure_path_exists(os.path.dirname(filename), 0o700)
shutil.move(temp.name, filename)
logger.log(QUIET, "Playlist ({0} tracks): {1}".format(len(songs), filename))
return filename
# get playlists with ordered lists of tracks
playlists = mcw.api.get_all_user_playlist_contents()
# concatenate all the playlist tracks into a single list
playlist_tracks = reduce(lambda x, y: x + y, map(lambda x: x['tracks'], playlists)) if len(playlists) else []
# remove duplicates and get mmw tracks to download
playlist_tracks = get_mmw_tracks(playlist_tracks)
# download any missing songs
logger.info("\nFetching Playlist songs...")
download_missing_google_songs(playlist_tracks)
# create the m3u files for the playlists
created_playlists = [];
for playlist in playlists:
playlist_filename = create_playlist_file(playlist['name'], playlist['tracks'], playlists_dir)
created_playlists.append(playlist_filename)
# create an m3u file for all favorited songs
favorites_playlist_name = cli['favorites'] if 'favorites' in cli else '___auto_favorites___'
# filter mobile client songs into favorites list
thumbs_up = [t for t in all_songs if int(t['rating']) > 3]
# most recent first
thumbs_up.sort(key=lambda song: (int(song.get('lastModifiedTimestamp')) * -1))
# get music_manager_wrapper style tracks to compare
thumbs_up_tracks = get_mmw_tracks(thumbs_up)
# download any missing favorited songs
logger.info("\nFetching Favorited songs...")
download_missing_google_songs(thumbs_up_tracks)
# create favorites playlist
created_playlist_filename = create_playlist_file(favorites_playlist_name, thumbs_up, playlists_dir)
created_playlists.append(created_playlist_filename)
if cli['removed']:
logger.info("Moving Removed songs...")
# path to move songs removed from google music
removed_dir = os.path.abspath(cli['removed'])
# ensure directory is there
utils.make_sure_path_exists(removed_dir, 0o700)
# local songs after sync
all_google_songs = matched_google_songs + filtered_google_songs
songs_to_move = compare_song_collections(all_local_songs(), all_google_songs)
for filepath in songs_to_move:
rel_file_path = os.path.relpath(filepath, cli['input'][0])
removed_filepath = os.path.join(removed_dir, rel_file_path)
utils.make_sure_path_exists(os.path.dirname(removed_filepath), 0o700)
logger.info("Removing {0} ~> {1}".format(filepath, removed_filepath))
if not cli['dry-run']:
shutil.move(filepath, removed_filepath)
# clean up empty folders
logger.info(" ")
if not cli['dry-run']:
removeEmptyFolders(cli['input'][0], False)
if cli['playlists']:
logger.info("Moving Removed playlists...")
# path to move songs removed from google music
removed_playlists_dir = os.path.join(os.path.abspath(cli['removed']), cli['playlists'])
# local playlists after sync
local_playlists = [os.path.join(dp, f) for dp, dn, fn in os.walk(playlists_dir) for f in fn]
playlists_to_remove = list(set(local_playlists) - set(created_playlists))
for filepath in playlists_to_remove:
rel_file_path = os.path.relpath(filepath, playlists_dir)
removed_filepath = os.path.join(removed_playlists_dir, rel_file_path)
utils.make_sure_path_exists(os.path.dirname(removed_filepath), 0o700)
logger.info("Removing {0}".format(rel_file_path))
if not cli['dry-run']:
shutil.move(filepath, removed_filepath)
# clean up empty folders
logger.info(" ")
if not cli['dry-run']:
removeEmptyFolders(playlists_dir, False)
else:
matched_google_songs, _ = mmw.get_google_songs()
logger.info("")
matched_local_songs, songs_to_filter, songs_to_exclude = mmw.get_local_songs(
cli['input'], include_filters=include_filters, exclude_filters=exclude_filters,
all_includes=cli['all-includes'], all_excludes=cli['all-excludes'],
exclude_patterns=cli['exclude'], max_depth=cli['max-depth']
)
logger.info("\nFinding missing songs...")
songs_to_upload = compare_song_collections(matched_local_songs, matched_google_songs)
# Sort lists for sensible output.
songs_to_upload.sort()
songs_to_exclude.sort()
if cli['dry-run']:
logger.info("\nFound {0} song(s) to upload".format(len(songs_to_upload)))
if songs_to_upload:
logger.info("\nSongs to upload:\n")
for song in songs_to_upload:
logger.log(QUIET, song)
else:
logger.info("\nNo songs to upload")
if songs_to_filter:
logger.info("\nSongs to filter:\n")
for song in songs_to_filter:
logger.log(QUIET, song)
else:
logger.info("\nNo songs to filter")
if songs_to_exclude:
logger.info("\nSongs to exclude:\n")
for song in songs_to_exclude:
logger.log(QUIET, song)
else:
logger.info("\nNo songs to exclude")
else:
if songs_to_upload:
logger.info("\nUploading {0} song(s) to Google Music\n".format(len(songs_to_upload)))
mmw.upload(songs_to_upload, enable_matching=cli['match'], delete_on_success=cli['delete-on-success'])
else:
logger.info("\nNo songs to upload")
# Delete local files if they already exist on Google Music.
if cli['delete-on-success']:
for song in matched_local_songs:
try:
os.remove(song)
except:
logger.warning("Failed to remove {} after successful upload".format(song))
mmw.logout()
mcw.logout()
logger.info("\nAll done!")
