def check_success(cls, response, msg):
if response.status_code != 200:
if msg == {'Error': 'BadAuthentication'}:
raise CallFailure('Invalid login credentials', cls.__name__)
log.warning(
"Received strange login response: %r"
"\n\nYou may need to enable less secure logins:\n"
" https://www.google.com/settings/security/lesssecureapps",
msg)
raise CallFailure(
"status code %s != 200, full response: %r " %
(response.status_code, msg), cls.__name__)
def check_success(cls, response, msg):
if msg.status != download_pb2.GetTracksToExportResponse.OK:
enum_desc = download_pb2._GETTRACKSTOEXPORTRESPONSE.enum_types[0]
res_name = enum_desc.values_by_number[msg.status].name
raise CallFailure(
"Track export (list) error code %s: %s." %
(msg.status, res_name), cls.__name__)
def perform(cls, session, *args, **kwargs):
"""Send, parse, validate and check success of this call.
*args and **kwargs are passed to protocol.build_transaction.
:param session: a PlaySession used to send this request.
"""
#TODO link up these docs
call_name = cls.__name__
if cls.gets_logged:
log.debug(
"%s(args=%s, kwargs=%s)", call_name,
[utils.truncate(a) for a in args],
dict((k, utils.truncate(v)) for (k, v) in kwargs.items()))
else:
log.debug("%s(<does not get logged>)", call_name)
req_kwargs = cls.build_request(*args, **kwargs)
response = session.send(req_kwargs, cls.required_auth)
#TODO check return code
try:
msg = cls.parse_response(response)
except ParseException:
if cls.gets_logged:
log.exception("couldn't parse %s response: %r", call_name,
response.content)
raise CallFailure(
"the server's response could not be understood."
" The call may still have succeeded, but it's unlikely.",
call_name)
if cls.gets_logged:
log.debug(cls.filter_response(msg))
try:
#order is important; validate only has a schema for a successful response
cls.check_success(response, msg)
cls.validate(response, msg)
except CallFailure:
raise
except ValidationException:
#TODO trim the response if it's huge
if cls.gets_logged:
msg_fmt = (
"the following response format for %s was not recognized."
"\nIf there has not been a compatibility update reported"
"[here](http://goo.gl/jTKNb),"
" please [create an issue](http://goo.gl/qbAW8) that includes"
" the following raw response:\n%r\n"
"\nA traceback follows:\n")
log.exception(msg_fmt, call_name, msg)
return msg
def check_success(cls, response, msg):
if msg.HasField('auth_status'
) and msg.auth_status != upload_pb2.UploadResponse.OK:
enum_desc = upload_pb2._UPLOADRESPONSE.enum_types[1]
res_name = enum_desc.values_by_number[msg.auth_status].name
raise CallFailure(
"Upload auth error code %s: %s."
" See http://goo.gl/O6xe7 for more information. " %
(msg.auth_status, res_name), cls.__name__)
def check_success(cls, response, msg):
if ('error' in msg or
not all([d.get('response_code', None) in ('OK', 'CONFLICT')
for d in msg.get('mutate_response', [])])):
raise CallFailure('The server reported failure while'
' changing the requested resource.'
" If this wasn't caused by invalid arguments"
' or server flakiness,'
' please open an issue.',
cls.__name__)
def check_success(cls, res):
#Failed responses always have a success=False key.
#Some successful responses do not have a success=True key, however.
#TODO remove utils.call_succeeded
if 'success' in res and not res['success']:
raise CallFailure(
"the server reported failure. This is usually"
"caused by bad arguments, but can also happen if requests"
"are made too quickly (eg creating a playlist then"
"modifying it before the server has created it)", cls.__name__)
def get_stream_url(self, id):
try:
stream_url = self._mobileclient.get_stream_url(id, self._device)
except NotLoggedIn:
if self.authenticate():
stream_url = self._mobileclient.get_stream_url(id, self._device)
else:
return ''
except CallFailure:
raise CallFailure('Could not play song with id: ' + id, 'get_stream_url')
return stream_url
def _make_call(self, protocol, *args, **kwargs):
"""Returns the response of a protocol.Call.
Additional kw/args are passed to protocol.build_transaction."""
#TODO link up these docs
call_name = protocol.__name__
self.log.debug("%s(args=%s, kwargs=%s)", call_name,
[utils.truncate(a) for a in args],
{k: utils.truncate(v)
for (k, v) in kwargs.items()})
request = protocol.build_request(*args, **kwargs)
response = self.session.send(request, protocol.get_auth(),
protocol.session_options)
#TODO check return code
try:
msg = protocol.parse_response(response)
except ParseException:
self.log.exception("couldn't parse %s response: %r", call_name,
response.content)
raise CallFailure(
"the server's response could not be understood."
" The call may still have succeeded, but it's unlikely.",
call_name)
self.log.debug(protocol.filter_response(msg))
try:
#order is important; validate only has a schema for a successful response
protocol.check_success(msg)
protocol.validate(msg)
except CallFailure:
raise
except ValidationException:
#TODO link to some protocol for reporting this
self.log.exception(
"please report the following unknown response format for %s: %r",
call_name, msg)
return msg
def check_success(cls, response, msg):
if response.status_code != 200:
raise CallFailure(('status code %s != 200' % response.status_code),
cls.__name__)
if 'xt' not in response.cookies:
raise CallFailure('did not receieve xt cookies', cls.__name__)
def perform(cls, session, validate, *args, **kwargs):
"""Send, parse, validate and check success of this call.
*args and **kwargs are passed to protocol.build_transaction.
:param session: a PlaySession used to send this request.
:param validate: if False, do not validate
"""
# TODO link up these docs
call_name = cls.__name__
if cls.gets_logged:
log.debug("%s(args=%s, kwargs=%s)",
call_name,
[utils.truncate(a) for a in args],
dict((k, utils.truncate(v)) for (k, v) in kwargs.items())
)
else:
log.debug("%s(<omitted>)", call_name)
req_kwargs = cls.build_request(*args, **kwargs)
response = session.send(req_kwargs, cls.required_auth)
# TODO trim the logged response if it's huge?
safe_req_kwargs = req_kwargs.copy()
if safe_req_kwargs.get('headers', {}).get('Authorization', None) is not None:
safe_req_kwargs['headers']['Authorization'] = '<omitted>'
if cls.fail_on_non_200:
try:
response.raise_for_status()
except requests.HTTPError as e:
err_msg = str(e)
if cls.gets_logged:
err_msg += "\n(requests kwargs: %r)" % (safe_req_kwargs)
err_msg += "\n(response was: %r)" % response.text
raise CallFailure(err_msg, call_name)
try:
parsed_response = cls.parse_response(response)
except ParseException:
err_msg = ("the server's response could not be understood."
" The call may still have succeeded, but it's unlikely.")
if cls.gets_logged:
err_msg += "\n(requests kwargs: %r)" % (safe_req_kwargs)
err_msg += "\n(response was: %r)" % response.text
log.exception("could not parse %s response: %r", call_name, response.text)
else:
log.exception("could not parse %s response: (omitted)", call_name)
raise CallFailure(err_msg, call_name)
if cls.gets_logged:
log.debug(cls.filter_response(parsed_response))
try:
# order is important; validate only has a schema for a successful response
cls.check_success(response, parsed_response)
if validate:
cls.validate(response, parsed_response)
except CallFailure as e:
if not cls.gets_logged:
raise
# otherwise, reraise a new exception with our req/res context
err_msg = ("{e_message}\n"
"(requests kwargs: {req_kwargs!r})\n"
"(response was: {content!r})").format(
e_message=str(e),
req_kwargs=safe_req_kwargs,
content=response.text)
raise_from(CallFailure(err_msg, e.callname), e)
except ValidationException as e:
# TODO shouldn't be using formatting
err_msg = "the response format for %s was not recognized." % call_name
err_msg += "\n\n%s\n" % e
if cls.gets_logged:
raw_response = response.text
if len(raw_response) > 10000:
raw_response = raw_response[:10000] + '...'
err_msg += ("\nFirst, try the develop branch."
" If you can recreate this error with the most recent code"
" please [create an issue](http://goo.gl/qbAW8) that includes"
" the above ValidationException"
" and the following request/response:\n%r\n\n%r\n"
"\nA traceback follows:\n") % (safe_req_kwargs, raw_response)
log.exception(err_msg)
return parsed_response
def check_succes(cls, response, msg):
if response.status_code == 200:
raise CallFailure("status code %s != 200" % response.status_code, cls.__name__)
def check_success(cls, response, msg):
if msg.HasField('getjobs_response'
) and not msg.getjobs_response.get_tracks_success:
raise CallFailure('get_tracks_success == False', cls.__name__)
def check_success(cls, res):
if res.HasField('auth_status'
) and res.auth_status != upload_pb2.UploadResponse.OK:
raise CallFailure(
"Too many uploader ids/machines are registered."
" See http://goo.gl/O6xe7 for more information.", cls.__name__)
def upload(self, filepaths, transcode_quality=3, enable_matching=False):
"""Uploads the given filepaths.
Any non-mp3 files will be `transcoded with avconv
<https://github.com/simon-weber/Unofficial-Google-Music-API/
blob/develop/gmusicapi/utils/utils.py#L18>`__ before being uploaded.
Return a 3-tuple ``(uploaded, matched, not_uploaded)`` of dictionaries, eg::
(
{'<filepath>': '<new server id>'}, # uploaded
{'<filepath>': '<new server id>'}, # matched
{'<filepath>': '<reason, eg ALREADY_UPLOADED>'} # not uploaded
)
:param filepaths: a list of filepaths, or a single filepath.
:param transcode_quality: if int, pass to avconv ``-qscale`` for libmp3lame
(lower-better int, roughly corresponding to `hydrogenaudio -vX settings
<http://wiki.hydrogenaudio.org/index.php?title=LAME#Recommended_encoder_settings>`__).
If string, pass to avconv ``-ab`` (eg ``'128k'`` for an average bitrate of 128k). The
default is ~175kbs vbr.
:param enable_matching: if ``True``, attempt to use `scan and match
<http://support.google.com/googleplay/bin/answer.py?hl=en&answer=2920799&topic=2450455>`__
to avoid uploading every song.
**WARNING**: currently, mismatched songs can *not* be fixed with the 'Fix Incorrect Match'
button or :func:`report_incorrect_match`. They would have to be deleted and reuploaded
with the Music Manager.
Fixing matches from gmusicapi will be supported in a future release; see issue `#89
<https://github.com/simon-weber/Unofficial-Google-Music-API/issues/89>`__.
All Google-supported filetypes are supported; see `Google's documentation
<http://support.google.com/googleplay/bin/answer.py?hl=en&answer=1100462>`__.
Unlike Google's Music Manager, this function will currently allow the same song to
be uploaded more than once if its tags are changed. This is subject to change in the future.
If ``PERMANENT_ERROR`` is given as a not_uploaded reason, attempts to reupload will never
succeed. The file will need to be changed before the server will reconsider it; the easiest
way is to change metadata tags (it's not important that the tag be uploaded, just that the
contents of the file change somehow).
"""
if self.uploader_id is None or self.uploader_name is None:
raise NotLoggedIn("Not authenticated as an upload device;"
" run Api.login(...perform_upload_auth=True...)"
" first.")
#TODO there is way too much code in this function.
#To return.
uploaded = {}
matched = {}
not_uploaded = {}
#Gather local information on the files.
local_info = {} # {clientid: (path, Track)}
for path in filepaths:
try:
track = musicmanager.UploadMetadata.fill_track_info(path)
except BaseException as e:
log.exception("problem gathering local info of '%r'", path)
user_err_msg = str(e)
if 'Non-ASCII strings must be converted to unicode' in str(e):
#This is a protobuf-specific error; they require either ascii or unicode.
#To keep behavior consistent, make no effort to guess - require users
# to decode first.
user_err_msg = (
"nonascii bytestrings must be decoded to unicode"
" (error: '%s')" % err_msg)
not_uploaded[path] = user_err_msg
else:
local_info[track.client_id] = (path, track)
if not local_info:
return uploaded, matched, not_uploaded
#TODO allow metadata faking
#Upload metadata; the server tells us what to do next.
res = self._make_call(musicmanager.UploadMetadata,
[track for (path, track) in local_info.values()],
self.uploader_id)
#TODO checking for proper contents should be handled in verification
md_res = res.metadata_response
responses = [r for r in md_res.track_sample_response]
sample_requests = [req for req in md_res.signed_challenge_info]
#Send scan and match samples if requested.
for sample_request in sample_requests:
path, track = local_info[
sample_request.challenge_info.client_track_id]
try:
res = self._make_call(musicmanager.ProvideSample, path,
sample_request, track, self.uploader_id)
except (IOError, ValueError) as e:
log.warning(
"couldn't create scan and match sample for '%s': %s", path,
str(e))
not_uploaded[path] = str(e)
else:
responses.extend(res.sample_response.track_sample_response)
#Read sample responses and prep upload requests.
to_upload = {} # {serverid: (path, Track, do_not_rematch?)}
for sample_res in responses:
path, track = local_info[sample_res.client_track_id]
if sample_res.response_code == upload_pb2.TrackSampleResponse.MATCHED:
log.info("matched '%s' to sid %s", path,
sample_res.server_track_id)
if enable_matching:
matched[path] = sample_res.server_track_id
else:
# request a reupload session (ie, hit 'fix incorrect match').
time.sleep(10) # wait for upload servers to sync
try:
self._make_call(webclient.ReportBadSongMatch,
[sample_res.server_track_id])
#Wait for server to register our request.
retries = 0
while retries < 5:
jobs = self._make_call(musicmanager.GetUploadJobs,
self.uploader_id)
matching = [
job for job in
jobs.getjobs_response.tracks_to_upload
if (job.client_id == sample_res.client_track_id
and job.status ==
upload_pb2.TracksToUpload.FORCE_REUPLOAD)
]
if matching:
reup_sid = matching[0].server_id
break
log.debug("wait for reup job (%s)", retries)
time.sleep(2)
retries += 1
else:
raise CallFailure(
"could not get reupload/rematch job for '%s'" %
path, 'GetUploadJobs')
except CallFailure as e:
log.exception(
"'%s' was matched without matching enabled", path)
matched[path] = sample_res.server_track_id
else:
log.info("will reupload '%s'", path)
to_upload[reup_sid] = (path, track, True)
elif sample_res.response_code == upload_pb2.TrackSampleResponse.UPLOAD_REQUESTED:
to_upload[sample_res.server_track_id] = (path, track, False)
else:
#Report the symbolic name of the response code enum.
enum_desc = upload_pb2._TRACKSAMPLERESPONSE.enum_types[0]
res_name = enum_desc.values_by_number[
sample_res.response_code].name
err_msg = "TrackSampleResponse code %s: %s" % (
sample_res.response_code, res_name)
log.warning("upload of '%s' rejected: %s", path, err_msg)
not_uploaded[path] = err_msg
#Send upload requests.
if to_upload:
#TODO reordering requests could avoid wasting time waiting for reup sync
self._make_call(musicmanager.UpdateUploadState, 'start',
self.uploader_id)
for server_id, (path, track, do_not_rematch) in to_upload.items():
#It can take a few tries to get an session.
should_retry = True
attempts = 0
while should_retry and attempts < 10:
session = self._make_call(musicmanager.GetUploadSession,
self.uploader_id, len(uploaded),
track, path, server_id,
do_not_rematch)
attempts += 1
got_session, error_details = \
musicmanager.GetUploadSession.process_session(session)
if got_session:
log.info("got an upload session for '%s'", path)
break
should_retry, reason, error_code = error_details
log.debug(
"problem getting upload session: %s\ncode=%s retrying=%s",
reason, error_code, should_retry)
if error_code == 200 and do_not_rematch:
#reupload requests need to wait on a server sync
#200 == already uploaded, so force a retry in this case
should_retry = True
time.sleep(6) # wait before retrying
else:
err_msg = "GetUploadSession error %s: %s" % (error_code,
reason)
log.warning("giving up on upload session for '%s': %s",
path, err_msg)
not_uploaded[path] = err_msg
continue # to next upload
#got a session, do the upload
#this terribly inconsistent naming isn't my fault: Google--
session = session['sessionStatus']
external = session['externalFieldTransfers'][0]
session_url = external['putInfo']['url']
content_type = external.get('content_type', 'audio/mpeg')
if track.original_content_type != locker_pb2.Track.MP3:
try:
log.info("transcoding '%s' to mp3", path)
contents = utils.transcode_to_mp3(
path, quality=transcode_quality)
except (IOError, ValueError) as e:
log.warning("error transcoding %s: %s", path, e)
not_uploaded[path] = "transcoding error: %s" % e
continue
else:
with open(path, 'rb') as f:
contents = f.read()
upload_response = self._make_call(musicmanager.UploadFile,
session_url, content_type,
contents)
success = upload_response.get('sessionStatus', {}).get('state')
if success:
uploaded[path] = server_id
else:
#404 == already uploaded? serverside check on clientid?
log.debug(
"could not finalize upload of '%s'. response: %s",
path, upload_response)
not_uploaded[
path] = 'could not finalize upload; details in log'
self._make_call(musicmanager.UpdateUploadState, 'stopped',
self.uploader_id)
return uploaded, matched, not_uploaded
