def testLabelAliases(self): fn = os.path.join(self.datadir, "022fe361-596c-43a0-8e22-bad712bb9548-aliases.xml") res = mbxml.parse_message(open(fn)) aliases = res["label"]["alias-list"] self.assertEqual(len(aliases), 4) a0 = aliases[0] self.assertEqual(a0["alias"], "EMI") self.assertEqual(a0["sort-name"], "EMI") a1 = aliases[1] self.assertEqual(a1["alias"], "EMI Records (UK)") self.assertEqual(a1["sort-name"], "EMI Records (UK)") fn = os.path.join(self.datadir, "e72fabf2-74a3-4444-a9a5-316296cbfc8d-aliases.xml") res = mbxml.parse_message(open(fn)) aliases = res["label"]["alias-list"] self.assertEqual(len(aliases), 1) a0 = aliases[0] self.assertEqual(a0["alias"], "Ki/oon Records Inc.") self.assertEqual(a0["sort-name"], "Ki/oon Records Inc.") self.assertEqual(a0["begin-date"], "2001-10") self.assertEqual(a0["end-date"], "2012-04")
def testWorkAliases(self): fn = os.path.join(self.datadir, "80737426-8ef3-3a9c-a3a6-9507afb93e93-aliases.xml") res = mbxml.parse_message(open(fn)) aliases = res["work"]["alias-list"] self.assertEqual(len(aliases), 2) a0 = aliases[0] self.assertEqual(a0["alias"], 'Symphonie Nr. 3 Es-Dur, Op. 55 "Eroica"') self.assertEqual(a0["sort-name"], 'Symphonie Nr. 3 Es-Dur, Op. 55 "Eroica"') a1 = aliases[1] self.assertEqual(a1["alias"], 'Symphony No. 3, Op. 55 "Eroica"') self.assertEqual(a1["sort-name"], 'Symphony No. 3, Op. 55 "Eroica"') fn = os.path.join(self.datadir, "3d7c7cd2-da79-37f4-98b8-ccfb1a4ac6c4-aliases.xml") res = mbxml.parse_message(open(fn)) aliases = res["work"]["alias-list"] self.assertEqual(len(aliases), 10) a0 = aliases[0] self.assertEqual(a0["alias"], "Adagio from Symphony No. 2 in E minor, Op. 27") self.assertEqual(a0["sort-name"], "Adagio from Symphony No. 2 in E minor, Op. 27")
def testTypesResult(self): fn = os.path.join(self.datadir, "f52bc6a1-c848-49e6-85de-f8f53459a624.xml") res = mbxml.parse_message(open(fn))["release-group"] self.assertEqual("Soundtrack", res["type"]) self.assertEqual("Album", res["primary-type"]) self.assertEqual(["Soundtrack"], res["secondary-type-list"])
def testTypesExist(self): fn = os.path.join(self.datadir, "f52bc6a1-c848-49e6-85de-f8f53459a624.xml") res = mbxml.parse_message(open(fn))["release-group"] self.assertTrue("type" in res) self.assertTrue("primary-type" in res) self.assertTrue("secondary-type-list" in res)
def testFields(self): fn = os.path.join(os.path.dirname(__file__), "data", "search-recording.xml") res = mbxml.parse_message(open(fn)) self.assertEqual(25, len(res["recording-list"])) self.assertEqual(1258, res["recording-count"]) one = res["recording-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(os.path.dirname(__file__), "data", "search-label.xml") res = mbxml.parse_message(open(fn)) self.assertEqual(1, len(res["label-list"])) one = res["label-list"][0] self.assertEqual("100", one["ext:score"])
def testTrackLength(self): """ Test that if there is a track length, then `track_or_recording_length` has that, but if not then fill the value from the recording length """ fn = os.path.join(self.datadir, "b66ebe6d-a577-4af8-9a2e-a029b2147716-recordings.xml") res = mbxml.parse_message(open(fn)) tracks = res["release"]["medium-list"][0]["track-list"] # No track length and recording length t1 = tracks[0] self.assertTrue("length" not in t1) self.assertEqual("180000", t1["recording"]["length"]) self.assertEqual("180000", t1["track_or_recording_length"]) # Track length and recording length same t2 = tracks[1] self.assertEqual("279000", t2["length"]) self.assertEqual("279000", t2["recording"]["length"]) self.assertEqual("279000", t2["track_or_recording_length"]) # Track length and recording length different t3 = tracks[2] self.assertEqual("60000", t3["length"]) self.assertEqual("80000", t3["recording"]["length"]) self.assertEqual("60000", t3["track_or_recording_length"]) # No track lengths t4 = tracks[3] self.assertTrue("length" not in t4["recording"]) self.assertTrue("length" not in t4) self.assertTrue("track_or_recording_length" not in t4)
def testFields(self): fn = os.path.join(DATA_DIR, "search-work.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["work-list"])) self.assertEqual(174, res["work-count"]) one = res["work-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-recording.xml") with open(fn, 'rb') as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["recording-list"])) self.assertEqual(1258, res["recording-count"]) one = res["recording-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(os.path.dirname(__file__), "data", "search-recording.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["recording-list"])) self.assertEqual(1258, res["recording-count"]) one = res["recording-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(os.path.dirname(__file__), "data", "search-release-group.xml") res = mbxml.parse_message(open(fn)) self.assertEqual(25, len(res["release-group-list"])) self.assertEqual(14641, res["release-group-count"]) one = res["release-group-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-release-group.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["release-group-list"])) self.assertEqual(14641, res["release-group-count"]) one = res["release-group-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-label.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(1, len(res["label-list"])) self.assertEqual(1, res["label-count"]) one = res["label-list"][0] self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(os.path.dirname(__file__), "data", "search-label.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(1, len(res["label-list"])) self.assertEqual(1, res["label-count"]) one = res["label-list"][0] self.assertEqual("100", one["ext:score"])
def testTrackNumber(self): """ Test that track number (number or text) and track position (always an increasing number) are both read properly """ fn = os.path.join(self.datadir, "212895ca-ee36-439a-a824-d2620cd10461-recordings.xml") res = mbxml.parse_message(open(fn)) tracks = res["release"]["medium-list"][0]["track-list"] # This release doesn't number intro tracks as numbered tracks, # so position and number get 'out of sync' self.assertEqual(['1', '2', '3'], [t["position"] for t in tracks[:3]]) self.assertEqual(['', '1', '2'], [t["number"] for t in tracks[:3]]) fn = os.path.join(self.datadir, "a81f3c15-2f36-47c7-9b0f-f684a8b0530f-recordings.xml") res = mbxml.parse_message(open(fn)) tracks = res["release"]["medium-list"][0]["track-list"] self.assertEqual(['1', '2'], [t["position"] for t in tracks]) self.assertEqual(['A', 'B'], [t["number"] for t in tracks])
def get_from_mb_xml(cls, xml, disc_id): """Parse Musicbrainz XML for a given disc_id. The XML should have been returned from a "/ws/2/discid/DISC_ID?inc=recordings artist" query. This returns a list of matching discs. """ return cls.get_from_mb_dict(mbxml.parse_message(xml), disc_id)
def testFields(self): fn = os.path.join(os.path.dirname(__file__), "data", "search-artist.xml") res = mbxml.parse_message(open(fn)) self.assertEqual(25, len(res["artist-list"])) one = res["artist-list"][0] self.assertEqual(9, len(one.keys())) # Score is a key that is only in search results - # so check for it here self.assertEqual("100", one["ext:score"])
def testArtistAliases(self): fn = os.path.join(self.datadir, "0e43fe9d-c472-4b62-be9e-55f971a023e1-aliases.xml") res = mbxml.parse_message(open(fn)) aliases = res["artist"]["alias-list"] self.assertEqual(len(aliases), 28) a0 = aliases[0] self.assertEqual(a0["alias"], "Prokofief") self.assertEqual(a0["sort-name"], "Prokofief") a17 = aliases[17] self.assertEqual(a17["alias"], "Sergei Sergeyevich Prokofiev") self.assertEqual(a17["sort-name"], "Prokofiev, Sergei Sergeyevich") self.assertEqual(a17["locale"], "en") self.assertEqual(a17["primary"], "primary") fn = os.path.join(self.datadir, "2736bad5-6280-4c8f-92c8-27a5e63bbab2-aliases.xml") res = mbxml.parse_message(open(fn)) self.assertFalse("alias-list" in res["artist"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-instrument.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(23, len(res["instrument-list"])) self.assertEqual(23, res["instrument-count"]) one = res["instrument-list"][0] self.assertEqual("100", one["ext:score"]) end = res["instrument-list"][-1] self.assertEqual("29", end["ext:score"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-event.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(3, res["event-count"]) self.assertEqual(3, len(res["event-list"])) one = res["event-list"][0] self.assertEqual("100", one["ext:score"]) two = res["event-list"][1] self.assertEqual(1, len(two["place-relation-list"]))
def testFields(self): fn = os.path.join(DATA_DIR, "search-artist.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["artist-list"])) self.assertEqual(349, res["artist-count"]) one = res["artist-list"][0] self.assertEqual(9, len(one.keys())) # Score is a key that is only in search results - # so check for it here self.assertEqual("100", one["ext:score"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-place.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(14, res["place-count"]) self.assertEqual(14, len(res["place-list"])) one = res["place-list"][0] self.assertEqual("100", one["ext:score"]) two = res["place-list"][1] self.assertEqual("63", two["ext:score"]) self.assertEqual("Southampton", two["disambiguation"])
def mb_parser_xml(resp): """Return a Python dict representing the XML response""" # Parse the response. try: return mbxml.parse_message(resp) except UnicodeError as exc: raise ResponseError(cause=exc) except Exception as exc: if isinstance(exc, ETREE_EXCEPTIONS): raise ResponseError(cause=exc) else: raise
def testFields(self): fn = os.path.join(DATA_DIR, "search-release.xml") with open(fn) as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["release-list"])) self.assertEqual(16739, res["release-count"]) one = res["release-list"][0] self.assertEqual("100", one["ext:score"]) # search results have a medium-list/track-count element self.assertEqual(4, one["medium-track-count"]) self.assertEqual(1, one["medium-count"]) self.assertEqual("CD", one["medium-list"][0]["format"])
def testFields(self): fn = os.path.join(DATA_DIR, "search-release.xml") with open(fn, 'rb') as msg: res = mbxml.parse_message(msg) self.assertEqual(25, len(res["release-list"])) self.assertEqual(16739, res["release-count"]) one = res["release-list"][0] self.assertEqual("100", one["ext:score"]) # search results have a medium-list/track-count element self.assertEqual(4, one["medium-track-count"]) self.assertEqual(1, one["medium-count"]) self.assertEqual("CD", one["medium-list"][0]["format"])
def testArtistCredit(self): """ If the artist credit is the same in the track and recording, make sure that the information is replicated in both objects, otherwise have distinct ones. """ # If no artist-credit in the track, copy in the recording one fn = os.path.join(self.datadir, "833d4c3a-2635-4b7a-83c4-4e560588f23a-recordings+artist-credits.xml") res = mbxml.parse_message(open(fn)) tracks = res["release"]["medium-list"][0]["track-list"] t1 = tracks[1] self.assertEqual(t1["artist-credit"], t1["recording"]["artist-credit"]) self.assertEqual("JT Bruce", t1["artist-credit-phrase"]) self.assertEqual(t1["recording"]["artist-credit-phrase"], t1["artist-credit-phrase"]) # Recording AC is different to track AC fn = os.path.join(self.datadir, "fbe4490e-e366-4da2-a37a-82162d2f41a9-recordings+artist-credits.xml") res = mbxml.parse_message(open(fn)) tracks = res["release"]["medium-list"][0]["track-list"] t1 = tracks[1] self.assertNotEqual(t1["artist-credit"], t1["recording"]["artist-credit"]) self.assertEqual("H. Lichner", t1["artist-credit-phrase"]) self.assertNotEqual(t1["recording"]["artist-credit-phrase"], t1["artist-credit-phrase"])
def getMetaData(self, releaseId): """Load metadata from disk""" xmlPath = self._get_xml_path(releaseId) if (not os.path.isfile(xmlPath)): logging.error("No XML metadata for %s", releaseId) return None with open(xmlPath, 'r') as xmlf: metaxml = xmlf.read() try: metadata = mbxml.parse_message(metaxml) except UnicodeError as exc: raise ResponseError(cause=exc) except Exception as exc: if isinstance(exc, ETREE_EXCEPTIONS): logging.error("Got some bad XML for %s!", releaseId) return else: raise return metadata
def _mb_request(path, method='GET', auth_required=False, client_required=False, args=None, data=None, body=None): """Makes a request for the specified `path` (endpoint) on /ws/2 on the globally-specified hostname. Parses the responses and returns the resulting object. `auth_required` and `client_required` control whether exceptions should be raised if the client and username/password are left unspecified, respectively. """ if args is None: args = {} else: args = dict(args) or {} if _useragent == "": raise UsageError("set a proper user-agent with " "set_useragent(\"application name\", \"application version\", \"contact info (preferably URL or email for your application)\")") if client_required: args["client"] = _client headers = {} if body: headers['Content-Type'] = 'application/xml; charset=UTF-8' else: # Explicitly indicate zero content length if no request data # will be sent (avoids HTTP 411 error). headers['Content-Length'] = '0' req = requests.Request( method, 'http://{0}/ws/2/{1}'.format(hostname, path), params=args, auth=HTTPDigestAuth(user, password) if auth_required else None, headers=headers, data=body, ) # Make request (with retries). session = requests.Session() adapter = requests.adapters.HTTPAdapter(max_retries=8) session.mount('http://', adapter) session.mount('https://', adapter) try: resp = session.send(req.prepare(), allow_redirects=True) except requests.RequestException as exc: raise NetworkError(cause=exc) if resp.status_code != 200: raise ResponseError( 'API responded with code {0}'.format(resp.status_code) ) # Parse the response. try: return mbxml.parse_message(resp.content) except UnicodeError as exc: raise ResponseError(cause=exc) except Exception as exc: if isinstance(exc, ETREE_EXCEPTIONS): raise ResponseError(cause=exc) else: raise
'releases list)') con.commit() fileList = os.listdir(rootPath) widgets = ["Releases: ", progressbar.Bar(marker="=", left="[", right="]"), " ", progressbar.Percentage() ] if len(fileList): pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(fileList)).start() for relId in fileList: metaPath = os.path.join(rootPath, relId, 'metadata.xml') #with codecs.open(metaPath, encoding='utf-8') as f: with open(metaPath, 'r') as f: metaXml = f.read() try: metadata = mbxml.parse_message(metaXml)['release'] except UnicodeError as exc: raise mb.ResponseError(cause=exc) except Exception as exc: if isinstance(exc, ETREE_EXCEPTIONS): logging.error("Got some bad XML for %s!", releaseId) else: raise ed = mbcat.extradata.ExtraData(relId) try: ed.load() except IOError as e: ed = None cur.execute('insert into releases(id, meta, sortstring, metatime, purchases, added, lent, listened, digital, count, comment, rating) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
def _mb_request(path, method='GET', auth_required=False, client_required=False, args=None, data=None, body=None): """Makes a request for the specified `path` (endpoint) on /ws/2 on the globally-specified hostname. Parses the responses and returns the resulting object. `auth_required` and `client_required` control whether exceptions should be raised if the client and username/password are left unspecified, respectively. """ if args is None: args = {} else: args = dict(args) or {} if _useragent == "": raise UsageError("set a proper user-agent with " "set_useragent(\"application name\", \"application version\", \"contact info (preferably URL or email for your application)\")") if client_required: args["client"] = _client # Encode Unicode arguments using UTF-8. for key, value in args.items(): if isinstance(value, compat.unicode): args[key] = value.encode('utf8') # Construct the full URL for the request, including hostname and # query string. url = compat.urlunparse(( 'http', hostname, '/ws/2/%s' % path, '', compat.urlencode(args), '' )) _log.debug("%s request for %s" % (method, url)) # Set up HTTP request handler and URL opener. httpHandler = compat.HTTPHandler(debuglevel=0) handlers = [httpHandler] # Add credentials if required. if auth_required: _log.debug("Auth required for %s" % url) if not user: raise UsageError("authorization required; " "use auth(user, pass) first") passwordMgr = _RedirectPasswordMgr() authHandler = _DigestAuthHandler(passwordMgr) authHandler.add_password("musicbrainz.org", (), user, password) handlers.append(authHandler) opener = compat.build_opener(*handlers) # Make request. req = _MusicbrainzHttpRequest(method, url, data) req.add_header('User-Agent', _useragent) _log.debug("requesting with UA %s" % _useragent) if body: req.add_header('Content-Type', 'application/xml; charset=UTF-8') elif not data and not req.has_header('Content-Length'): # Explicitly indicate zero content length if no request data # will be sent (avoids HTTP 411 error). req.add_header('Content-Length', '0') resp = _safe_read(opener, req, body) # Parse the response. try: return mbxml.parse_message(resp) except UnicodeError as exc: raise ResponseError(cause=exc) except Exception as exc: if isinstance(exc, ETREE_EXCEPTIONS): raise ResponseError(cause=exc) else: raise
def parse_response_body(self, body): return mbxml.parse_message(BytesIO(body))
widgets = [ "Releases: ", progressbar.Bar(marker="=", left="[", right="]"), " ", progressbar.Percentage() ] if len(fileList): pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(fileList)).start() for relId in fileList: metaPath = os.path.join(rootPath, relId, 'metadata.xml') #with codecs.open(metaPath, encoding='utf-8') as f: with open(metaPath, 'r') as f: metaXml = f.read() try: metadata = mbxml.parse_message(metaXml)['release'] except UnicodeError as exc: raise mb.ResponseError(cause=exc) except Exception as exc: if isinstance(exc, ETREE_EXCEPTIONS): logging.error("Got some bad XML for %s!", releaseId) else: raise ed = mbcat.extradata.ExtraData(relId) try: ed.load() except IOError as e: ed = None cur.execute(
def digestXml(self, releaseId, meta_xml): self.digestMetaDict(releaseId, mbxml.parse_message(meta_xml))