def main(): rewrite = "--rewrite" in sys.argv drop = dropbox.Dropbox() new_artists = set() for au_file in drop.tracks(): try: tpe1 = au_file.mutagen_id3["TPE1"].text[0] except: print "** file: %r" % au_file.path raise if artists.standardize(tpe1) is None: new_artists.add(tpe1) to_print = list(new_artists) if rewrite: to_print.extend(artists.all()) to_print.sort(key=artists.sort_key) output = None if rewrite: output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8") for tpe1 in to_print: if output: output.write(tpe1) output.write("\n") else: print tpe1.encode("utf-8") if rewrite: print "Artist whitelist updated"
def main(): rewrite = ("--rewrite" in sys.argv) drop = dropbox.Dropbox() new_artists = set() for au_file in drop.tracks(): try: tpe1 = au_file.mutagen_id3["TPE1"].text[0] except: print '** file: %r' % au_file.path raise if artists.standardize(tpe1) is None: new_artists.add(tpe1) to_print = list(new_artists) if rewrite: to_print.extend(artists.all()) to_print.sort(key=artists.sort_key) output = None if rewrite: output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8") for tpe1 in to_print: if output: output.write(tpe1) output.write("\n") else: print tpe1.encode("utf-8") if rewrite: print "Artist whitelist updated"
def main_generator(rewrite): drop = dropbox.Dropbox() new_artists = set() for au_file in drop.tracks(): try: tpe1 = au_file.mutagen_id3["TPE1"].text[0] except: cprint('** file: %r' % au_file.path) raise if artists.standardize(tpe1) is None: new_artists.add(tpe1) to_print = list(new_artists) if rewrite: to_print.extend(artists.all()) to_print.sort(key=artists.sort_key) output = None if rewrite: output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8") for tpe1 in to_print: if output: output.write(tpe1) output.write("\n") else: cprint(tpe1.encode("utf-8")) yield if rewrite: cprint('Artist whitelist updated', type='highlight') else: cprint('Found %d new artists' % len(to_print), type='success')
def main_generator(rewrite): drop = dropbox.Dropbox() new_artists = set() for au_file in drop.tracks(): try: tpe1 = au_file.mutagen_id3["TPE1"].text[0] except: cprint(u'** file: %r' % au_file.path) raise if artists.standardize(tpe1) is None: new_artists.add(tpe1) to_print = list(new_artists) if rewrite: to_print.extend(artists.all()) to_print.sort(key=artists.sort_key) output = None if rewrite: output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8") for tpe1 in to_print: if output: output.write(tpe1) output.write("\n") else: cprint(tpe1) yield if rewrite: cprint('Artist whitelist updated', type='success') else: cprint('Found %d new artists' % len(to_print), type='success')
def do_push_artists(self): # patch credentials if not request.headers.get('Authorization'): abort(401) else: auth = request.headers['Authorization'].lstrip('Basic ') username, password = base64.b64decode(auth).split(':') if username and password: conf.CHIRPRADIO_AUTH = '%s %s' % (username, password) chirpradio.connect() else: abort(401) dry_run = False # reload artists from file artists._init() # Find all of the library artists all_library_artists = set(artists.all()) # Find all of the artists in the cloud. all_chirpradio_artists = set() mapped = 0 t1 = time.time() for art in models.Artist.fetch_all(): if art.revoked: continue std_name = artists.standardize(art.name) if std_name != art.name: #print "Mapping %d: %s => %s" % (mapped, art.name, std_name) mapped += 1 art.name = std_name idx = search.Indexer() idx._transaction = art.parent_key() idx.add_artist(art) if not dry_run: idx.save() all_chirpradio_artists.add(art.name) to_push = list(all_library_artists.difference(all_chirpradio_artists)) Messages.add_message("Pushing %d artists" % len(to_push), 'warning') while to_push: # Push the artists in batches of 50 this_push = to_push[:50] to_push = to_push[50:] idx = search.Indexer() for name in this_push: #print name art = models.Artist.create(parent=idx.transaction, name=name) idx.add_artist(art) if not dry_run: idx.save() #print "+++++ Indexer saved" Messages.add_message("Artist push complete. OK!", 'success')
def main_generator(): chirpradio.connect() dry_run = False # Find all of the library artists all_library_artists = set(artists.all()) # Find all of the artists in the cloud. all_chirpradio_artists = set() mapped = 0 t1 = time.time() for art in models.Artist.fetch_all(): if art.revoked: continue std_name = artists.standardize(art.name) if std_name != art.name: cprint(u"Mapping {}: {} => {}".format(mapped, art.name, std_name)) mapped += 1 art.name = std_name idx = search.Indexer() idx._transaction = art.parent_key() idx.add_artist(art) if not dry_run: idx.save() all_chirpradio_artists.add(art.name) yield to_push = list(all_library_artists.difference(all_chirpradio_artists)) cprint("Pushing %d artists" % len(to_push)) while to_push: # Push the artists in batches of 50 this_push = to_push[:50] to_push = to_push[50:] idx = search.Indexer() for name in this_push: cprint(name) art = models.Artist.create(parent=idx.transaction, name=name) idx.add_artist(art) if not dry_run: idx.save() cprint("+++++ Indexer saved") yield
def add_artists(self): error = False drop = dropbox.Dropbox() new_artists = set() for au_file in drop.tracks(): try: tpe1 = au_file.mutagen_id3["TPE1"].text[0] except: Messages.add_messaage('** file: %r' % au_file.path, 'error') error = True # TODO propagate error to client raise if artists.standardize(tpe1) is None: new_artists.add(tpe1) # do not write if errors if not error and new_artists: to_print = list(new_artists) to_print.extend(artists.all()) to_print.sort(key=artists.sort_key) output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8") for tpe1 in to_print: output.write(tpe1) output.write("\n") output.close() # reload whitelist from file artists._init() message = "Artist whitelist updated.<br>New artists added:<br>" message += "<br>".join(list(new_artists)) Messages.add_message(message, 'success') # push to github self.push_to_github()
def _fix_file_tags(au_file): """Modify a file's tags to conform to CHIRP's tagging standards. Args: au_file: An AudioFile object Returns: A new mutagen.id3.ID3 object with the corrected tags. Raises: ImportFileError: if the tagging is broken or cannot be fixed. """ if constants.MUTAGEN_UFID_KEY in au_file.mutagen_id3: raise ImportFileError(["File already contains CHIRP UFID tag"]) new_id3 = mutagen.id3.ID3() # Build up a list of tags, stripping out ones that are not on our # whitelist. for tag in au_file.mutagen_id3.values(): if tag.FrameID in constants.ID3_TAG_BLACKLIST: raise ImportFileError(["Found blacklisted tag: %r" % tag]) if not (tag.FrameID in constants.ID3_TAG_WHITELIST and tag.FrameID not in constants.ID3_TAG_STRIPPED_ON_IMPORT): continue # Manually filter out TPOS # TODO(trow): We don't want to do this long-term. if tag.FrameID == "TPOS": continue # Standardize TPE tags, filtering out unknown artists at # TPE2 or lower. if tag.FrameID.startswith("TPE"): name_std = artists.standardize(unicode(tag)) if name_std is None: if tag.FrameID != "TPE1": sys.stderr.write( "*** Filtering %s %s\n" % (tag.FrameID, unicode(tag).encode("utf-8"))) continue raise ImportFileError( [u"Unknown artist %r in %s" % (unicode(tag), tag.FrameID)]) else: tag.text = [name_std] # If the TBPM tag is present and contains a string of the form # "xxx BPM", strip off the suffix. If xxx is not an integer, # round it off. If it is <= 0, discard the tag entirely. if tag.FrameID == "TBPM": tbpm = unicode(tag) if tbpm.endswith(" BPM"): tbpm = tbpm[:-4] try: tbpm = int(float(tbpm)) except ValueError: continue if tbpm <= 0: continue tag.text = [unicode(tbpm)] new_id3.add(tag) # Add our own TLEN tag. tlen_tag = mutagen.id3.TLEN(text=[unicode(au_file.duration_ms)], encoding=constants.DEFAULT_ID3_TEXT_ENCODING) new_id3.add(tlen_tag) # Add tags containing the number of frames and the frame size. frame_count_tag = mutagen.id3.TXXX( desc=constants.TXXX_FRAME_COUNT_DESCRIPTION, text=[unicode(au_file.frame_count)], encoding=constants.DEFAULT_ID3_TEXT_ENCODING) new_id3.add(frame_count_tag) frame_size_tag = mutagen.id3.TXXX( desc=constants.TXXX_FRAME_SIZE_DESCRIPTION, text=[unicode(au_file.frame_size)], encoding=constants.DEFAULT_ID3_TEXT_ENCODING) new_id3.add(frame_size_tag) # Add a TXXX tag with the album ID. txxx = mutagen.id3.TXXX(encoding=constants.DEFAULT_ID3_TEXT_ENCODING, desc=constants.TXXX_ALBUM_ID_DESCRIPTION, text=[unicode(au_file.album_id)]) new_id3.add(txxx) # Add a TFLT tag indicating that this is an MP3. tflt_tag = mutagen.id3.TFLT(text=[u"MPG/3"], encoding=constants.DEFAULT_ID3_TEXT_ENCODING) new_id3.add(tflt_tag) # Add the standard CHIRP TOWN tag. town_tag = mutagen.id3.TOWN(text=[constants.TOWN_FILE_OWNER], encoding=constants.DEFAULT_ID3_TEXT_ENCODING) new_id3.add(town_tag) return new_id3
def test_standardized_none_is_none(self): self.assertEqual(artists.standardize(None), None)
def test_real_data(self): self.assertTrue(len(artists._global_whitelist) > 2000) self.assertTrue(len(artists._global_mappings) >= 2) # Check some known whitelist items. for expected, raw in (("Bob Dylan", "bob dylan"), ("The Fall", "fall"), ): self.assertEqual(expected, artists.standardize(raw)) # Check a known mapping. for expected, raw in (("Gordon Staples", "Gordon Stapes"), ): self.assertEqual(expected, artists.standardize(raw)) # Repeatedly merge the global whitelist and mappings until the two # dicts stabilize. If merging caused anything to change, write # out corrected forms of the files, print a banner and a diff # to stdout, and cause the test to fail. whitelist, mappings = artists.merge_whitelist_and_mappings( artists._global_whitelist, artists._global_raw_mappings) fixed_whitelist_filename = artists._WHITELIST_FILE + ".fixed" fixed_mappings_filename = artists._MAPPINGS_FILE + ".fixed" # Delete these files if they already exist. for filename in (fixed_whitelist_filename, fixed_mappings_filename): try: os.unlink(filename) except OSError: pass test_should_succeed = True if whitelist != artists._global_whitelist: test_should_succeed = False print "\n\n" print "*" * 70 print "***" print "*** Whitelist is not properly normalized" print "***" print "*** Diff:" out = codecs.open(fixed_whitelist_filename, "w", "utf-8") for white in sorted(whitelist.values(), key=artists.sort_key): out.write(u"%s\n" % white) out.close() diff_cmd = "diff -u %s %s" % (artists._WHITELIST_FILE, fixed_whitelist_filename) sys.stdout.flush() os.system(diff_cmd) sys.stdout.flush() print "*" * 70 print "\n\n" if mappings != artists._global_raw_mappings: test_should_succeed = False print "\n\n" print "*" * 70 print "***" print "*** Mappings are not properly normalized" print "***" print "*** Diff:" out = codecs.open(fixed_mappings_filename, "w", "utf-8") for before in sorted(mappings, key=artists.sort_key): out.write(u"%s %s %s\n" % (before, artists._MAPPINGS_SEP, mappings[before])) out.close() diff_cmd = "diff -u %s %s" % (artists._MAPPINGS_FILE, fixed_mappings_filename) sys.stdout.flush() os.system(diff_cmd) sys.stdout.flush() print "*" * 70 print "\n\n" self.assertTrue(test_should_succeed)
def test_real_data(self): self.assertTrue(len(artists._global_whitelist) > 2000) self.assertTrue(len(artists._global_mappings) >= 2) # Check some known whitelist items. for expected, raw in ( ("Bob Dylan", "bob dylan"), ("The Fall", "fall"), ): self.assertEqual(expected, artists.standardize(raw)) # Check a known mapping. for expected, raw in (("Gordon Staples", "Gordon Stapes"), ): self.assertEqual(expected, artists.standardize(raw)) # Repeatedly merge the global whitelist and mappings until the two # dicts stabilize. If merging caused anything to change, write # out corrected forms of the files, print a banner and a diff # to stdout, and cause the test to fail. whitelist, mappings = artists.merge_whitelist_and_mappings( artists._global_whitelist, artists._global_raw_mappings) fixed_whitelist_filename = artists._WHITELIST_FILE + ".fixed" fixed_mappings_filename = artists._MAPPINGS_FILE + ".fixed" # Delete these files if they already exist. for filename in (fixed_whitelist_filename, fixed_mappings_filename): try: os.unlink(filename) except OSError: pass test_should_succeed = True if whitelist != artists._global_whitelist: test_should_succeed = False print "\n\n" print "*" * 70 print "***" print "*** Whitelist is not properly normalized" print "***" print "*** Diff:" out = codecs.open(fixed_whitelist_filename, "w", "utf-8") for white in sorted(whitelist.values(), key=artists.sort_key): out.write(u"%s\n" % white) out.close() diff_cmd = "diff -u %s %s" % (artists._WHITELIST_FILE, fixed_whitelist_filename) sys.stdout.flush() os.system(diff_cmd) sys.stdout.flush() print "*" * 70 print "\n\n" if mappings != artists._global_raw_mappings: test_should_succeed = False print "\n\n" print "*" * 70 print "***" print "*** Mappings are not properly normalized" print "***" print "*** Diff:" out = codecs.open(fixed_mappings_filename, "w", "utf-8") for before in sorted(mappings, key=artists.sort_key): out.write(u"%s %s %s\n" % (before, artists._MAPPINGS_SEP, mappings[before])) out.close() diff_cmd = "diff -u %s %s" % (artists._MAPPINGS_FILE, fixed_mappings_filename) sys.stdout.flush() os.system(diff_cmd) sys.stdout.flush() print "*" * 70 print "\n\n" self.assertTrue(test_should_succeed)