def main():
rewrite = "--rewrite" in sys.argv
drop = dropbox.Dropbox()
new_artists = set()
for au_file in drop.tracks():
try:
tpe1 = au_file.mutagen_id3["TPE1"].text[0]
except:
print "** file: %r" % au_file.path
raise
if artists.standardize(tpe1) is None:
new_artists.add(tpe1)
to_print = list(new_artists)
if rewrite:
to_print.extend(artists.all())
to_print.sort(key=artists.sort_key)
output = None
if rewrite:
output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
for tpe1 in to_print:
if output:
output.write(tpe1)
output.write("\n")
else:
print tpe1.encode("utf-8")
if rewrite:
print "Artist whitelist updated"
def main():
rewrite = ("--rewrite" in sys.argv)
drop = dropbox.Dropbox()
new_artists = set()
for au_file in drop.tracks():
try:
tpe1 = au_file.mutagen_id3["TPE1"].text[0]
except:
print '** file: %r' % au_file.path
raise
if artists.standardize(tpe1) is None:
new_artists.add(tpe1)
to_print = list(new_artists)
if rewrite:
to_print.extend(artists.all())
to_print.sort(key=artists.sort_key)
output = None
if rewrite:
output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
for tpe1 in to_print:
if output:
output.write(tpe1)
output.write("\n")
else:
print tpe1.encode("utf-8")
if rewrite:
print "Artist whitelist updated"
def main_generator(rewrite):
drop = dropbox.Dropbox()
new_artists = set()
for au_file in drop.tracks():
try:
tpe1 = au_file.mutagen_id3["TPE1"].text[0]
except:
cprint('** file: %r' % au_file.path)
raise
if artists.standardize(tpe1) is None:
new_artists.add(tpe1)
to_print = list(new_artists)
if rewrite:
to_print.extend(artists.all())
to_print.sort(key=artists.sort_key)
output = None
if rewrite:
output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
for tpe1 in to_print:
if output:
output.write(tpe1)
output.write("\n")
else:
cprint(tpe1.encode("utf-8"))
yield
if rewrite:
cprint('Artist whitelist updated', type='highlight')
else:
cprint('Found %d new artists' % len(to_print), type='success')
def main_generator(rewrite):
drop = dropbox.Dropbox()
new_artists = set()
for au_file in drop.tracks():
try:
tpe1 = au_file.mutagen_id3["TPE1"].text[0]
except:
cprint(u'** file: %r' % au_file.path)
raise
if artists.standardize(tpe1) is None:
new_artists.add(tpe1)
to_print = list(new_artists)
if rewrite:
to_print.extend(artists.all())
to_print.sort(key=artists.sort_key)
output = None
if rewrite:
output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
for tpe1 in to_print:
if output:
output.write(tpe1)
output.write("\n")
else:
cprint(tpe1)
yield
if rewrite:
cprint('Artist whitelist updated', type='success')
else:
cprint('Found %d new artists' % len(to_print), type='success')
def do_push_artists(self):
# patch credentials
if not request.headers.get('Authorization'):
abort(401)
else:
auth = request.headers['Authorization'].lstrip('Basic ')
username, password = base64.b64decode(auth).split(':')
if username and password:
conf.CHIRPRADIO_AUTH = '%s %s' % (username, password)
chirpradio.connect()
else:
abort(401)
dry_run = False
# reload artists from file
artists._init()
# Find all of the library artists
all_library_artists = set(artists.all())
# Find all of the artists in the cloud.
all_chirpradio_artists = set()
mapped = 0
t1 = time.time()
for art in models.Artist.fetch_all():
if art.revoked:
continue
std_name = artists.standardize(art.name)
if std_name != art.name:
#print "Mapping %d: %s => %s" % (mapped, art.name, std_name)
mapped += 1
art.name = std_name
idx = search.Indexer()
idx._transaction = art.parent_key()
idx.add_artist(art)
if not dry_run:
idx.save()
all_chirpradio_artists.add(art.name)
to_push = list(all_library_artists.difference(all_chirpradio_artists))
Messages.add_message("Pushing %d artists" % len(to_push), 'warning')
while to_push:
# Push the artists in batches of 50
this_push = to_push[:50]
to_push = to_push[50:]
idx = search.Indexer()
for name in this_push:
#print name
art = models.Artist.create(parent=idx.transaction, name=name)
idx.add_artist(art)
if not dry_run:
idx.save()
#print "+++++ Indexer saved"
Messages.add_message("Artist push complete. OK!", 'success')
def do_push_artists(self):
# patch credentials
if not request.headers.get('Authorization'):
abort(401)
else:
auth = request.headers['Authorization'].lstrip('Basic ')
username, password = base64.b64decode(auth).split(':')
if username and password:
conf.CHIRPRADIO_AUTH = '%s %s' % (username, password)
chirpradio.connect()
else:
abort(401)
dry_run = False
# reload artists from file
artists._init()
# Find all of the library artists
all_library_artists = set(artists.all())
# Find all of the artists in the cloud.
all_chirpradio_artists = set()
mapped = 0
t1 = time.time()
for art in models.Artist.fetch_all():
if art.revoked:
continue
std_name = artists.standardize(art.name)
if std_name != art.name:
#print "Mapping %d: %s => %s" % (mapped, art.name, std_name)
mapped += 1
art.name = std_name
idx = search.Indexer()
idx._transaction = art.parent_key()
idx.add_artist(art)
if not dry_run:
idx.save()
all_chirpradio_artists.add(art.name)
to_push = list(all_library_artists.difference(all_chirpradio_artists))
Messages.add_message("Pushing %d artists" % len(to_push), 'warning')
while to_push:
# Push the artists in batches of 50
this_push = to_push[:50]
to_push = to_push[50:]
idx = search.Indexer()
for name in this_push:
#print name
art = models.Artist.create(parent=idx.transaction, name=name)
idx.add_artist(art)
if not dry_run:
idx.save()
#print "+++++ Indexer saved"
Messages.add_message("Artist push complete. OK!", 'success')
def main_generator():
chirpradio.connect()
dry_run = False
# Find all of the library artists
all_library_artists = set(artists.all())
# Find all of the artists in the cloud.
all_chirpradio_artists = set()
mapped = 0
t1 = time.time()
for art in models.Artist.fetch_all():
if art.revoked:
continue
std_name = artists.standardize(art.name)
if std_name != art.name:
cprint(u"Mapping {}: {} => {}".format(mapped, art.name, std_name))
mapped += 1
art.name = std_name
idx = search.Indexer()
idx._transaction = art.parent_key()
idx.add_artist(art)
if not dry_run:
idx.save()
all_chirpradio_artists.add(art.name)
yield
to_push = list(all_library_artists.difference(all_chirpradio_artists))
cprint("Pushing %d artists" % len(to_push))
while to_push:
# Push the artists in batches of 50
this_push = to_push[:50]
to_push = to_push[50:]
idx = search.Indexer()
for name in this_push:
cprint(name)
art = models.Artist.create(parent=idx.transaction, name=name)
idx.add_artist(art)
if not dry_run:
idx.save()
cprint("+++++ Indexer saved")
yield
def add_artists(self):
error = False
drop = dropbox.Dropbox()
new_artists = set()
for au_file in drop.tracks():
try:
tpe1 = au_file.mutagen_id3["TPE1"].text[0]
except:
Messages.add_messaage('** file: %r' % au_file.path, 'error')
error = True
# TODO propagate error to client
raise
if artists.standardize(tpe1) is None:
new_artists.add(tpe1)
# do not write if errors
if not error and new_artists:
to_print = list(new_artists)
to_print.extend(artists.all())
to_print.sort(key=artists.sort_key)
output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
for tpe1 in to_print:
output.write(tpe1)
output.write("\n")
output.close()
# reload whitelist from file
artists._init()
message = "Artist whitelist updated.<br>New artists added:<br>"
message += "<br>".join(list(new_artists))
Messages.add_message(message, 'success')
# push to github
self.push_to_github()
def add_artists(self):
error = False
drop = dropbox.Dropbox()
new_artists = set()
for au_file in drop.tracks():
try:
tpe1 = au_file.mutagen_id3["TPE1"].text[0]
except:
Messages.add_messaage('** file: %r' % au_file.path, 'error')
error = True
# TODO propagate error to client
raise
if artists.standardize(tpe1) is None:
new_artists.add(tpe1)
# do not write if errors
if not error and new_artists:
to_print = list(new_artists)
to_print.extend(artists.all())
to_print.sort(key=artists.sort_key)
output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
for tpe1 in to_print:
output.write(tpe1)
output.write("\n")
output.close()
# reload whitelist from file
artists._init()
message = "Artist whitelist updated.<br>New artists added:<br>"
message += "<br>".join(list(new_artists))
Messages.add_message(message, 'success')
# push to github
self.push_to_github()
def _fix_file_tags(au_file):
"""Modify a file's tags to conform to CHIRP's tagging standards.
Args:
au_file: An AudioFile object
Returns:
A new mutagen.id3.ID3 object with the corrected tags.
Raises:
ImportFileError: if the tagging is broken or cannot be fixed.
"""
if constants.MUTAGEN_UFID_KEY in au_file.mutagen_id3:
raise ImportFileError(["File already contains CHIRP UFID tag"])
new_id3 = mutagen.id3.ID3()
# Build up a list of tags, stripping out ones that are not on our
# whitelist.
for tag in au_file.mutagen_id3.values():
if tag.FrameID in constants.ID3_TAG_BLACKLIST:
raise ImportFileError(["Found blacklisted tag: %r" % tag])
if not (tag.FrameID in constants.ID3_TAG_WHITELIST
and tag.FrameID not in constants.ID3_TAG_STRIPPED_ON_IMPORT):
continue
# Manually filter out TPOS
# TODO(trow): We don't want to do this long-term.
if tag.FrameID == "TPOS":
continue
# Standardize TPE tags, filtering out unknown artists at
# TPE2 or lower.
if tag.FrameID.startswith("TPE"):
name_std = artists.standardize(unicode(tag))
if name_std is None:
if tag.FrameID != "TPE1":
sys.stderr.write(
"*** Filtering %s %s\n" %
(tag.FrameID, unicode(tag).encode("utf-8")))
continue
raise ImportFileError(
[u"Unknown artist %r in %s" % (unicode(tag), tag.FrameID)])
else:
tag.text = [name_std]
# If the TBPM tag is present and contains a string of the form
# "xxx BPM", strip off the suffix. If xxx is not an integer,
# round it off. If it is <= 0, discard the tag entirely.
if tag.FrameID == "TBPM":
tbpm = unicode(tag)
if tbpm.endswith(" BPM"):
tbpm = tbpm[:-4]
try:
tbpm = int(float(tbpm))
except ValueError:
continue
if tbpm <= 0:
continue
tag.text = [unicode(tbpm)]
new_id3.add(tag)
# Add our own TLEN tag.
tlen_tag = mutagen.id3.TLEN(text=[unicode(au_file.duration_ms)],
encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
new_id3.add(tlen_tag)
# Add tags containing the number of frames and the frame size.
frame_count_tag = mutagen.id3.TXXX(
desc=constants.TXXX_FRAME_COUNT_DESCRIPTION,
text=[unicode(au_file.frame_count)],
encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
new_id3.add(frame_count_tag)
frame_size_tag = mutagen.id3.TXXX(
desc=constants.TXXX_FRAME_SIZE_DESCRIPTION,
text=[unicode(au_file.frame_size)],
encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
new_id3.add(frame_size_tag)
# Add a TXXX tag with the album ID.
txxx = mutagen.id3.TXXX(encoding=constants.DEFAULT_ID3_TEXT_ENCODING,
desc=constants.TXXX_ALBUM_ID_DESCRIPTION,
text=[unicode(au_file.album_id)])
new_id3.add(txxx)
# Add a TFLT tag indicating that this is an MP3.
tflt_tag = mutagen.id3.TFLT(text=[u"MPG/3"],
encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
new_id3.add(tflt_tag)
# Add the standard CHIRP TOWN tag.
town_tag = mutagen.id3.TOWN(text=[constants.TOWN_FILE_OWNER],
encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
new_id3.add(town_tag)
return new_id3
def test_standardized_none_is_none(self):
self.assertEqual(artists.standardize(None), None)
def test_real_data(self):
self.assertTrue(len(artists._global_whitelist) > 2000)
self.assertTrue(len(artists._global_mappings) >= 2)
# Check some known whitelist items.
for expected, raw in (("Bob Dylan", "bob dylan"),
("The Fall", "fall"),
):
self.assertEqual(expected, artists.standardize(raw))
# Check a known mapping.
for expected, raw in (("Gordon Staples", "Gordon Stapes"),
):
self.assertEqual(expected, artists.standardize(raw))
# Repeatedly merge the global whitelist and mappings until the two
# dicts stabilize. If merging caused anything to change, write
# out corrected forms of the files, print a banner and a diff
# to stdout, and cause the test to fail.
whitelist, mappings = artists.merge_whitelist_and_mappings(
artists._global_whitelist, artists._global_raw_mappings)
fixed_whitelist_filename = artists._WHITELIST_FILE + ".fixed"
fixed_mappings_filename = artists._MAPPINGS_FILE + ".fixed"
# Delete these files if they already exist.
for filename in (fixed_whitelist_filename, fixed_mappings_filename):
try:
os.unlink(filename)
except OSError:
pass
test_should_succeed = True
if whitelist != artists._global_whitelist:
test_should_succeed = False
print "\n\n"
print "*" * 70
print "***"
print "*** Whitelist is not properly normalized"
print "***"
print "*** Diff:"
out = codecs.open(fixed_whitelist_filename, "w", "utf-8")
for white in sorted(whitelist.values(),
key=artists.sort_key):
out.write(u"%s\n" % white)
out.close()
diff_cmd = "diff -u %s %s" % (artists._WHITELIST_FILE,
fixed_whitelist_filename)
sys.stdout.flush()
os.system(diff_cmd)
sys.stdout.flush()
print "*" * 70
print "\n\n"
if mappings != artists._global_raw_mappings:
test_should_succeed = False
print "\n\n"
print "*" * 70
print "***"
print "*** Mappings are not properly normalized"
print "***"
print "*** Diff:"
out = codecs.open(fixed_mappings_filename, "w", "utf-8")
for before in sorted(mappings, key=artists.sort_key):
out.write(u"%s %s %s\n" % (before,
artists._MAPPINGS_SEP,
mappings[before]))
out.close()
diff_cmd = "diff -u %s %s" % (artists._MAPPINGS_FILE,
fixed_mappings_filename)
sys.stdout.flush()
os.system(diff_cmd)
sys.stdout.flush()
print "*" * 70
print "\n\n"
self.assertTrue(test_should_succeed)
def test_standardized_none_is_none(self):
self.assertEqual(artists.standardize(None), None)
def test_real_data(self):
self.assertTrue(len(artists._global_whitelist) > 2000)
self.assertTrue(len(artists._global_mappings) >= 2)
# Check some known whitelist items.
for expected, raw in (
("Bob Dylan", "bob dylan"),
("The Fall", "fall"),
):
self.assertEqual(expected, artists.standardize(raw))
# Check a known mapping.
for expected, raw in (("Gordon Staples", "Gordon Stapes"), ):
self.assertEqual(expected, artists.standardize(raw))
# Repeatedly merge the global whitelist and mappings until the two
# dicts stabilize. If merging caused anything to change, write
# out corrected forms of the files, print a banner and a diff
# to stdout, and cause the test to fail.
whitelist, mappings = artists.merge_whitelist_and_mappings(
artists._global_whitelist, artists._global_raw_mappings)
fixed_whitelist_filename = artists._WHITELIST_FILE + ".fixed"
fixed_mappings_filename = artists._MAPPINGS_FILE + ".fixed"
# Delete these files if they already exist.
for filename in (fixed_whitelist_filename, fixed_mappings_filename):
try:
os.unlink(filename)
except OSError:
pass
test_should_succeed = True
if whitelist != artists._global_whitelist:
test_should_succeed = False
print "\n\n"
print "*" * 70
print "***"
print "*** Whitelist is not properly normalized"
print "***"
print "*** Diff:"
out = codecs.open(fixed_whitelist_filename, "w", "utf-8")
for white in sorted(whitelist.values(), key=artists.sort_key):
out.write(u"%s\n" % white)
out.close()
diff_cmd = "diff -u %s %s" % (artists._WHITELIST_FILE,
fixed_whitelist_filename)
sys.stdout.flush()
os.system(diff_cmd)
sys.stdout.flush()
print "*" * 70
print "\n\n"
if mappings != artists._global_raw_mappings:
test_should_succeed = False
print "\n\n"
print "*" * 70
print "***"
print "*** Mappings are not properly normalized"
print "***"
print "*** Diff:"
out = codecs.open(fixed_mappings_filename, "w", "utf-8")
for before in sorted(mappings, key=artists.sort_key):
out.write(u"%s %s %s\n" %
(before, artists._MAPPINGS_SEP, mappings[before]))
out.close()
diff_cmd = "diff -u %s %s" % (artists._MAPPINGS_FILE,
fixed_mappings_filename)
sys.stdout.flush()
os.system(diff_cmd)
sys.stdout.flush()
print "*" * 70
print "\n\n"
self.assertTrue(test_should_succeed)
