def main():
    rewrite = "--rewrite" in sys.argv

    drop = dropbox.Dropbox()
    new_artists = set()
    for au_file in drop.tracks():
        try:
            tpe1 = au_file.mutagen_id3["TPE1"].text[0]
        except:
            print "** file: %r" % au_file.path
            raise
        if artists.standardize(tpe1) is None:
            new_artists.add(tpe1)

    to_print = list(new_artists)
    if rewrite:
        to_print.extend(artists.all())
    to_print.sort(key=artists.sort_key)

    output = None
    if rewrite:
        output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
    for tpe1 in to_print:
        if output:
            output.write(tpe1)
            output.write("\n")
        else:
            print tpe1.encode("utf-8")
    if rewrite:
        print "Artist whitelist updated"
def main():
    rewrite = ("--rewrite" in sys.argv)

    drop = dropbox.Dropbox()
    new_artists = set()
    for au_file in drop.tracks():
        try:
            tpe1 = au_file.mutagen_id3["TPE1"].text[0]
        except:
            print '** file: %r' % au_file.path
            raise
        if artists.standardize(tpe1) is None:
            new_artists.add(tpe1)

    to_print = list(new_artists)
    if rewrite:
        to_print.extend(artists.all())
    to_print.sort(key=artists.sort_key)

    output = None
    if rewrite:
        output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
    for tpe1 in to_print:
        if output:
            output.write(tpe1)
            output.write("\n")
        else:
            print tpe1.encode("utf-8")
    if rewrite:
        print "Artist whitelist updated"
Beispiel #3
0
def main_generator(rewrite):
    drop = dropbox.Dropbox()
    new_artists = set()
    for au_file in drop.tracks():
        try:
            tpe1 = au_file.mutagen_id3["TPE1"].text[0]
        except:
            cprint('** file: %r' % au_file.path)
            raise
        if artists.standardize(tpe1) is None:
            new_artists.add(tpe1)

    to_print = list(new_artists)
    if rewrite:
        to_print.extend(artists.all())
    to_print.sort(key=artists.sort_key)

    output = None
    if rewrite:
        output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
    for tpe1 in to_print:
        if output:
            output.write(tpe1)
            output.write("\n")
        else:
            cprint(tpe1.encode("utf-8"))
        yield

    if rewrite:
        cprint('Artist whitelist updated', type='highlight')
    else:
        cprint('Found %d new artists' % len(to_print), type='success')
def main_generator(rewrite):
    drop = dropbox.Dropbox()
    new_artists = set()
    for au_file in drop.tracks():
        try:
            tpe1 = au_file.mutagen_id3["TPE1"].text[0]
        except:
            cprint(u'** file: %r' % au_file.path)
            raise
        if artists.standardize(tpe1) is None:
            new_artists.add(tpe1)

    to_print = list(new_artists)
    if rewrite:
        to_print.extend(artists.all())
    to_print.sort(key=artists.sort_key)

    output = None
    if rewrite:
        output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
    for tpe1 in to_print:
        if output:
            output.write(tpe1)
            output.write("\n")
        else:
            cprint(tpe1)
        yield

    if rewrite:
        cprint('Artist whitelist updated', type='success')
    else:
        cprint('Found %d new artists' % len(to_print), type='success')
Beispiel #5
0
    def do_push_artists(self):
        # patch credentials
        if not request.headers.get('Authorization'):
            abort(401)
        else:
            auth = request.headers['Authorization'].lstrip('Basic ')
            username, password = base64.b64decode(auth).split(':')
            if username and password:
                conf.CHIRPRADIO_AUTH = '%s %s' % (username, password)
                chirpradio.connect()
            else:
                abort(401)

        dry_run = False

        # reload artists from file
        artists._init()

        # Find all of the library artists
        all_library_artists = set(artists.all())

        # Find all of the artists in the cloud.
        all_chirpradio_artists = set()
        mapped = 0
        t1 = time.time()
        for art in models.Artist.fetch_all():
            if art.revoked:
                continue
            std_name = artists.standardize(art.name)
            if std_name != art.name:
                #print "Mapping %d: %s => %s" % (mapped, art.name, std_name)
                mapped += 1
                art.name = std_name
                idx = search.Indexer()
                idx._transaction = art.parent_key()
                idx.add_artist(art)
                if not dry_run:
                    idx.save()
            all_chirpradio_artists.add(art.name)

        to_push = list(all_library_artists.difference(all_chirpradio_artists))

        Messages.add_message("Pushing %d artists" % len(to_push), 'warning')
        while to_push:
            # Push the artists in batches of 50
            this_push = to_push[:50]
            to_push = to_push[50:]
            idx = search.Indexer()
            for name in this_push:
                #print name
                art = models.Artist.create(parent=idx.transaction, name=name)
                idx.add_artist(art)
            if not dry_run:
                idx.save()
            #print "+++++ Indexer saved"

        Messages.add_message("Artist push complete. OK!", 'success')
Beispiel #6
0
    def do_push_artists(self):
        # patch credentials
        if not request.headers.get('Authorization'):
            abort(401)
        else:
            auth = request.headers['Authorization'].lstrip('Basic ')
            username, password = base64.b64decode(auth).split(':')
            if username and password:
                conf.CHIRPRADIO_AUTH = '%s %s' % (username, password)
                chirpradio.connect()
            else:
                abort(401)

        dry_run = False

        # reload artists from file
        artists._init()

        # Find all of the library artists
        all_library_artists = set(artists.all())

        # Find all of the artists in the cloud.
        all_chirpradio_artists = set()
        mapped = 0
        t1 = time.time()
        for art in models.Artist.fetch_all():
            if art.revoked:
                continue
            std_name = artists.standardize(art.name)
            if std_name != art.name:
                #print "Mapping %d: %s => %s" % (mapped, art.name, std_name)
                mapped += 1
                art.name = std_name
                idx = search.Indexer()
                idx._transaction = art.parent_key()
                idx.add_artist(art)
                if not dry_run:
                    idx.save()
            all_chirpradio_artists.add(art.name)

        to_push = list(all_library_artists.difference(all_chirpradio_artists))

        Messages.add_message("Pushing %d artists" % len(to_push), 'warning')
        while to_push:
            # Push the artists in batches of 50
            this_push = to_push[:50]
            to_push = to_push[50:]
            idx = search.Indexer()
            for name in this_push:
                #print name
                art = models.Artist.create(parent=idx.transaction, name=name)
                idx.add_artist(art)
            if not dry_run:
                idx.save()
            #print "+++++ Indexer saved"

        Messages.add_message("Artist push complete. OK!", 'success')
def main_generator():
    chirpradio.connect()

    dry_run = False

    # Find all of the library artists
    all_library_artists = set(artists.all())

    # Find all of the artists in the cloud.
    all_chirpradio_artists = set()
    mapped = 0
    t1 = time.time()
    for art in models.Artist.fetch_all():
        if art.revoked:
            continue
        std_name = artists.standardize(art.name)
        if std_name != art.name:
            cprint(u"Mapping {}: {} => {}".format(mapped, art.name, std_name))
            mapped += 1
            art.name = std_name
            idx = search.Indexer()
            idx._transaction = art.parent_key()
            idx.add_artist(art)
            if not dry_run:
                idx.save()
        all_chirpradio_artists.add(art.name)
        yield

    to_push = list(all_library_artists.difference(all_chirpradio_artists))

    cprint("Pushing %d artists" % len(to_push))
    while to_push:
        # Push the artists in batches of 50
        this_push = to_push[:50]
        to_push = to_push[50:]
        idx = search.Indexer()
        for name in this_push:
            cprint(name)
            art = models.Artist.create(parent=idx.transaction, name=name)
            idx.add_artist(art)
        if not dry_run:
            idx.save()
        cprint("+++++ Indexer saved")
        yield
    def add_artists(self):
        error = False
        drop = dropbox.Dropbox()
        new_artists = set()
        for au_file in drop.tracks():
            try:
                tpe1 = au_file.mutagen_id3["TPE1"].text[0]
            except:
                Messages.add_messaage('** file: %r' % au_file.path, 'error')
                error = True

                # TODO propagate error to client
                raise

            if artists.standardize(tpe1) is None:
                new_artists.add(tpe1)

        # do not write if errors
        if not error and new_artists:
            to_print = list(new_artists)
            to_print.extend(artists.all())
            to_print.sort(key=artists.sort_key)

            output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
            for tpe1 in to_print:
                output.write(tpe1)
                output.write("\n")
            output.close()

            # reload whitelist from file
            artists._init()

            message = "Artist whitelist updated.<br>New artists added:<br>"
            message += "<br>".join(list(new_artists))
            Messages.add_message(message, 'success')

            # push to github
            self.push_to_github()
Beispiel #9
0
    def add_artists(self):
        error = False
        drop = dropbox.Dropbox()
        new_artists = set()
        for au_file in drop.tracks():
            try:
                tpe1 = au_file.mutagen_id3["TPE1"].text[0]
            except:
                Messages.add_messaage('** file: %r' % au_file.path, 'error')
                error = True

                # TODO propagate error to client
                raise

            if artists.standardize(tpe1) is None:
                new_artists.add(tpe1)

        # do not write if errors
        if not error and new_artists:
            to_print = list(new_artists)
            to_print.extend(artists.all())
            to_print.sort(key=artists.sort_key)

            output = codecs.open(artists._WHITELIST_FILE, "w", "utf-8")
            for tpe1 in to_print:
                output.write(tpe1)
                output.write("\n")
            output.close()

            # reload whitelist from file
            artists._init()

            message = "Artist whitelist updated.<br>New artists added:<br>"
            message += "<br>".join(list(new_artists))
            Messages.add_message(message, 'success')

            # push to github
            self.push_to_github()
Beispiel #10
0
def _fix_file_tags(au_file):
    """Modify a file's tags to conform to CHIRP's tagging standards.

    Args:
      au_file: An AudioFile object

    Returns:
      A new mutagen.id3.ID3 object with the corrected tags.

    Raises:
      ImportFileError: if the tagging is broken or cannot be fixed.
    """
    if constants.MUTAGEN_UFID_KEY in au_file.mutagen_id3:
        raise ImportFileError(["File already contains CHIRP UFID tag"])

    new_id3 = mutagen.id3.ID3()

    # Build up a list of tags, stripping out ones that are not on our
    # whitelist.
    for tag in au_file.mutagen_id3.values():
        if tag.FrameID in constants.ID3_TAG_BLACKLIST:
            raise ImportFileError(["Found blacklisted tag: %r" % tag])
        if not (tag.FrameID in constants.ID3_TAG_WHITELIST
                and tag.FrameID not in constants.ID3_TAG_STRIPPED_ON_IMPORT):
            continue
        # Manually filter out TPOS
        # TODO(trow): We don't want to do this long-term.
        if tag.FrameID == "TPOS":
            continue
        # Standardize TPE tags, filtering out unknown artists at
        # TPE2 or lower.
        if tag.FrameID.startswith("TPE"):
            name_std = artists.standardize(unicode(tag))
            if name_std is None:
                if tag.FrameID != "TPE1":
                    sys.stderr.write(
                        "*** Filtering %s %s\n" %
                        (tag.FrameID, unicode(tag).encode("utf-8")))
                    continue
                raise ImportFileError(
                    [u"Unknown artist %r in %s" % (unicode(tag), tag.FrameID)])
            else:
                tag.text = [name_std]
        # If the TBPM tag is present and contains a string of the form
        # "xxx BPM", strip off the suffix.  If xxx is not an integer,
        # round it off.  If it is <= 0, discard the tag entirely.
        if tag.FrameID == "TBPM":
            tbpm = unicode(tag)
            if tbpm.endswith(" BPM"):
                tbpm = tbpm[:-4]
            try:
                tbpm = int(float(tbpm))
            except ValueError:
                continue
            if tbpm <= 0:
                continue
            tag.text = [unicode(tbpm)]

        new_id3.add(tag)

    # Add our own TLEN tag.
    tlen_tag = mutagen.id3.TLEN(text=[unicode(au_file.duration_ms)],
                                encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
    new_id3.add(tlen_tag)

    # Add tags containing the number of frames and the frame size.

    frame_count_tag = mutagen.id3.TXXX(
        desc=constants.TXXX_FRAME_COUNT_DESCRIPTION,
        text=[unicode(au_file.frame_count)],
        encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
    new_id3.add(frame_count_tag)

    frame_size_tag = mutagen.id3.TXXX(
        desc=constants.TXXX_FRAME_SIZE_DESCRIPTION,
        text=[unicode(au_file.frame_size)],
        encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
    new_id3.add(frame_size_tag)

    # Add a TXXX tag with the album ID.
    txxx = mutagen.id3.TXXX(encoding=constants.DEFAULT_ID3_TEXT_ENCODING,
                            desc=constants.TXXX_ALBUM_ID_DESCRIPTION,
                            text=[unicode(au_file.album_id)])
    new_id3.add(txxx)

    # Add a TFLT tag indicating that this is an MP3.
    tflt_tag = mutagen.id3.TFLT(text=[u"MPG/3"],
                                encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
    new_id3.add(tflt_tag)

    # Add the standard CHIRP TOWN tag.
    town_tag = mutagen.id3.TOWN(text=[constants.TOWN_FILE_OWNER],
                                encoding=constants.DEFAULT_ID3_TEXT_ENCODING)
    new_id3.add(town_tag)

    return new_id3
 def test_standardized_none_is_none(self):
     self.assertEqual(artists.standardize(None), None)
    def test_real_data(self):
        self.assertTrue(len(artists._global_whitelist) > 2000)
        self.assertTrue(len(artists._global_mappings) >= 2)

        # Check some known whitelist items.
        for expected, raw in (("Bob Dylan", "bob dylan"),
                              ("The Fall", "fall"),
                              ):
            self.assertEqual(expected, artists.standardize(raw))
        # Check a known mapping.
        for expected, raw in (("Gordon Staples", "Gordon Stapes"),
                              ):
            self.assertEqual(expected, artists.standardize(raw))

        # Repeatedly merge the global whitelist and mappings until the two
        # dicts stabilize.  If merging caused anything to change, write
        # out corrected forms of the files, print a banner and a diff
        # to stdout, and cause the test to fail.
        whitelist, mappings = artists.merge_whitelist_and_mappings(
            artists._global_whitelist, artists._global_raw_mappings)
        
        fixed_whitelist_filename = artists._WHITELIST_FILE + ".fixed"
        fixed_mappings_filename = artists._MAPPINGS_FILE + ".fixed"
        # Delete these files if they already exist.
        for filename in (fixed_whitelist_filename, fixed_mappings_filename):
            try:
                os.unlink(filename)
            except OSError:
                pass

        test_should_succeed = True

        if whitelist != artists._global_whitelist:
            test_should_succeed = False
            print "\n\n"
            print "*" * 70
            print "***"
            print "*** Whitelist is not properly normalized"
            print "***"
            print "*** Diff:"
            out = codecs.open(fixed_whitelist_filename, "w", "utf-8")
            for white in sorted(whitelist.values(),
                                key=artists.sort_key):
                out.write(u"%s\n" % white)
            out.close()
            diff_cmd = "diff -u %s %s" % (artists._WHITELIST_FILE,
                                          fixed_whitelist_filename)
            sys.stdout.flush()
            os.system(diff_cmd)
            sys.stdout.flush()
            print "*" * 70
            print "\n\n"

        if mappings != artists._global_raw_mappings:
            test_should_succeed = False
            print "\n\n"
            print "*" * 70
            print "***"
            print "*** Mappings are not properly normalized"
            print "***"
            print "*** Diff:"
            out = codecs.open(fixed_mappings_filename, "w", "utf-8")
            for before in sorted(mappings, key=artists.sort_key):
                out.write(u"%s %s %s\n" % (before,
                                           artists._MAPPINGS_SEP,
                                           mappings[before]))
            out.close()
            diff_cmd = "diff -u %s %s" % (artists._MAPPINGS_FILE,
                                          fixed_mappings_filename)
            sys.stdout.flush()
            os.system(diff_cmd)
            sys.stdout.flush()
            print "*" * 70
            print "\n\n"

        self.assertTrue(test_should_succeed)
Beispiel #13
0
 def test_standardized_none_is_none(self):
     self.assertEqual(artists.standardize(None), None)
Beispiel #14
0
    def test_real_data(self):
        self.assertTrue(len(artists._global_whitelist) > 2000)
        self.assertTrue(len(artists._global_mappings) >= 2)

        # Check some known whitelist items.
        for expected, raw in (
            ("Bob Dylan", "bob dylan"),
            ("The Fall", "fall"),
        ):
            self.assertEqual(expected, artists.standardize(raw))
        # Check a known mapping.
        for expected, raw in (("Gordon Staples", "Gordon Stapes"), ):
            self.assertEqual(expected, artists.standardize(raw))

        # Repeatedly merge the global whitelist and mappings until the two
        # dicts stabilize.  If merging caused anything to change, write
        # out corrected forms of the files, print a banner and a diff
        # to stdout, and cause the test to fail.
        whitelist, mappings = artists.merge_whitelist_and_mappings(
            artists._global_whitelist, artists._global_raw_mappings)

        fixed_whitelist_filename = artists._WHITELIST_FILE + ".fixed"
        fixed_mappings_filename = artists._MAPPINGS_FILE + ".fixed"
        # Delete these files if they already exist.
        for filename in (fixed_whitelist_filename, fixed_mappings_filename):
            try:
                os.unlink(filename)
            except OSError:
                pass

        test_should_succeed = True

        if whitelist != artists._global_whitelist:
            test_should_succeed = False
            print "\n\n"
            print "*" * 70
            print "***"
            print "*** Whitelist is not properly normalized"
            print "***"
            print "*** Diff:"
            out = codecs.open(fixed_whitelist_filename, "w", "utf-8")
            for white in sorted(whitelist.values(), key=artists.sort_key):
                out.write(u"%s\n" % white)
            out.close()
            diff_cmd = "diff -u %s %s" % (artists._WHITELIST_FILE,
                                          fixed_whitelist_filename)
            sys.stdout.flush()
            os.system(diff_cmd)
            sys.stdout.flush()
            print "*" * 70
            print "\n\n"

        if mappings != artists._global_raw_mappings:
            test_should_succeed = False
            print "\n\n"
            print "*" * 70
            print "***"
            print "*** Mappings are not properly normalized"
            print "***"
            print "*** Diff:"
            out = codecs.open(fixed_mappings_filename, "w", "utf-8")
            for before in sorted(mappings, key=artists.sort_key):
                out.write(u"%s %s %s\n" %
                          (before, artists._MAPPINGS_SEP, mappings[before]))
            out.close()
            diff_cmd = "diff -u %s %s" % (artists._MAPPINGS_FILE,
                                          fixed_mappings_filename)
            sys.stdout.flush()
            os.system(diff_cmd)
            sys.stdout.flush()
            print "*" * 70
            print "\n\n"

        self.assertTrue(test_should_succeed)