Exemplo n.º 1
0
    def fingerprint_directory(self, path, extensions, nprocesses=None):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):

            # don't refingerprint already fingerprinted files
            if decoder.unique_hash(filename) in self.songhashes_set:
                dejavu.shared.UITEXTLOGGER.emit(
                    "%s already fingerprinted, continuing..." % filename)
                print("%s already fingerprinted, continuing..." % filename)
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = zip(filenames_to_fingerprint,
                           [self.limit] * len(filenames_to_fingerprint))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker, worker_input)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes, file_hash = iterator.next()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                dejavu.shared.UITEXTLOGGER.emit("Failed fingerprinting")
                print("Failed fingerprinting")
                # Print traceback because we can't reraise it here
                traceback.print_exc(file=sys.stdout)
            else:
                dejavu.shared.UITEXTLOGGER.emit(
                    "Saving finger prints to Database for %s" % song_name)
                print("Saving finger prints to Database for %s" % song_name)
                sid = self.db.insert_song(song_name, file_hash)

                self.db.insert_hashes(sid, hashes)
                self.db.set_song_fingerprinted(sid)
                self.get_fingerprinted_songs()
                dejavu.shared.UITEXTLOGGER.emit(
                    "Finished saving finger prints to Database for %s" %
                    song_name)
                print("Finished saving finger prints to Database for %s" %
                      song_name)

        pool.close()
        pool.join()
Exemplo n.º 2
0
    def fingerprint_directory(self,
                              path,
                              extensions,
                              nprocesses=None,
                              splited=False,
                              splited_song_name=""):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):

            # don't refingerprint already fingerprinted files
            if decoder.path_to_songname(filename) in self.songnames_set:
                print "%s already fingerprinted, continuing..." % filename
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = zip(filenames_to_fingerprint,
                           [self.limit] * len(filenames_to_fingerprint))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker, worker_input)

        if splited and splited_song_name:
            sid = self.db.insert_song(splited_song_name)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes = iterator.next()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                print("Failed fingerprinting")
                # Print traceback because we can't reraise it here
                traceback.print_exc(file=sys.stdout)
            else:
                if not splited:
                    sid = self.db.insert_song(song_name)
                self.db.insert_hashes(sid, hashes)
                self.db.set_song_fingerprinted(sid)
                self.get_fingerprinted_songs()

        pool.close()
        pool.join()
Exemplo n.º 3
0
 def recognize_directory(self, path, extensions):
     for filename, _ in decoder.find_files(path, extensions):
         result = self.recognize_file(filename)
         if result["confidence"] >= self.confidence_limit:
             msg = ("From file %s we recognized: %s" % (filename, result))
             msg = "[" + datetime.datetime.now().strftime(
                 "%Y-%m-%d %H:%M:%S") + "]" + msg
             print msg
             logging.info(msg)
Exemplo n.º 4
0
def fingerprint_directory(self, path, extensions, nprocesses=None):
    # Try to use the maximum amount of processes if not given.
    try:
        nprocesses = nprocesses or multiprocessing.cpu_count()
    except NotImplementedError:
        nprocesses = 1
    else:
        nprocesses = 1 if nprocesses <= 0 else nprocesses

    pool = multiprocessing.Pool(nprocesses)

    filenames_to_fingerprint = []
    for filename, _ in decoder.find_files(path, extensions):
        filenames_to_fingerprint.append(filename)

    # Prepare _fingerprint_worker input
    worker_input = zip(filenames_to_fingerprint,
                       [self.limit] * len(filenames_to_fingerprint))

    # Send off our tasks
    iterator = pool.imap_unordered(_fingerprint_worker,
                                   worker_input)

    # Loop till we have all of them
    batch_list = []
    while True:
        try:
            song_name, hashes, file_hash, filename = iterator.next()
            hashes_list = list(hashes)
        except multiprocessing.TimeoutError:
            continue
        except StopIteration:
            break
        except:
            print("Failed fingerprinting")
            # Print traceback because we can't reraise it here
            traceback.print_exc(file=sys.stdout)
        else:
            batch_list.append({
                "song_name": song_name,
                "hashes_list": hashes_list,
                "file_hash": file_hash,
                "filename": filename,
            })

            if len(batch_list) == BATCH_NUMBER:
                call_fingerprint_create_api(batch_list)
                batch_list = []

    if len(batch_list) < BATCH_NUMBER:
        call_fingerprint_create_api(batch_list)

    pool.close()
    pool.join()
Exemplo n.º 5
0
    def fingerprint_directory(self, path, extensions, nprocesses=None):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):

            # don't refingerprint already fingerprinted files
            if decoder.path_to_songname(filename) in self.songnames_set:
                print "%s already fingerprinted, continuing..." % filename
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = zip(filenames_to_fingerprint,
                           [self.limit] * len(filenames_to_fingerprint))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker,
                                       worker_input)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes = iterator.next()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                print("Failed fingerprinting")

                # Print traceback because we can't reraise it here
                import traceback, sys
                traceback.print_exc(file=sys.stdout)
            else:
                sid = self.db.insert_song(song_name)

                self.db.insert_hashes(sid, hashes)
                self.db.set_song_fingerprinted(sid)
                self.get_fingerprinted_songs()

        pool.close()
        pool.join()
Exemplo n.º 6
0
    def fingerprint_directory(self, path, extensions, nprocesses=None):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):

            # don't refingerprint already fingerprinted files
            if decoder.unique_hash(filename) in self.songhashes_set:
                logging.getLogger('dejavu').warn(
                    "%s already fingerprinted, continuing..." % filename)
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = zip(filenames_to_fingerprint,
                           [self.limit] * len(filenames_to_fingerprint))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker, worker_input)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes, file_hash, audio_length = iterator.next()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                logging.getLogger('dejavu').exception("Failed fingerprinting")
            else:
                logging.getLogger('dejavu').debug("Inserting " + song_name +
                                                  " in database")
                sid = self.db.insert_song(song_name, file_hash, audio_length)

                self.db.insert_hashes(sid,
                                      set([(x[0], int(x[1])) for x in hashes]))
                self.db.set_song_fingerprinted(sid)
                self.get_fingerprinted_songs()
                logging.getLogger('dejavu').info(song_name +
                                                 " inserted in database")
        pool.close()
        pool.join()
Exemplo n.º 7
0
    def fingerprint_directory(self, path, extensions, nprocesses=None):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):
            # don't refingerprint already fingerprinted files
            if self.db.get_song_by_hash(
                    decoder.unique_hash(filename)) is not None:
                logger.info("%s already fingerprinted, continuing..." %
                            filename)
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = list(
            zip(filenames_to_fingerprint,
                [self.limit] * len(filenames_to_fingerprint)))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker, worker_input)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes, file_hash = next(iterator)
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                logger.error("Failed fingerprinting")
                # Print traceback because we can't reraise it here
                traceback.print_exc(file=sys.stdout)
            else:
                sid = self.db.insert_song(song_name, file_hash)

                self.db.insert_hashes(sid, hashes)
                self.db.set_song_fingerprinted(sid)

        pool.close()
        pool.join()
Exemplo n.º 8
0
    def fingerprint_directory(self,
                              path,
                              extensions,
                              output_dir=None,
                              nprocesses=None):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):
            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = zip(filenames_to_fingerprint,
                           [None] * len(filenames_to_fingerprint))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker, worker_input)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes, file_hash = iterator.next()
                f = open(os.path.join(output_dir, song_name + '.fingerp'), 'w')
                sorted_hashes = sorted(list(hashes), key=lambda t: t[1])
                for t in sorted_hashes:
                    f.write(str(t[0]) + " - " + str(t[1]) + "\n")
                    f.flush()
                f.close()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                print("Failed fingerprinting")
                # Print traceback because we can't reraise it here
                traceback.print_exc(file=sys.stdout)

        pool.close()
        pool.join()
Exemplo n.º 9
0
    def fingerprint_directory(self, path, extensions, nprocesses=None):
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        results = []
        for filename, _ in decoder.find_files(path, extensions):

            # don't refingerprint already fingerprinted files
            if decoder.path_to_songname(filename) in self.songnames_set:
                print "%s already fingerprinted, continuing..." % filename
                continue

            result = pool.apply_async(_fingerprint_worker,
                                      (filename, self.db, self.limit))
            results.append(result)

        while len(results):
            for result in results[:]:
                # TODO: Handle errors gracefully and return them to the callee
                # in some way.
                try:
                    result.get(timeout=2)
                except multiprocessing.TimeoutError:
                    continue
                except:
                    import traceback, sys
                    traceback.print_exc(file=sys.stdout)
                    results.remove(result)
                else:
                    results.remove(result)

        pool.close()
        pool.join()
Exemplo n.º 10
0
    def fingerprint_translation_record_directory(self,
                                                 path,
                                                 extensions,
                                                 nprocesses=None):

        msg = "Starting fingerprint translation record directory"
        print msg
        self.logger.info(msg)

        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count()
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)

        filenames_to_fingerprint = []
        for filename, _ in decoder.find_files(path, extensions):

            basename = os.path.basename(filename)
            if "_recording" in basename:
                continue
            # don't refingerprint already fingerprinted files
            if decoder.unique_hash(filename) in self.songhashes_set:
                msg = "%s already fingerprinted, continuing..." % filename
                print msg
                self.logger.info(msg)
                #os.remove(filename)
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        worker_input = zip(filenames_to_fingerprint,
                           [self.limit] * len(filenames_to_fingerprint))

        # Send off our tasks
        iterator = pool.imap_unordered(_fingerprint_worker, worker_input)

        # Loop till we have all of them
        while True:
            try:
                song_name, hashes, file_hash = iterator.next()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                msg = "Failed fingerprinting"
                print msg
                self.logger.info(msg)
                # Print traceback because we can't reraise it here
                traceback.print_exc(file=sys.stdout)
            else:
                sid = self.db.insert_song(song_name, file_hash)

                self.db.insert_repeat_hashes(sid, hashes)
                self.db.set_song_fingerprinted(sid)
                self.get_fingerprinted_songs()

        for filename in filenames_to_fingerprint:
            os.remove(filename)

        pool.close()
        pool.join()
Exemplo n.º 11
0
    def fingerprint_directory(self, path, extensions, nprocesses=None):
        '''对音频进行编码,并存到数据库中,每个音频先分成声道,再对每个声道按照一节一节的编码
            Args:
                path: 音频的路径
                extensions: 扩展名
                nprocesses: 多进程的数量
        '''
        # Try to use the maximum amount of processes if not given.
        try:
            nprocesses = nprocesses or multiprocessing.cpu_count(
            )  # 返回当前系统有多少个cpu
        except NotImplementedError:
            nprocesses = 1
        else:
            nprocesses = 1 if nprocesses <= 0 else nprocesses

        pool = multiprocessing.Pool(nprocesses)  # 进程池

        filenames_to_fingerprint = []  # 需要进行fingerprint的音频路径
        for filename, _ in decoder.find_files(path, extensions):

            # don't refingerprint already fingerprinted files
            if decoder.unique_hash(
                    filename
            ) in self.songhashes_set:  # 对当前音频进行sha1编码,然后和所有歌曲的sha1编码list对比,有则不重复编码
                print("%s already fingerprinted, continuing..." % filename)
                continue

            filenames_to_fingerprint.append(filename)

        # Prepare _fingerprint_worker input
        # worker_input为要进行fingerprint的zip元组
        worker_input = zip(
            filenames_to_fingerprint, [self.limit] *
            len(filenames_to_fingerprint))  # [None]*2==[None, None]

        # Send off our tasks
        # 每个多进程处理一首歌,进程数与cpu数相同,比如有5首歌,开了4个进程,这个4个进程处理完4首歌后,再从进程池里拿一个进程处理最后一首歌
        iterator = pool.imap_unordered(
            _fingerprint_worker, worker_input
        )  # imap_unordered和map类似,第一个参数是函数,第二个参数是迭代器,将迭代器中的数放到函数里执行

        # Loop till we have all of them
        while True:
            try:
                # _fingerprint_worker()对每个音频编码的结果
                song_name, hashes, file_hash = iterator.next()
            except multiprocessing.TimeoutError:
                continue
            except StopIteration:
                break
            except:
                print("Failed fingerprinting")
                # Print traceback because we can't reraise it here
                traceback.print_exc(file=sys.stdout)
            else:
                # 这个地方就可以把song_name和file_hash和hashes存到文本给洪宁,不过还得看一下,他是怎么听歌识曲的
                sid = self.db.insert_song(
                    song_name, file_hash)  # 将音频name和使用sha1对音频编码存到song表里

                self.db.insert_hashes(
                    sid,
                    hashes)  # 将插入song表中对应的id和hashes存到fingerprint表中,这样sid就是外键
                self.db.set_song_fingerprinted(
                    sid)  # 将song表中用于标志一首歌的指纹被存到数据库中的属性,改为1
                self.get_fingerprinted_songs()

        pool.close()
        pool.join()