def main(p_port):
    if p_port == 0:
        print 'port could not be set as 0'
	log.e('port could not be set as 0')
	exit(1)
    log.c('www listening on port : %s' % p_port)
    app = Application()
    app.listen(p_port)
    log.i(app.settings['template_path'])
    log.i(app.settings['static_path'])
    tornado.ioloop.IOLoop.instance().start()
Exemple #2
0
def make_response(data, success=True):

    try:
        if success:
            data['response'] = 'success'
        else:
            data['response'] = 'error'
        return json.dumps(data, ensure_ascii=False, encoding='iso-8859-1')
    except:
        log.e(('Response Error!', data), discard=True)
        data = {}
        data['response'] = 'error'
        return json.dumps(data)
Exemple #3
0
def parseDataset():

    CLASSES = [
        c for c in sorted(os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train')))
    ]

    for c in CLASSES:
        log.i(c)
        index = []
        afiles = [
            f for f in sorted(
                os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train', c)))
        ]
        max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1
        for i in range(len(afiles)):

            spec_cnt = 0
            try:
                print(i + 1, '/', len(afiles), c, afiles[i])
                specs, noise = getSpecs(
                    os.path.join(cfg.TRAINSET_PATH, 'train', c, afiles[i]))
                for s in range(len(specs)):
                    if np.isnan(noise[s]):
                        noise[s] = 0.0
                    if noise[s] >= cfg.SPEC_SIGNAL_THRESHOLD:
                        filepath = os.path.join(cfg.DATASET_PATH, c)
                        if not os.path.exists(filepath):
                            os.makedirs(filepath)
                    elif noise[s] <= cfg.NOISE_THRESHOLD:
                        if RANDOM.choice([True, False], p=[0.60, 0.40]):
                            filepath = os.path.join(cfg.NOISE_PATH)
                            if not os.path.exists(filepath):
                                os.makedirs(filepath)
                        else:
                            filepath = None

                    else:
                        filepath = None
                    if filepath:
                        filename = str(
                            int(noise[s] *
                                10000)).zfill(4) + '_' + afiles[i].split(
                                    '.')[0] + '_' + str(s).zfill(3)
                        # Write to HDD
                        cv2.imwrite(os.path.join(filepath, filename + '.png'),
                                    specs[s] * 255.0)
                    if spec_cnt >= max_specs:
                        break
            except:
                log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
                continue
Exemple #4
0
def parseDataset():

    # List of classes, subfolders as class names
    CLASSES = [
        c for c in sorted(os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train')))
    ]

    # Parse every class
    for c in CLASSES:

        # List all audio files
        afiles = [
            f for f in sorted(
                os.listdir(os.path.join(cfg.TRAINSET_PATH, 'train', c)))
        ]

        # Calculate maximum specs per file
        max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1

        # Get specs for every audio file
        for i in range(len(afiles)):

            spec_cnt = 0

            try:

                # Stats
                print i + 1, '/', len(afiles), c, afiles[i],

                # Get specs and signal to noise ratios
                specs, noise = getSpecs(
                    os.path.join(cfg.TRAINSET_PATH, 'train', c, afiles[i]))

                # Save specs if it contains signal
                for s in range(len(specs)):

                    # NaN?
                    if np.isnan(noise[s]):
                        noise[s] = 0.0

                    # Above SIGNAL_THRESHOLD?
                    if noise[s] >= cfg.SPEC_SIGNAL_THRESHOLD:

                        # Create target path for accepted specs
                        filepath = os.path.join(cfg.DATASET_PATH, c)
                        if not os.path.exists(filepath):
                            os.makedirs(filepath)

                        # Count specs
                        spec_cnt += 1

                    else:

                        # Create target path for rejected specs -
                        # but we don't want to save every sample (only 10%)
                        if RANDOM.choice([True, False], p=[0.1, 0.90]):
                            filepath = os.path.join(cfg.NOISE_PATH)
                            if not os.path.exists(filepath):
                                os.makedirs(filepath)
                        else:
                            filepath = None

                    if filepath:
                        # Filename contains s2n-ratio
                        filename = str(
                            int(noise[s] *
                                10000)).zfill(4) + '_' + afiles[i].split(
                                    '.')[0] + '_' + str(s).zfill(3)

                        # Write to HDD
                        cv2.imwrite(os.path.join(filepath, filename + '.png'),
                                    specs[s] * 255.0)

                    # Do we have enough specs already?
                    if spec_cnt >= max_specs:
                        break

                # Stats
                log.i((spec_cnt, 'specs'))

            except:
                log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
                continue
Exemple #5
0
def parseDataset():
    spec_all = []

    # List of classes, subfolders as class names
    CLASSES = [
        c for c in sorted(os.listdir(os.path.join(cfg.TRAINSET_PATH, 'audio')))
    ]
    stop = 0
    #switch = 0
    # Parse every class
    for c in CLASSES:
        #if stop==1:
        #break
        #if c == 'norfli':
        #   switch = 1
        #if switch == 1:
        # List all audio files
        afiles = [
            f for f in sorted(
                os.listdir(os.path.join(cfg.TRAINSET_PATH, 'audio', c)))
        ]

        # Calculate maximum specs per file
        max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1

        # Get specs for every audio file
        for i in range(len(afiles)):

            spec_cnt = 0

            try:

                # Stats
                print(i + 1, '/', len(afiles), c, afiles[i])

                # Get specs and signal to noise ratios
                specs, noise = getSpecs(
                    os.path.join(cfg.TRAINSET_PATH, 'audio', c, afiles[i]))
                # print('number of specs = ' + str(specs))
                threshold = 0
                max_index = 0
                # Save specs if it contains signal
                for s in range(len(specs)):
                    # NaN?
                    if np.isnan(noise[s]):
                        noise[s] = 0.0

                    # Above SIGNAL_THRESHOLD?
                    if noise[s] >= threshold:
                        threshold = noise[s]
                        max_index = s
                        #print(threshold)
                        # Create target path for accepted specs
                        spec_filepath = os.path.join(cfg.DATASET_PATH, c)
                        if not os.path.exists(spec_filepath):
                            os.makedirs(spec_filepath)

                    else:
                        # Create target path for rejected specs -
                        # but we don't want to save every sample (only 10%)
                        if RANDOM.choice([True, False], p=[0.1, 0.90]):
                            filepath = os.path.join(cfg.NOISE_PATH)
                            if not os.path.exists(filepath):
                                os.makedirs(filepath)
                            if filepath:
                                # Filename contains s2n-ratio
                                filename = str(int(
                                    noise[s] *
                                    10000)).zfill(4) + '_' + afiles[i].split(
                                        '.')[0] + '_' + str(s).zfill(3)

                                # Write to HDD
                                cv2.imwrite(
                                    os.path.join(filepath, filename + '.png'),
                                    specs[s] * 255.0)

                        else:
                            filepath = None

                    if s == (len(specs) - 1):
                        # Filename contains s2n-ratio
                        filename = str(
                            int(noise[max_index] *
                                10000)).zfill(4) + '_' + afiles[i].split(
                                    '.')[0] + '_' + str(s).zfill(3)

                        # Write to HDD
                        cv2.imwrite(
                            os.path.join(spec_filepath, filename + '.png'),
                            specs[max_index] * 255.0)

                        # Save all spec into a list
                        spec_all.append(specs[max_index])

                        # Count specs
                        spec_cnt += 1

                    # Do we have enough specs already?
                    if spec_cnt >= max_specs:
                        break

                # Stats
                log.i((spec_cnt, 'specs'))

            except:
                log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
                continue
        stop += 1
    print(spec_all)
    spec_all = np.array(spec_all)
    spec_mean = np.mean(spec_all)
    spec_std = np.std(spec_all)
    print(spec_mean)
    print(spec_std)
Exemple #6
0
def parseDataset():

    filedata = {}
    with open(r"flockfiles.json", 'r') as f:
        filedata = json.load(f)

    # List of classes, subfolders as class names
    CLASSES = [c for c in sorted(os.listdir(r""))]
    CLASSES = CLASSES[28:]

    # Parse every class
    for c in CLASSES:
        log.i(c)
        index = []
        # List all audio files
        print(c)
        afiles = [f for f in sorted(os.listdir(os.path.join(r"", c)))]
        print(len(afiles))
        # Calculate maximum specs per file
        for i in range(len(afiles)):
            if (afiles[i][:-4] + '.WAV' in filedata[c]):
                index.append(i)
                print("del one")
        index.reverse()
        for i in index:
            del afiles[i]
        print(len(afiles))

        max_specs = cfg.MAX_SPECS_PER_CLASS // len(afiles) + 1

        # Get specs for every audio file
        for i in range(len(afiles)):

            spec_cnt = 0

            try:

                # Stats
                print(i + 1, '/', len(afiles), c, afiles[i])

                # Get specs and signal to noise ratios
                specs, noise = getSpecs(os.path.join(r"", c, afiles[i]))
                # Save specs if it contains signal
                for s in range(len(specs)):
                    # NaN?
                    if np.isnan(noise[s]):
                        noise[s] = 0.0

                    # Above SIGNAL_THRESHOLD?
                    if noise[s] >= cfg.SPEC_SIGNAL_THRESHOLD:
                        # Create target path for accepted specs
                        filepath = os.path.join(cfg.DATASET_PATH, c)
                        if not os.path.exists(filepath):
                            os.makedirs(filepath)
                    elif noise[s] <= cfg.NOISE_THRESHOLD:
                        if RANDOM.choice([True, False], p=[0.60, 0.40]):
                            filepath = os.path.join(cfg.NOISE_PATH)
                            if not os.path.exists(filepath):
                                os.makedirs(filepath)
                        else:
                            filepath = None

                    else:
                        filepath = None
                    if filepath:
                        # Filename contains s2n-ratio
                        filename = str(
                            int(noise[s] *
                                10000)).zfill(4) + '_' + afiles[i].split(
                                    '.')[0] + '_' + str(s).zfill(3)
                        # Write to HDD
                        cv2.imwrite(os.path.join(filepath, filename + '.png'),
                                    specs[s] * 255.0)
                    # Do we have enough specs already?
                    if spec_cnt >= max_specs:
                        break

            except:
                log.e((spec_cnt, 'specs', 'ERROR DURING SPEC EXTRACT'))
                continue
Exemple #7
0
def test(SNAPSHOTS):

    # Do we have more than one snapshot?
    if not isinstance(SNAPSHOTS, (list, tuple)):
        SNAPSHOTS = [SNAPSHOTS]

    # Load snapshots
    test_functions = []
    for s in SNAPSHOTS:

        # Settings
        NET = s['net']
        cfg.CLASSES = s['classes']
        cfg.IM_DIM = s['im_dim']
        cfg.IM_SIZE = s['im_size']

        # Compile test function
        test_net = birdnet.test_function(NET, hasTargets=False, layer_index=-1)
        test_functions.append(test_net)

    # Parse Testset
    TEST = parseTestSet()

    # Status
    log.i('START TESTING...')
    stats.clearStats()
    stats.tic('test_time')

    # Make predictions
    for spec_batch, labels, filename in bg.threadedGenerator(
            getSpecBatches(TEST)):

        try:

            # Status
            stats.tic('pred_time')

            # Prediction
            prediction_batch = []
            for test_func in test_functions:
                if len(prediction_batch) == 0:
                    prediction_batch = test_func(spec_batch)
                else:
                    prediction_batch += test_func(spec_batch)
            prediction_batch /= len(test_functions)

            # Eliminate the scores for 'Noise'
            if 'Noise' in cfg.CLASSES:
                prediction_batch[:, cfg.CLASSES.index('Noise')] = np.min(
                    prediction_batch)

            # Prediction pooling
            p_pool = predictionPooling(prediction_batch)

            # Get class labels
            p_labels = {}
            for i in range(p_pool.shape[0]):
                p_labels[cfg.CLASSES[i]] = p_pool[i]

            # Sort by score
            p_sorted = sorted(p_labels.items(),
                              key=operator.itemgetter(1),
                              reverse=True)

            # Calculate MLRAP (MRR for single labels)
            targets = np.zeros(p_pool.shape[0], dtype='float32')
            for label in labels:
                if label in cfg.CLASSES:
                    targets[cfg.CLASSES.index(label)] = 1.0
            lrap = metrics.lrap(np.expand_dims(p_pool, 0),
                                np.expand_dims(targets, 0))
            stats.setValue('lrap', lrap, mode='append')

            # Show sample stats
            log.i((filename), new_line=True)
            log.i(('\tLABELS:', labels), new_line=True)
            log.i(('\tTOP PREDICTION:', p_sorted[0][0],
                   int(p_sorted[0][1] * 1000) / 10.0, '%'),
                  new_line=True)
            log.i(('\tLRAP:', int(lrap * 1000) / 1000.0), new_line=False)
            log.i(('\tMLRAP:',
                   int(np.mean(stats.getValue('lrap')) * 1000) / 1000.0),
                  new_line=True)

            # Save some stats
            if p_sorted[0][0] == labels[0]:
                stats.setValue('top1_correct', 1, 'add')
                stats.setValue('top1_confidence', p_sorted[0][1], 'append')
            else:
                stats.setValue('top1_incorrect', 1, 'add')
            stats.toc('pred_time')
            stats.setValue('time_per_batch', stats.getValue('pred_time'),
                           'append')

        except KeyboardInterrupt:
            cfg.DO_BREAK = True
            break
        except:
            log.e('ERROR WHILE TRAINING!')
            continue

    # Stats
    stats.toc('test_time')
    log.i(('TESTING DONE!', 'TIME:', stats.getValue('test_time'), 's'))
    log.r(('FINAL MLRAP:', np.mean(stats.getValue('lrap'))))
    log.r(('TOP 1 ACCURACY:',
           max(
               0,
               float(stats.getValue('top1_correct')) /
               (stats.getValue('top1_correct') +
                stats.getValue('top1_incorrect')))))
    log.r(('TOP 1 MEAN CONFIDENCE:',
           max(0, np.mean(stats.getValue('top1_confidence')))))
    log.r(('TIME PER BATCH:',
           int(np.mean(stats.getValue('time_per_batch')) * 1000), 'ms'))

    return np.mean(stats.getValue('lrap')), int(
        np.mean(stats.getValue('time_per_file')) * 1000)
Exemple #8
0
                        help="Target script name",
                        action="store",
                        default=None)
    parser.add_argument("-c",
                        "--target-count",
                        help="Target instance count",
                        action="store",
                        type=int,
                        default=1)
    parser.add_argument("--shell",
                        help="Run command as shell mode",
                        action="store_true",
                        default=False)
    parser.add_argument("--socket",
                        help="Unix domain socket path",
                        action="store",
                        default=None)

    args = parser.parse_args()
    log.init(args.name)

    log.i(__name__, "Start QB Incubator")
    if args.script is None and args.target is None:
        log.e(__name__, "--script and --target are not set.")
        exit(1)

    if args.script is not None:
        script_main(args.shell, args.script, args.socket)
    else:
        target_main(args.shell, args.target, args.target_count, args.socket)
    def __exit__(self, exc_type, exc_val, exc_tb):
        try:
	    self.session.rollback()
	except:
	    log.e('session rollback error')
	self.session.close()