Ejemplo n.º 1
0
def main():
    args = create_parser(usage).parse_args()
    args.tags_file = abspath(args.tags_file) if args.tags_file else None
    args.folder = abspath(args.folder)
    args.output_folder = abspath(args.output_folder)
    noise_min, noise_max = args.noise_ratio_low, args.noise_ratio_high

    data = TrainData.from_both(args.tags_file, args.folder, args.folder)
    noise_data = NoiseData(args.noise_folder)
    print('Data:', data)

    def translate_filename(source: str, n=0) -> str:
        assert source.startswith(args.folder)
        relative_file = source[len(args.folder):].strip(os.path.sep)
        if n > 0:
            base, ext = splitext(relative_file)
            relative_file = base + '.' + str(n) + ext
        return join(args.output_folder, relative_file)

    all_filenames = sum(data.train_files + data.test_files, [])
    for i, filename in enumerate(all_filenames):
        print('{0:.2%}  \r'.format(i / (len(all_filenames) - 1)), end='', flush=True)

        audio = load_audio(filename)
        for n in range(args.inflation_factor):
            altered = noise_data.noised_audio(audio, noise_min + (noise_max - noise_min) * random())
            output_filename = translate_filename(filename, n)

            makedirs(dirname(output_filename), exist_ok=True)
            save_audio(output_filename, altered)

    print('Done!')

    if args.tags_file and args.tags_file.startswith(args.folder):
        shutil.copy2(args.tags_file, translate_filename(args.tags_file))
Ejemplo n.º 2
0
def main():
    args = TrainData.parse_args(create_parser(usage))

    inject_params(args.model)
    save_params(args.model)

    data = TrainData.from_both(args.db_file, args.db_folder, args.data_dir)
    print('Data:', data)
    (inputs, outputs), test_data = data.load(True, not args.no_validation)

    print('Inputs shape:', inputs.shape)
    print('Outputs shape:', outputs.shape)

    if test_data:
        print('Test inputs shape:', test_data[0].shape)
        print('Test outputs shape:', test_data[1].shape)

    if 0 in inputs.shape or 0 in outputs.shape:
        print('Not enough data to train')
        exit(1)

    model = create_model(args.model, args.no_validation, args.extra_metrics)
    model.summary()

    from keras.callbacks import ModelCheckpoint
    checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor,
                                 save_best_only=args.save_best)

    try:
        model.fit(inputs, outputs, 5000, args.epochs, validation_data=test_data,
                  callbacks=[checkpoint])
    except KeyboardInterrupt:
        print()
    finally:
        model.save(args.model)
Ejemplo n.º 3
0
def main():
    args = TrainData.parse_args(create_parser(usage))

    inject_params(args.model)

    data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder)
    train, test = data.load(args.use_train, not args.use_train, shuffle=False)
    inputs, targets = train if args.use_train else test

    filenames = sum(data.train_files if args.use_train else data.test_files,
                    [])
    predictions = Listener.find_runner(args.model)(args.model).predict(inputs)
    stats = Stats(predictions, targets, filenames)

    print('Data:', data)

    if not args.no_filenames:
        fp_files = stats.calc_filenames(False, True, args.threshold)
        fn_files = stats.calc_filenames(False, False, args.threshold)
        print('=== False Positives ===')
        print('\n'.join(fp_files))
        print()
        print('=== False Negatives ===')
        print('\n'.join(fn_files))
        print()
    print(stats.counts_str(args.threshold))
    print()
    print(stats.summary_str(args.threshold))
Ejemplo n.º 4
0
def main():
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
    stdout = sys.stdout
    sys.stdout = sys.stderr

    parser = create_parser(usage)
    parser.add_argument('-v',
                        '--version',
                        action='version',
                        version=__version__)
    parser.add_argument(
        'chunk_size',
        type=int,
        nargs='?',
        default=-1,
        help='Number of bytes to read before making a prediction.'
        'Higher values are less computationally expensive')
    parser.usage = parser.format_usage().strip().replace('usage: ',
                                                         '') + ' < audio.wav'
    args = parser.parse_args()

    if sys.stdin.isatty():
        parser.error('Please pipe audio via stdin using < audio.wav')

    listener = Listener(args.model_name, args.chunk_size)

    try:
        while True:
            conf = listener.update(sys.stdin.buffer)
            stdout.buffer.write((str(conf) + '\n').encode('ascii'))
            stdout.buffer.flush()
    except (EOFError, KeyboardInterrupt):
        pass
Ejemplo n.º 5
0
def _main():
    parser = create_parser(usage)
    parser.add_argument('file_label',
                        nargs='?',
                        help='File label (Ex. recording-##)')
    args = parser.parse_args()
    show_input()
    args.file_label = args.file_label or input(
        "File label (Ex. recording-##): ")
    args.file_label = args.file_label + (''
                                         if '#' in args.file_label else '-##')
    hide_input()

    p = pyaudio.PyAudio()

    while True:
        print('Press space to record (esc to exit)...')

        if not wait_to_continue():
            break

        print('Recording...')
        d = record_until_key(p, args)
        name = next_name(args.file_label)
        save_audio(name, d, args)
        print('Saved as ' + name)

    p.terminate()
Ejemplo n.º 6
0
def main():
    args = create_parser(usage).parse_args()
    import numpy as np

    model_data = {
        name: Stats.from_np_dict(data)
        for name, data in np.load(args.input_file)['data'].item().items()
    }
    model_name = args.model_key or basename(splitext(args.model)[0])

    if model_name not in model_data:
        print("Could not find model '{}' in saved models in stats file: {}".
              format(model_name, list(model_data)))
        raise SystemExit(1)

    stats = model_data[model_name]

    save_spots = (stats.outputs != 0) & (stats.outputs != 1)
    if save_spots.sum() == 0:
        print('No data (or all NaN)')
        return

    stats.outputs = stats.outputs[save_spots]
    stats.targets = stats.targets[save_spots]
    inv = -np.log(1 / stats.outputs - 1)

    pos = np.extract(stats.targets > 0.5, inv)
    pos_mu = pos.mean().item()
    pos_std = sqrt(np.mean((pos - pos_mu)**2)) * args.smoothing

    print('Peak: {:.2f} mu, {:.2f} std'.format(pos_mu, pos_std))
    pr = inject_params(args.model)
    pr.__dict__.update(threshold_config=((pos_mu, pos_std), ))
    save_params(args.model)
    print('Saved params to {}.params'.format(args.model))
Ejemplo n.º 7
0
def main():
    args = create_parser(usage).parse_args()
    
    print('chunk_size: ', args.chunk_size)

    def on_activation():
        activate_notify()

        if args.save_dir:
            global chunk_num
            nm = join(args.save_dir, args.save_prefix + session_id + '.' + str(chunk_num) + '.wav')
            save_audio(nm, audio_buffer)
            print()
            print('Saved to ' + nm + '.')
            chunk_num += 1

    def on_prediction(conf):
        print('!' if conf > 0.8 else '.', end='', flush=True)

    listener = Listener(args.model, args.chunk_size)
    audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float)

    def get_prediction(chunk):
        nonlocal audio_buffer
        audio = buffer_to_audio(chunk)
        audio_buffer = np.concatenate((audio_buffer[len(audio):], audio))
        return listener.update(chunk)

    engine = ListenerEngine(listener, args.chunk_size)
    engine.get_prediction = get_prediction
    runner = PreciseRunner(engine, args.threshold, on_activation=on_activation,
                           on_prediction=on_prediction)
    runner.start()
    Event().wait()  # Wait forever
Ejemplo n.º 8
0
def main():
    args = TrainData.parse_args(create_parser(usage))
    data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder)
    data_files = data.train_files if args.use_train else data.test_files
    listener = PocketsphinxListener(args.key_phrase, args.dict_file,
                                    args.hmm_folder, args.threshold)

    print('Data:', data)
    stats = test_pocketsphinx(listener, data_files)
    show_stats(stats, not args.no_filenames)
Ejemplo n.º 9
0
    def __init__(self):
        super().__init__(create_parser(usage))
        self.bb = BlackBoxOptimizer(file=self.args.trials_name)
        if not self.test:
            data = TrainData.from_both(self.args.tags_file,
                                       self.args.tags_folder, self.args.folder)
            _, self.test = data.load(False, True)

        from keras.callbacks import ModelCheckpoint
        for i in list(self.callbacks):
            if isinstance(i, ModelCheckpoint):
                self.callbacks.remove(i)
Ejemplo n.º 10
0
 def __init__(self):
     parser = create_parser(usage)
     super().__init__(parser)
     if self.args.invert_samples:
         parser.error('--invert-samples should be left blank')
     self.args.samples_file = (self.args.samples_file
                               or '{model_base}.samples.json').format(
                                   model_base=self.model_base)
     self.samples, self.hash_to_ind = self.load_sample_data(
         self.args.samples_file, self.train)
     self.metrics_fiti = Fitipy(self.model_base + '.logs',
                                'sampling-metrics.txt')
def main():
    args = TrainData.parse_args(create_parser(usage))

    for i in (join(args.folder, 'not-wake-word', 'generated'),
              join(args.folder, 'test', 'not-wake-word', 'generated')):
        makedirs(i, exist_ok=True)

    trainer = IncrementalTrainer(args)
    try:
        trainer.train_incremental()
    except KeyboardInterrupt:
        print()
Ejemplo n.º 12
0
def main():
    parser = create_parser(usage)
    parser.add_argument(
        'models',
        nargs='*',
        help='List of model filenames in format: wake-word.yy-mm-dd.net')
    args = TrainData.parse_args(parser)
    if not (bool(args.pocketsphinx_dict) == bool(args.pocketsphinx_folder) ==
            bool(args.pocketsphinx_wake_word)):
        parser.error('Must pass all or no Pocketsphinx arguments')

    data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder)
    data_files = data.train_files if args.use_train else data.test_files
    print('Data:', data)

    metrics = {}

    if args.pocketsphinx_dict and args.pocketsphinx_folder and args.pocketsphinx_wake_word:
        if not isfile(args.pocketsphinx_dict):
            parser.error('No such file: ' + args.pocketsphinx_dict)
        if not isdir(args.pocketsphinx_folder):
            parser.error('No such folder: ' + args.pocketsphinx_folder)
        listener = PocketsphinxListener(args.pocketsphinx_wake_word,
                                        args.pocketsphinx_dict,
                                        args.pocketsphinx_folder,
                                        args.pocketsphinx_threshold)
        stats = test_pocketsphinx(listener, data_files)
        metrics[args.pocketsphinx_dict] = stats_to_dict(stats)

    for model_name in args.models:
        print('Calculating', model_name + '...')
        inject_params(model_name)

        train, test = data.load(args.use_train, not args.use_train)
        inputs, targets = train if args.use_train else test
        predictions = Listener.find_runner(model_name)(model_name).predict(
            inputs)

        stats = Stats(predictions, targets, sum(data_files, []))

        print('----', model_name, '----')
        print(stats.counts_str())
        print()
        print(stats.summary_str())
        print()
        metrics[model_name] = stats.to_dict(args.threshold)

    print('Writing to:', args.output)
    with open(args.output, 'w') as f:
        json.dump(metrics, f)
Ejemplo n.º 13
0
def main():
    args = TrainData.parse_args(create_parser(usage))

    inject_params(args.model)

    data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder)
    train, test = data.load(args.use_train, not args.use_train, shuffle=False)
    inputs, targets = train if args.use_train else test

    filenames = sum(data.train_files if args.use_train else data.test_files,
                    [])
    predictions = Listener.find_runner(args.model)(args.model).predict(inputs)
    stats = calc_stats(filenames, targets, predictions)

    print('Data:', data)
    show_stats(stats, not args.no_filenames)
Ejemplo n.º 14
0
def main():
    args = TrainData.parse_args(create_parser(usage))

    inject_params(args.model)

    data = TrainData.from_both(args.db_file, args.db_folder, args.data_dir)
    train, test = data.load(args.use_train, not args.use_train)
    inputs, targets = train if args.use_train else test

    filenames = sum(data.train_files if args.use_train else data.test_files,
                    [])
    predictions = load_precise_model(args.model).predict(inputs)
    stats = calc_stats(filenames, targets, predictions)

    print('Data:', data)
    show_stats(stats, not args.no_filenames)
Ejemplo n.º 15
0
def main():
    args = create_parser(usage).parse_args()

    def on_activation():
        Popen(['aplay', '-q', 'data/activate.wav'])

    def on_prediction(conf):
        print('!' if conf > 0.5 else '.', end='', flush=True)

    runner = PreciseRunner(ListenerEngine(
        PocketsphinxListener(args.key_phrase, args.dict_file, args.hmm_folder,
                             args.threshold, args.chunk_size)),
                           3,
                           on_activation=on_activation,
                           on_prediction=on_prediction)
    runner.start()
    Event().wait()  # Wait forever
Ejemplo n.º 16
0
def main():
    args = create_parser(usage).parse_args()
    sensitivity = 0.5

    def on_activation():
        activate_notify()

        if args.save_dir:
            global chunk_num
            nm = join(
                args.save_dir,
                args.save_prefix + session_id + '.' + str(chunk_num) + '.wav')
            save_audio(nm, audio_buffer)
            print()
            print('Saved to ' + nm + '.')
            chunk_num += 1

    def on_prediction(conf):
        if args.light_mode:
            print('!' if conf > 0.7 else '.', end='', flush=True)
        else:
            max_width = 80
            width = min(get_terminal_size()[0], max_width)
            units = int(round(conf * width))
            bar = 'X' * units + '-' * (width - units)
            cutoff = round((1.0 - sensitivity) * width)
            print(bar[:cutoff] + bar[cutoff:].replace('X', 'x'))

    listener = Listener(args.model, args.chunk_size)
    audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float)

    def get_prediction(chunk):
        nonlocal audio_buffer
        audio = buffer_to_audio(chunk)
        audio_buffer = np.concatenate((audio_buffer[len(audio):], audio))
        return listener.update(chunk)

    engine = ListenerEngine(listener, args.chunk_size)
    engine.get_prediction = get_prediction
    runner = PreciseRunner(engine,
                           args.threshold,
                           sensitivity=sensitivity,
                           on_activation=on_activation,
                           on_prediction=on_prediction)
    runner.start()
    Event().wait()  # Wait forever
Ejemplo n.º 17
0
    def __init__(self):
        super().__init__(create_parser(usage))

        for i in (join(self.args.folder, 'not-wake-word', 'generated'),
                  join(self.args.folder, 'test', 'not-wake-word',
                       'generated')):
            makedirs(i, exist_ok=True)

        self.trained_fns = load_trained_fns(self.args.model)
        self.audio_buffer = np.zeros(pr.buffer_samples, dtype=float)

        if not isfile(self.args.model):
            params = ModelParams(skip_acc=self.args.no_validation,
                                 extra_metrics=self.args.extra_metrics)
            create_model(self.args.model, params).save(self.args.model)
        self.listener = Listener(self.args.model,
                                 self.args.chunk_size,
                                 runner_cls=KerasRunner)
Ejemplo n.º 18
0
def main():
    args = create_parser(usage).parse_args()

    def on_activation():
        activate_notify()

        if args.save_dir:
            global chunk_num
            nm = join(
                args.save_dir,
                args.save_prefix + session_id + '.' + str(chunk_num) + '.wav')
            save_audio(nm, audio_buffer)
            print()
            print('Saved to ' + nm + '.')
            chunk_num += 1

    def on_prediction(conf):
        global detecting_flag
        # print('!' if conf > 0.5 else '.', end='', flush=True)
        if conf > 0.5:
            detecting_flag = True
        if conf < 0.5 and detecting_flag:
            print(colored("Yeah! I'm Here.", 'green'))
            detecting_flag = False

    sunshine_model = './ok-sunshine.net'
    listener = Listener(sunshine_model, args.chunk_size)
    audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float)

    def get_prediction(chunk):
        nonlocal audio_buffer
        audio = buffer_to_audio(chunk)
        audio_buffer = np.concatenate((audio_buffer[len(audio):], audio))
        return listener.update(chunk)

    engine = ListenerEngine(listener, args.chunk_size)
    engine.get_prediction = get_prediction
    runner = PreciseRunner(engine,
                           args.threshold,
                           on_activation=on_activation,
                           on_prediction=on_prediction)
    runner.start()
    Event().wait()  # Wait forever
Ejemplo n.º 19
0
    def __init__(self):
        super().__init__(create_parser(usage))

        for i in (join(self.args.folder, 'not-wake-word', 'generated'),
                  join(self.args.folder, 'test', 'not-wake-word',
                       'generated')):
            makedirs(i, exist_ok=True)

        self.trained_fns = load_trained_fns(self.args.model)
        self.audio_buffer = np.zeros(pr.buffer_samples, dtype=float)

        params = ModelParams(skip_acc=self.args.no_validation,
                             extra_metrics=self.args.extra_metrics,
                             loss_bias=1.0 - self.args.sensitivity)
        model = create_model(self.args.model, params)
        self.listener = Listener(self.args.model,
                                 self.args.chunk_size,
                                 runner_cls=KerasRunner)
        self.listener.runner = KerasRunner(self.args.model)
        self.listener.runner.model = model
        self.samples_since_train = 0
Ejemplo n.º 20
0
def _main():
    parser = create_parser(usage)
    parser.add_argument('file_label',
                        nargs='?',
                        help='File label (Ex. recording-##)')
    args = parser.parse_args()
    show_input()
    args.file_label = args.file_label or input(
        "File label (Ex. recording-##): ")
    args.file_label = args.file_label + (''
                                         if '#' in args.file_label else '-##')
    hide_input()

    p = pyaudio.PyAudio()

    print("================================================")
    info = p.get_host_api_info_by_index(0)
    numdevices = info.get('deviceCount')
    for i in range(0, numdevices):
        if (p.get_device_info_by_host_api_device_index(
                0, i).get('maxInputChannels')) > 0:
            print("Input Device id ", i, " - ",
                  p.get_device_info_by_host_api_device_index(0, i).get('name'),
                  p.get_device_info_by_index(i).get('defaultSampleRate'))

    print("================================================")

    while True:
        print('Press space to record (esc to exit)...')

        if not wait_to_continue():
            break

        print('Recording...')
        d = record_until_key(p, args)
        name = next_name(args.file_label)
        save_audio(name, d, args)
        print('Saved as ' + name)

    p.terminate()
Ejemplo n.º 21
0
    def __init__(self):
        parser = create_parser(usage)
        self.args = args = TrainData.parse_args(parser)
        self.audio_buffer = np.zeros(pr.buffer_samples, dtype=float)
        self.vals_buffer = np.zeros(pr.buffer_samples, dtype=float)

        params = ModelParams(skip_acc=args.no_validation,
                             extra_metrics=args.extra_metrics,
                             loss_bias=1.0 - args.sensitivity)
        self.model = create_model(args.model, params)
        self.listener = Listener('',
                                 args.chunk_size,
                                 runner_cls=lambda x: None)

        from keras.callbacks import ModelCheckpoint, TensorBoard
        checkpoint = ModelCheckpoint(args.model,
                                     monitor=args.metric_monitor,
                                     save_best_only=args.save_best)
        epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch')
        self.epoch = epoch_fiti.read().read(0, int)

        def on_epoch_end(a, b):
            self.epoch += 1
            epoch_fiti.write().write(self.epoch, str)

        self.model_base = splitext(self.args.model)[0]

        self.callbacks = [
            checkpoint,
            TensorBoard(log_dir=self.model_base + '.logs', ),
            LambdaCallback(on_epoch_end=on_epoch_end)
        ]

        self.data = TrainData.from_both(args.tags_file, args.tags_folder,
                                        args.folder)
        pos_files, neg_files = self.data.train_files
        self.neg_files_it = iter(cycle(neg_files))
        self.pos_files_it = iter(cycle(pos_files))
Ejemplo n.º 22
0
def main():
    args = create_parser(usage).parse_args()

    def on_activation():
        activate_notify()
        # TODO: trigger VMSE

    def on_prediction(conf):
        if args.basic_mode:
            print('!' if conf > 0.7 else '.', end='', flush=True)
        else:
            max_width = 80
            width = min(get_terminal_size()[0], max_width)
            units = int(round(conf * width))
            bar = 'X' * units + '-' * (width - units)
            cutoff = round((1.0 - args.sensitivity) * width)
            print(bar[:cutoff] + bar[cutoff:].replace('X', 'x'))

    listener = Listener(args.model, args.chunk_size)
    audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float)

    def get_prediction(chunk):
        nonlocal audio_buffer
        audio = buffer_to_audio(chunk)
        audio_buffer = np.concatenate((audio_buffer[len(audio):], audio))
        return listener.update(chunk)

    engine = ListenerEngine(listener, args.chunk_size)
    engine.get_prediction = get_prediction
    runner = PreciseRunner(engine,
                           args.trigger_level,
                           sensitivity=args.sensitivity,
                           on_activation=on_activation,
                           on_prediction=on_prediction)
    runner.start()
    Event().wait()  # Wait forever
Ejemplo n.º 23
0
def main():
    args = TrainData.parse_args(create_parser(usage))

    inject_params(args.model)

    data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder)
    #print(data)
    #print(args.use_train)

    train, test = data.load(args.use_train, not args.use_train)
    #print(train)
    #print(test)
    inputs, targets = train if args.use_train else test
    #print(inputs)

    filenames = sum(data.train_files if args.use_train else data.test_files,
                    [])

    #for i in range(0, len(targets)):
    #    print(filenames[i], "\t", targets[i])

    print(args.model)

    predictions = Listener.find_runner(args.model)(args.model).predict(inputs)

    #for i, filename in enumerate(filenames):
    #print(filename, "\t", predictions[i], targets[i])
    #print(predictions[i])

    #print (predictions)
    #print (len(predictions))

    stats = calc_stats(filenames, targets, predictions)

    print('Data:', data)
    show_stats(stats, not args.no_filenames)
Ejemplo n.º 24
0
def main():
    args = create_parser(usage).parse_args()
    github = load_github()

    summaries = {}
    repo = SkillRepo(path=join(gettempdir(), 'mycroft-skills-repo'))
    for skill_entry in MycroftSkillsManager(repo=repo).list():
        if not skill_entry.url:
            continue
        print('Generating {}...'.format(skill_entry.name))
        try:
            summary = generate_summary(github, skill_entry)
        except GithubException as e:
            print('Failed to generate summary:', repr(e))
            continue
        summaries[skill_entry.name] = summary

    if args.output_file:
        with open(args.output_file, 'w') as f:
            json.dump(summaries, f)
    else:
        print(json.dumps(summaries, indent=4))
    if args.upload:
        upload_summaries(github, summaries)
Ejemplo n.º 25
0
def main():
    args = create_parser(usage).parse_args()

    model_name = args.model.replace('.net', '')
    convert(args.model, args.out.format(model=model_name))
Ejemplo n.º 26
0
def main():
    args = create_parser(usage).parse_args()
    num_seen_file = Fitipy(args.cache_file + '.num')
    topics_cache = args.cache_file + '.topics.json'
    if not isfile(topics_cache):
        print('Generating topics...')
        with open(topics_cache, 'w') as f:
            json.dump(get_keywords_uiuc(), f)
    with open(topics_cache) as f:
        topics = json.load(f)

    num_seen = num_seen_file.read().read(0, int)
    with open(args.auth_file) as f:
        auth = yaml.load(f)
    email = auth['username']
    password = auth['password']
    server = auth.get('pop3_host', 'pop3.' + email.split('@')[-1])
    client = StatelessClass(EmailReceiver,
                            email=email,
                            password=password,
                            server=server)  # type: EmailReceiver

    print('Waiting for emails...')
    while True:
        num_messages = len(client.get_list())
        if num_messages < num_seen:
            num_seen = num_messages
            num_seen_file.write().write(num_seen)
        if num_messages <= num_seen:
            time.sleep(1)
            continue
        for msg_id in range(num_seen + 1, num_messages + 1):
            email = client.get_email(msg_id)
            print('Found new email from {} titled {}.'.format(
                email['From'], email['Subject']))
            email_txt = '\n'.join(email['text'])
            email_txt = BeautifulSoup(email_txt).text
            email_txt = re.sub(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))', '',
                               email_txt)

            freq = calc_freq(email_txt, topics)
            tags = relevant_topics(freq)
            print('Found the following tags:', ', '.join(tags))
            events = Event.find()
            matched_events = [
                event for event in events if event.get('emailSrc') and
                SequenceMatcher(a=event['emailSrc'], b=email_txt).ratio() > 0.9
            ]
            if matched_events:
                print('Ignoring, similar to {} other emails'.format(
                    len(matched_events)))
            else:
                Event.add({
                    'name': email['Subject'],
                    'description': email_txt,
                    'location': '',
                    'time': int(time.time()),
                    'tags': tags,
                    'emailSrc': email_txt
                })
            num_seen += 1
            num_seen_file.write().write(num_seen, str)
Ejemplo n.º 27
0
def main():
    args = create_parser(usage).parse_args()
    os.chdir(os.getcwd() + "/Precise")

    def on_activation_normal():
        print("activated\n", flush=True)

    def on_activation_safe():
        global chunk_num
        nm = join(
            args.save_dir,
            args.save_prefix + session_id + '.' + str(chunk_num) + '.wav')
        save_audio(nm, audio_buffer)
        print()
        print('Saved to ' + nm + '.')
        chunk_num += 1

    def on_prediction(conf):
        max_width = 80
        width = min(get_terminal_size()[0], max_width)
        units = int(round(conf * width))
        bar = 'X' * units + '-' * (width - units)
        cutoff = round((1.0 - args.sensitivity) * width)
        print(bar[:cutoff] + bar[cutoff:].replace('X', 'x') + "\n", flush=True)

    def get_prediction(chunk):
        nonlocal audio_buffer
        audio = buffer_to_audio(chunk)
        audio_buffer = np.concatenate((audio_buffer[len(audio):], audio))
        return listener.update(chunk)

    while True:
        line = sys.stdin.readline().rstrip()

        if ("start" in line):
            if (os.path.isfile("./model.pb")):
                listener = Listener("model.pb", args.chunk_size)
            else:
                listener = Listener("model.net", args.chunk_size)
            engine = ListenerEngine(listener, args.chunk_size)
            if ("visual" not in line):
                if (line == "start normal"):
                    runner = PreciseRunner(engine,
                                           args.trigger_level,
                                           sensitivity=args.sensitivity,
                                           on_activation=on_activation_normal)
                elif (line == "start safe"):
                    runner = PreciseRunner(engine,
                                           args.trigger_level,
                                           sensitivity=args.sensitivity,
                                           on_activation=on_activation_safe)
            else:
                if (line == "start normal visual"):
                    audio_buffer = np.zeros(listener.pr.buffer_samples,
                                            dtype=float)
                    engine.get_prediction = get_prediction
                    runner = PreciseRunner(engine,
                                           args.trigger_level,
                                           sensitivity=args.sensitivity,
                                           on_activation=on_activation_normal,
                                           on_prediction=on_prediction)
                elif (line == "start safe visual"):
                    audio_buffer = np.zeros(listener.pr.buffer_samples,
                                            dtype=float)
                    engine.get_prediction = get_prediction
                    runner = PreciseRunner(engine,
                                           args.trigger_level,
                                           sensitivity=args.sensitivity,
                                           on_activation=on_activation_safe,
                                           on_prediction=on_prediction)
            runner.start()
        elif (line == "stop"):
            runner.stop()
        elif (line == "running"):
            print(runner.running)
Ejemplo n.º 28
0
def main():
    args = create_parser(usage).parse_args()
    filenames = glob(join(args.folder, '*.wav'))
    shuffle(filenames)
    wav_id = -1
    stream = None
    stop_event = Event()
    stop_event.set()
    p = PyAudio()
    atexit.register(p.terminate)

    def play_audio(audio_file):
        nonlocal stream
        if stream:
            stop_event.clear()
            stop_event.wait()
            stream.stop_stream()
            stream.close()
            stream = None
        audio = load_audio(audio_file)[-pr.buffer_samples:]
        audio /= 2 * min(audio.mean() + 4 * audio.std(), abs(audio).max())
        stream = p.open(format=paFloat32,
                        channels=1,
                        rate=pr.sample_rate,
                        output=True)
        stream.start_stream()

        def write_audio():
            data = audio.astype('float32').tostring()
            chunk_size = 1024
            for pos in range(chunk_size, len(data) + chunk_size, chunk_size):
                if not stop_event.is_set():
                    stop_event.set()
                    return
                stream.write(data[pos - chunk_size:pos])
            while stop_event.is_set():
                sleep(chunk_size / pr.sample_rate)
            stop_event.set()

        Thread(target=write_audio, daemon=True).start()

    tags_file = Fitipy(args.tags_file)
    tags = tags_file.read().dict()

    def submit():
        nonlocal wav_id
        if wav_id >= 0:
            tags[basename(splitext(filenames[wav_id])[0])] = float(
                slider.get())
            tags_file.write().dict(tags)
        wav_id += 1
        play_audio(filenames[wav_id])

    submit()

    master = Tk()
    label = Label(master, text='0')
    label.pack()

    def on_slider_change(x):
        label['text'] = str(int(float(x)))

    slider = Scale(master, from_=0, to=100, command=on_slider_change)
    slider.pack()

    Button(master, text='Submit', command=submit).pack()
    Button(master,
           text='Replay',
           command=lambda: play_audio(filenames[wav_id])).pack()
    mainloop()
    stream.stop_stream()
    stream.close()
Ejemplo n.º 29
0
def main():
    parser = create_parser(usage)
    parser.add_argument(
        'models',
        nargs='*',
        help='Either Keras (.net) or TensorFlow (.pb) models to test')
    args = TrainData.parse_args(parser)
    if not args.models and not args.input_file and args.folder:
        args.input_file = args.folder
    if bool(args.models) == bool(args.input_file):
        parser.error('Please specify either a list of models or an input file')

    if not args.output_file:
        load_plt()  # Error early if matplotlib not installed
    import numpy as np

    if args.models:
        data = TrainData.from_both(args.tags_file, args.tags_folder,
                                   args.folder)
        print('Data:', data)
        filenames = sum(
            data.train_files if args.use_train else data.test_files, [])
        loader = CachedDataLoader(
            partial(data.load,
                    args.use_train,
                    not args.use_train,
                    shuffle=False))
        model_data = calc_stats(args.models, loader, args.use_train, filenames)
    else:
        model_data = {
            name: Stats.from_np_dict(data)
            for name, data in np.load(args.input_file)['data'].item().items()
        }
        for name, stats in model_data.items():
            print('=== {} ===\n{}\n\n{}\n'.format(name, stats.counts_str(),
                                                  stats.summary_str()))

    if args.output_file:
        np.savez(args.output_file,
                 data={
                     name: stats.to_np_dict()
                     for name, stats in model_data.items()
                 })
    else:
        plt = load_plt()
        decoder = ThresholdDecoder(pr.threshold_config, pr.threshold_center)
        thresholds = [
            decoder.encode(i)
            for i in np.linspace(0.0, 1.0, args.resolution)[1:-1]
        ]
        for model_name, stats in model_data.items():
            x = [stats.false_positives(i) for i in thresholds]
            y = [stats.false_negatives(i) for i in thresholds]
            plt.plot(x, y, marker='x', linestyle='-', label=model_name)
            if args.labels:
                for x, y, threshold in zip(x, y, thresholds):
                    plt.annotate('{:.4f}'.format(threshold), (x, y))

        plt.legend()
        plt.xlabel('False Positives')
        plt.ylabel('False Negatives')
        plt.show()
Ejemplo n.º 30
0
 def __init__(self):
     self.args = create_parser(usage).parse_args()
     inject_params(self.args.model)
     self.runner = Listener.find_runner(self.args.model)(self.args.model)
     self.audio_buffer = np.zeros(pr.buffer_samples, dtype=float)