Пример #1
0
def audio_encoder(sources):
    # Parse configuration
    read_config_file = (sources.argv.argv.skip(1).let(
        argparse.argparse,
        parser=Observable.just(
            argparse.Parser(description="audio encode server")),
        arguments=Observable.from_([
            argparse.ArgumentDef(name='--config',
                                 help="Path of the server configuration file")
        ])).filter(lambda i: i.key == 'config').map(
            lambda i: file.Read(id='config', path=i.value)))
    config = sources.file.response.let(parse_config)

    # Transcode request handling
    encode_init = (config.map(
        lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))

    encode_request = (
        sources.httpd.route.filter(lambda i: i.id == 'flac_transcode').
        flat_map(lambda i: i.request).map(lambda i: encoder.EncodeMp3(
            id=i.context, data=i.data, key=i.match_info['key'])))
    encoder_request = Observable.merge(encode_init, encode_request)

    # http server
    http_init = (config.flat_map(lambda i: Observable.from_([
        httpd.Initialize(request_max_size=0),
        httpd.AddRoute(
            methods=['POST'],
            path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
            id='flac_transcode',
        ),
        httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
    ])))

    http_response = (sources.encoder.response.map(lambda i: httpd.Response(
        data='ok'.encode('utf-8'),
        context=i.id,
    )))
    http = Observable.merge(http_init, http_response)

    # merge sink requests
    file_requests = read_config_file

    return Sink(
        encoder=encoder.Sink(request=encoder_request),
        file=file.Sink(request=file_requests),
        httpd=httpd.Sink(control=http),
    )
Пример #2
0
    def test_parse_bad_arg(self):
        args = argparse.argparse(
            Observable.from_(["--bar", "barz"]),
            Observable.just(argparse.Parser(description="test_parse")),
            Observable.from_([
                argparse.ArgumentDef(name="--foo"),
            ]))

        actual_result = None

        def on_error(error):
            nonlocal actual_result
            actual_result = error

        args.subscribe(on_error=on_error)
        self.assertIn("unrecognized arguments", actual_result)
Пример #3
0
    def test_parse(self):
        args = argparse.argparse(
            Observable.from_(["--foo", "fooz"]),
            Observable.just(argparse.Parser(description="test_parse")),
            Observable.from_([
                argparse.ArgumentDef(name="--foo"),
            ]))

        expected_result = [argparse.Argument(key="foo", value="fooz")]

        actual_result = None

        def set_result(i):
            nonlocal actual_result
            actual_result = i

        args.to_list().subscribe(set_result)
        self.assertEqual(expected_result, actual_result)
Пример #4
0
def read_configuration(source):
    args = argparse.argparse(
        argv=source.argv.skip(1),
        parser=Observable.just(argparse.Parser(description="deepspeaker")),
        arguments=Observable.from_([
            argparse.ArgumentDef(name='--config',
                                 help="Path of the configuration file")
        ]))

    read_config_file = Observable.just(
        file.Context(id='config',
                     observable=args.filter(lambda i: i.key == 'config').map(
                         lambda i: file.Read(id='config', path=i.value))))

    config = (source.file_response.filter(lambda i: i.id == "config").flat_map(
        lambda i: i.observable
    ).flat_map(lambda i: i.data).map(lambda i: json.loads(
        i, object_hook=lambda d: namedtuple('x', d.keys())(*d.values()))))

    return Sink(file_request=read_config_file, configuration=config)
Пример #5
0
def deepspeech_server(sources):
    argv = sources.argv.argv
    stt = sources.httpd.route
    stt_response = sources.deepspeech.text.share()
    ds_logs = sources.deepspeech.log
    config_data = sources.file.response

    http_ds_error, route_ds_error = make_error_router()

    args = argparse.argparse(
        argv=argv.skip(1).subscribe_on(aio_scheduler),
        parser=Observable.just(
            argparse.Parser(description="deepspeech server")),
        arguments=Observable.from_([
            argparse.ArgumentDef(name='--config',
                                 help="Path of the server configuration file")
        ]))

    config_file = (args.filter(lambda i: i.key == 'config').map(
        lambda i: file.Read(id='config', path=i.value)))
    config = parse_config(config_data).subscribe_on(aio_scheduler)

    logs_config = (config.flat_map(lambda i: Observable.from_(i.log.level).map(
        lambda i: logging.SetLevel(logger=i.logger, level=i.level)).concat(
            Observable.just(logging.SetLevelDone()))))

    logs = Observable.merge(logs_config, ds_logs)
    log_ready = sources.logging.response.take(1)

    ds_stt = (stt.flat_map(lambda i: i.request).map(
        lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)))

    ds_arg = (
        # config is hot, the combine operator allows to keep its last value
        # until logging is initialized
        log_ready.combine_latest(
            config, lambda _, i: i).map(lambda i: deepspeech.Initialize(
                model=i.deepspeech.model,
                alphabet=i.deepspeech.alphabet,
                lm=i.deepspeech.lm,
                trie=i.deepspeech.trie,
                features=deepspeech.FeaturesParameters(
                    n_features=i.deepspeech.features.n_features,
                    n_context=i.deepspeech.features.n_context,
                    beam_width=i.deepspeech.features.beam_width,
                    lm_alpha=i.deepspeech.features.lm_alpha,
                    lm_beta=i.deepspeech.features.lm_beta,
                ) if i.deepspeech.features is not None else None)))
    ds = ds_stt.merge(ds_arg)

    http_init = (config.flat_map(lambda i: Observable.from_([
        httpd.Initialize(request_max_size=i.server.http.request_max_size),
        httpd.AddRoute(
            methods=['POST'],
            path='/stt',
            id='stt',
        ),
        httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
    ])))

    http_response = (stt_response.let(lambda x: route_ds_error(
        x,
        error_map=lambda e: httpd.Response(
            data="Speech to text error".encode('utf-8'),
            context=e.args[0].context,
            status=500))).map(lambda i: httpd.Response(
                data=i.text.encode('utf-8'),
                context=i.context,
            )))

    http = Observable.merge(http_init, http_response, http_ds_error)

    return DeepspeechSink(file=file.Sink(request=config_file),
                          logging=logging.Sink(request=logs),
                          deepspeech=deepspeech.Sink(speech=ds),
                          httpd=httpd.Sink(control=http))
def audio_encoder(sources):
    http_s3_error, route_s3_error = make_error_router()
    http_encode_error, route_encode_error = make_error_router()

    # Parse configuration
    parsed_argv = (sources.argv.argv.skip(1).let(
        argparse.argparse,
        parser=Observable.just(
            argparse.Parser(description="audio encode server")),
        arguments=Observable.from_([
            argparse.ArgumentDef(name='--config',
                                 help="Path of the server configuration file")
        ])).filter(lambda i: i.key == 'config').subscribe_on(
            aio_scheduler).share())

    # monitor and parse config file
    monitor_init = (parsed_argv.flat_map(lambda i: Observable.from_([
        inotify.AddWatch(
            id='config', path=i.value, flags=aionotify.Flags.MODIFY),
        inotify.Start(),
    ])))

    config_update = (sources.inotify.response.debounce(5000).map(
        lambda i: True).start_with(True))

    read_config_file = (Observable.combine_latest(
        parsed_argv, config_update,
        lambda config, _: file.Read(id='config', path=config.value)))
    config = sources.file.response.let(parse_config)

    # Transcode request handling
    encode_init = (config.map(lambda i: i.encode).distinct_until_changed().map(
        lambda i: encoder.Configure(samplerate=i.samplerate,
                                    bitdepth=i.bitdepth)))

    encode_request = (
        sources.httpd.route.filter(lambda i: i.id == 'flac_transcode').
        flat_map(lambda i: i.request).flat_map(lambda i: Observable.just(
            i, encode_scheduler)).map(lambda i: encoder.EncodeMp3(
                id=i.context, data=i.data, key=i.match_info['key'])))
    encoder_request = Observable.merge(encode_init, encode_request)

    # store encoded file
    store_requests = (sources.encoder.response.let(
        catch_or_flat_map,
        error_router=route_encode_error,
        error_map=lambda i: httpd.Response(data='encode error'.encode('utf-8'),
                                           context=i.args[0].id,
                                           status=500)).
                      observe_on(s3_scheduler).map(lambda i: s3.UploadObject(
                          key=i.key + '.flac',
                          data=i.data,
                          id=i.id,
                      )))

    # acknowledge http request
    http_response = (sources.s3.response.let(
        catch_or_flat_map,
        error_router=route_s3_error,
        error_map=lambda i: httpd.Response(data='upload error'.encode('utf-8'),
                                           context=i.args[0].id,
                                           status=500)).map(
                                               lambda i: httpd.Response(
                                                   data='ok'.encode('utf-8'),
                                                   context=i.id,
                                               )))

    # http server
    http_init = (config.take(1).flat_map(lambda i: Observable.from_([
        httpd.Initialize(request_max_size=0),
        httpd.AddRoute(
            methods=['POST'],
            path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
            id='flac_transcode',
        ),
        httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
    ])))
    http = Observable.merge(http_init, http_response, http_s3_error,
                            http_encode_error)

    # s3 database
    s3_init = (config.take(1).map(lambda i: s3.Configure(
        access_key=i.s3.access_key,
        secret_key=i.s3.secret_key,
        bucket=i.s3.bucket,
        endpoint_url=i.s3.endpoint_url,
        region_name=i.s3.region_name,
    )))

    # merge sink requests
    file_requests = read_config_file
    s3_requests = Observable.merge(s3_init, store_requests)

    return Sink(
        encoder=encoder.Sink(request=encoder_request),
        s3=s3.Sink(request=s3_requests),
        file=file.Sink(request=file_requests),
        httpd=httpd.Sink(control=http),
        inotify=inotify.Sink(request=monitor_init),
    )
def audio_encoder(sources):
    # Parse configuration
    read_config_file = (
        sources.argv.argv.skip(1)
        .let(argparse.argparse,
            parser=Observable.just(
                argparse.Parser(description="audio encode server")),
            arguments=Observable.from_([
                argparse.ArgumentDef(
                    name='--config', help="Path of the server configuration file")
            ]))
        .filter(lambda i: i.key == 'config')
        .map(lambda i: file.Read(id='config', path=i.value))
    )
    config = sources.file.response.let(parse_config)

    # Transcode request handling
    encode_init = (
        config
        .map(lambda i: encoder.Initialize(storage_path=i.encode.storage_path))
    )

    encode_request = (
        sources.httpd.route
        .filter(lambda i: i.id == 'flac_transcode')
        .flat_map(lambda i: i.request)
        .do_action(lambda i: print("[{}]http req: {}".format(datetime.datetime.now(), threading.get_ident())))
        #.observe_on(encode_scheduler)
        .flat_map(lambda i: Observable.just(i, encode_scheduler))
        .do_action(lambda i: print("[{}]encode req: {}".format(datetime.datetime.now(), threading.get_ident())))
        .map(lambda i: encoder.EncodeMp3(
            id=i.context,
            data=i.data,
            key=i.match_info['key']))
    )
    encoder_request = Observable.merge(encode_init, encode_request)

    # store encoded file
    store_requests = (
        sources.encoder.response
        .do_action(lambda i: print("[{}]encode res: {}".format(datetime.datetime.now(), threading.get_ident())))
        .observe_on(s3_scheduler)
        .do_action(lambda i: print("[{}]s3 req: {}".format(datetime.datetime.now(), threading.get_ident())))
        .map(lambda i: s3.UploadObject(
            key=i.key + '.flac',
            data=i.data,
            id=i.id,
        ))
    )

    # acknowledge http request
    http_response = (
        sources.s3.response
        .do_action(lambda i: print("[{}]s3 res: {}".format(datetime.datetime.now(), threading.get_ident())))
        .do_action(lambda i: print("httpd res: {}".format(threading.get_ident())))
        .map(lambda i: httpd.Response(
            data='ok'.encode('utf-8'),
            context=i.id,
        ))
    )

    # http server
    http_init = (
        config
        .flat_map(lambda i: Observable.from_([
            httpd.Initialize(request_max_size=0),
            httpd.AddRoute(
                methods=['POST'],
                path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
                id='flac_transcode',
            ),
            httpd.StartServer(
                host=i.server.http.host,
                port=i.server.http.port),
        ]))
    )
    http = Observable.merge(http_init, http_response)

    # s3 database
    s3_init = (
        config
        .map(lambda i: s3.Configure(
            access_key=i.s3.access_key,
            secret_key=i.s3.secret_key,
            bucket=i.s3.bucket,
            endpoint_url=i.s3.endpoint_url,
            region_name=i.s3.region_name,
        ))
    )

    # merge sink requests
    file_requests = read_config_file
    s3_requests = Observable.merge(s3_init, store_requests)

    return Sink(
        encoder=encoder.Sink(request=encoder_request),
        s3=s3.Sink(request=s3_requests),
        file=file.Sink(request=file_requests),
        httpd=httpd.Sink(control=http),
    )