Ejemplo n.º 1
0
def audio_encoder(sources):
    # Parse configuration
    parser = create_arg_parser()

    read_request, read_response = sources.argv.argv.pipe(
        ops.skip(1),
        argparse.parse(parser),
        ops.filter(lambda i: i.key == 'config'),
        ops.map(lambda i: file.Read(id='config', path=i.value)),
        file.read(sources.file.response),
    )
    config = read_response.pipe(
        ops.filter(lambda i: i.id == "config"),
        ops.flat_map(lambda i: i.data),
        parse_config,
    )

    # Transcode request handling
    encode_init = config.pipe(
        ops.map(
            lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))

    encode_request = sources.httpd.route.pipe(
        ops.filter(lambda i: i.id == 'flac_transcode'),
        ops.flat_map(lambda i: i.request),
        ops.map(lambda i: encoder.EncodeMp3(
            id=i.context, data=i.data, key=i.match_info['key'])),
    )
    encoder_request = rx.merge(encode_init, encode_request)

    # http server
    http_init = config.pipe(
        ops.flat_map(lambda i: rx.from_([
            httpd.Initialize(request_max_size=0),
            httpd.AddRoute(
                methods=['POST'],
                path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
                id='flac_transcode',
            ),
            httpd.StartServer(host=i.server.http.host, port=i.server.http.port
                              ),
        ])))

    http_response = sources.encoder.response.pipe(
        ops.map(lambda i: httpd.Response(
            data='ok'.encode('utf-8'),
            context=i.id,
        )))
    http = rx.merge(http_init, http_response)

    # merge sink requests
    file_requests = read_request

    return Sink(
        encoder=encoder.Sink(request=encoder_request),
        file=file.Sink(request=file_requests),
        httpd=httpd.Sink(control=http),
    )
Ejemplo n.º 2
0
    def test_get(self):
        client_thread = None
        response = None
        loop = asyncio.new_event_loop()
        loop.set_debug(True)
        asyncio.set_event_loop(loop)

        sink = httpd.Sink(control=Subject())

        def setup(sink):
            sink.control.on_next(httpd.Initialize()),
            sink.control.on_next(
                httpd.AddRoute(methods=['GET'], path='/foo', id='foo'))
            sink.control.on_next(httpd.StartServer(host='localhost',
                                                   port=8080)),

        def do_get():
            nonlocal response
            req = urllib.request.urlopen('http://localhost:8080/foo')
            response = req.read()

        # todo rxpy threadpool
        client_thread = threading.Thread(target=do_get)

        def on_server_item(i):
            if type(i) is httpd.ServerStarted:
                nonlocal client_thread
                client_thread.start()
            elif type(i) == httpd.ServerStopped:
                asyncio.get_event_loop().stop()

        def on_route_item(i):
            sink.control.on_next(httpd.Response(context=i.context,
                                                data=b'foo'))
            loop.call_soon(sink.control.on_next, httpd.StopServer())

        loop.call_soon(setup, sink)
        source = httpd.make_driver(loop).call(sink)
        source.route.pipe(
            ops.filter(lambda i: i.id == 'foo'),
            ops.flat_map(lambda i: i.request),
        ).subscribe(on_route_item)

        source.server \
            .subscribe(on_server_item)

        loop.run_forever()
        loop.close()
        client_thread.join()

        self.assertEqual(b'foo', response)
Ejemplo n.º 3
0
def echo_server(source):
    init = rx.from_([
        httpd.Initialize(),
        httpd.AddRoute(methods=['GET'], path='/echo/{what}', id='echo'),
        httpd.StartServer(host='localhost', port=8080),
    ])

    echo = source.httpd.route.pipe(
        ops.flat_map(lambda i: i.request),
        ops.map(lambda i: httpd.Response(
            context=i.context, data=i.match_info['what'].encode('utf-8'))),
    )

    control = rx.merge(init, echo)
    return EchoSink(httpd=httpd.Sink(control=control))
Ejemplo n.º 4
0
    def test_add_route(self):
        routes = [
            httpd.AddRoute(methods=['GET'], path='/foo', id='foo'),
            httpd.AddRoute(methods=['POST'], path='/bar', id='bar'),
            httpd.AddRoute(methods=['PUT'], path='/biz', id='biz'),
        ]
        actual_routes = []
        loop = asyncio.new_event_loop()
        loop.set_debug(True)
        asyncio.set_event_loop(loop)

        sink = httpd.Sink(control=Subject())

        def setup(sink):
            sink.control.on_next(httpd.Initialize()),
            sink.control.on_next(httpd.StartServer(host='localhost',
                                                   port=9999)),
            for route in routes:
                sink.control.on_next(route)

        def on_route_item(i):
            if type(i) is httpd.RouteAdded:
                actual_routes.append(i)
                # stop mainloop when last route is created
                if i.id == routes[-1].id:
                    asyncio.get_event_loop().stop()

        loop.call_soon(setup, sink)
        source = httpd.make_driver(loop).call(sink)
        source.route.subscribe(on_route_item)
        loop.run_forever()
        loop.close()

        self.assertEqual(len(routes), len(actual_routes))
        for index, route in enumerate(actual_routes):
            self.assertEqual(routes[index].path, route.path)
            self.assertEqual(routes[index].id, route.id)
            self.assertIsInstance(route.request, rx.Observable)
Ejemplo n.º 5
0
    def test_start_server(self):
        loop = asyncio.new_event_loop()
        loop.set_debug(True)
        asyncio.set_event_loop(loop)

        sink = httpd.Sink(control=Subject())

        def on_httpd_item(i):
            if type(i) is httpd.ServerStarted:
                sink.control.on_next(httpd.StopServer())
            elif type(i) == httpd.ServerStopped:
                asyncio.get_event_loop().stop()

        def control_stream(sink):
            sink.control.on_next(httpd.Initialize())
            sink.control.on_next(httpd.StartServer(host='localhost',
                                                   port=9999))

        loop.call_soon(control_stream, sink)
        source = httpd.make_driver(loop).call(sink)
        source.server.subscribe(on_httpd_item)

        loop.run_forever()
        loop.close()
Ejemplo n.º 6
0
def deepspeech_server(aio_scheduler, sources):
    argv = sources.argv.argv
    stt = sources.httpd.route
    stt_response = sources.deepspeech.text
    ds_logs = sources.deepspeech.log

    http_ds_error, route_ds_error = make_error_router()

    args = parse_arguments(argv)

    read_request, read_response = args.pipe(
        ops.map(lambda i: file.Read(id='config', path=i.value)),
        file.read(sources.file.response),
    )
    read_request = read_request.pipe(
        ops.subscribe_on(aio_scheduler),
    )
    config = parse_config(read_response)

    logs_config = config.pipe(
        ops.flat_map(lambda i: rx.from_(i.log.level, scheduler=ImmediateScheduler())),
        ops.map(lambda i: logging.SetLevel(logger=i.logger, level=i.level)),
    )
    logs = rx.merge(logs_config, ds_logs)

    ds_stt = stt.pipe(
        ops.flat_map(lambda i: i.request),
        ops.map(lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)),
    )

    # config is hot, the combine operator allows to keep its last value
    # until logging is initialized
    ds_arg = config.pipe(
        ops.map(lambda i: deepspeech.Initialize(
            model=i.deepspeech.model,
            scorer=deepspeech.Scorer(
                scorer=getattr(i.deepspeech, 'scorer', None),
                lm_alpha=getattr(i.deepspeech, 'lm_alpha', None),
                lm_beta=getattr(i.deepspeech, 'lm_beta', None),
            ),
            beam_width=getattr(i.deepspeech, 'beam_width', None),
        )),
    )
    ds = rx.merge(ds_stt, ds_arg)

    http_init = config.pipe(
        ops.flat_map(lambda i: rx.from_([
            httpd.Initialize(request_max_size=i.server.http.request_max_size),
            httpd.AddRoute(
                methods=['POST'],
                path='/stt',
                id='stt',
                headers=MultiDict([('Content-Type', 'text/plain')]),
            ),
            httpd.StartServer(
                host=i.server.http.host,
                port=i.server.http.port),
        ])),
    )

    http_response = stt_response.pipe(
        route_ds_error(
            error_map=lambda e: httpd.Response(
                data="Speech to text error".encode('utf-8'),
                context=e.args[0].context,
                status=500
        )),
        ops.map(lambda i: httpd.Response(
            data=i.text.encode('utf-8'),
            context=i.context,
        )),
    )

    http = rx.merge(http_init, http_response, http_ds_error)

    return DeepspeechSink(
        file=file.Sink(request=read_request),
        logging=logging.Sink(request=logs),
        deepspeech=deepspeech.Sink(speech=ds),
        httpd=httpd.Sink(control=http)
    )
Ejemplo n.º 7
0
def audio_encoder(sources):
    # Parse configuration
    parser = create_arg_parser()

    read_request, read_response = sources.argv.argv.pipe(
        ops.skip(1),
        argparse.parse(parser),
        ops.filter(lambda i: i.key == 'config'),
        ops.map(lambda i: file.Read(id='config', path=i.value)),
        file.read(sources.file.response),
    )
    config = read_response.pipe(
        ops.filter(lambda i: i.id == "config"),
        ops.flat_map(lambda i: i.data),
        parse_config,
    )

    # Transcode request handling
    encode_init = config.pipe(
        ops.map(
            lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))

    encode_request = sources.httpd.route.pipe(
        ops.filter(lambda i: i.id == 'flac_transcode'),
        ops.flat_map(lambda i: i.request),
        ops.do_action(lambda i: print("[{}]http req: {}".format(
            datetime.datetime.now(), threading.get_ident()))),
        #.observe_on(encode_scheduler)
        ops.flat_map(lambda i: Observable.just(i, encode_scheduler)),
        ops.do_action(lambda i: print("[{}]encode req: {}".format(
            datetime.datetime.now(), threading.get_ident()))),
        ops.map(lambda i: encoder.EncodeMp3(
            id=i.context, data=i.data, key=i.match_info['key'])),
    )
    encoder_request = rx.merge(encode_init, encode_request)

    # store encoded file
    store_requests = sources.encoder.response.pipe(
        ops.do_action(lambda i: print("[{}]encode res: {}".format(
            datetime.datetime.now(), threading.get_ident()))),
        ops.observe_on(s3_scheduler),
        ops.do_action(lambda i: print("[{}]s3 req: {}".format(
            datetime.datetime.now(), threading.get_ident()))),
        ops.map(lambda i: s3.UploadObject(
            key=i.key + '.flac',
            data=i.data,
            id=i.id,
        )),
    )

    # acknowledge http request
    http_response = sources.s3.response.pipe(
        ops.do_action(lambda i: print("[{}]s3 res: {}".format(
            datetime.datetime.now(), threading.get_ident()))),
        ops.do_action(
            lambda i: print("httpd res: {}".format(threading.get_ident()))),
        ops.map(lambda i: httpd.Response(
            data='ok'.encode('utf-8'),
            context=i.id,
        )),
    )

    # http server
    http_init = config.pipe(
        ops.flat_map(lambda i: rx.from_([
            httpd.Initialize(request_max_size=0),
            httpd.AddRoute(
                methods=['POST'],
                path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
                id='flac_transcode',
            ),
            httpd.StartServer(host=i.server.http.host, port=i.server.http.port
                              ),
        ])))
    http = rx.merge(http_init, http_response)

    # s3 database
    s3_init = config.pipe(
        ops.map(lambda i: s3.Configure(
            access_key=i.s3.access_key,
            secret_key=i.s3.secret_key,
            bucket=i.s3.bucket,
            endpoint_url=i.s3.endpoint_url,
            region_name=i.s3.region_name,
        )))

    # merge sink requests
    file_requests = read_request
    s3_requests = rx.merge(s3_init, store_requests)

    return Sink(
        encoder=encoder.Sink(request=encoder_request),
        s3=s3.Sink(request=s3_requests),
        file=file.Sink(request=file_requests),
        httpd=httpd.Sink(control=http),
    )
def audio_encoder(sources):
    # Parse configuration
    parser = create_arg_parser()

    parsed_argv = sources.argv.argv.pipe(
        ops.skip(1),
        argparse.parse(parser),
        ops.filter(lambda i: i.key == 'config'),
        ops.subscribe_on(aio_scheduler),
        ops.share(),
    )

    # monitor and parse config file
    monitor_init = parsed_argv.pipe(
        ops.flat_map(lambda i: rx.from_([
            inotify.AddWatch(
                id='config', path=i.value, flags=aionotify.Flags.MODIFY),
            inotify.Start(),
        ])))

    config_update = sources.inotify.response.pipe(
        ops.debounce(5.0, scheduler=aio_scheduler),
        ops.map(lambda i: True),
        ops.start_with(True),
    )

    read_request, read_response = rx.combine_latest(
        parsed_argv, config_update).pipe(
            ops.starmap(
                lambda config, _: file.Read(id='config', path=config.value)),
            file.read(sources.file.response),
        )

    config = read_response.pipe(
        ops.filter(lambda i: i.id == "config"),
        ops.flat_map(lambda i: i.data),
        parse_config,
    )

    # Transcode request handling
    encode_init = config.pipe(
        ops.map(lambda i: i.encode),
        ops.distinct_until_changed(),
        ops.map(lambda i: encoder.Configure(samplerate=i.samplerate,
                                            bitdepth=i.bitdepth)),
    )

    encode_request = sources.httpd.route.pipe(
        ops.filter(lambda i: i.id == 'flac_transcode'),
        ops.flat_map(lambda i: i.request),
        ops.flat_map(lambda i: rx.just(i, encode_scheduler)),
        ops.map(lambda i: encoder.EncodeMp3(
            id=i.context, data=i.data, key=i.match_info['key'])),
    )
    encoder_request = rx.merge(encode_init, encode_request)

    # store encoded file
    store_requests = sources.encoder.response.pipe(
        ops.observe_on(s3_scheduler),
        ops.map(lambda i: s3.UploadObject(
            key=i.key + '.flac',
            data=i.data,
            id=i.id,
        )),
    )

    # acknowledge http request
    http_response = sources.s3.response.pipe(
        ops.map(lambda i: httpd.Response(
            data='ok'.encode('utf-8'),
            context=i.id,
        )))

    # http server
    http_init = config.pipe(
        ops.take(1),
        ops.flat_map(lambda i: rx.from_([
            httpd.Initialize(request_max_size=0),
            httpd.AddRoute(
                methods=['POST'],
                path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
                id='flac_transcode',
            ),
            httpd.StartServer(host=i.server.http.host, port=i.server.http.port
                              ),
        ])),
    )
    http = rx.merge(http_init, http_response)

    # s3 database
    s3_init = config.pipe(
        ops.take(1),
        ops.map(lambda i: s3.Configure(
            access_key=i.s3.access_key,
            secret_key=i.s3.secret_key,
            bucket=i.s3.bucket,
            endpoint_url=i.s3.endpoint_url,
            region_name=i.s3.region_name,
        )),
    )

    # merge sink requests
    file_requests = read_request
    s3_requests = rx.merge(s3_init, store_requests)

    return Sink(
        encoder=encoder.Sink(request=encoder_request),
        s3=s3.Sink(request=s3_requests),
        file=file.Sink(request=file_requests),
        httpd=httpd.Sink(control=http),
        inotify=inotify.Sink(request=monitor_init),
    )