def audio_encoder(sources):
# Parse configuration
parser = create_arg_parser()
read_request, read_response = sources.argv.argv.pipe(
ops.skip(1),
argparse.parse(parser),
ops.filter(lambda i: i.key == 'config'),
ops.map(lambda i: file.Read(id='config', path=i.value)),
file.read(sources.file.response),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config,
)
# Transcode request handling
encode_init = config.pipe(
ops.map(
lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))
encode_request = sources.httpd.route.pipe(
ops.filter(lambda i: i.id == 'flac_transcode'),
ops.flat_map(lambda i: i.request),
ops.map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])),
)
encoder_request = rx.merge(encode_init, encode_request)
# http server
http_init = config.pipe(
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port
),
])))
http_response = sources.encoder.response.pipe(
ops.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
http = rx.merge(http_init, http_response)
# merge sink requests
file_requests = read_request
return Sink(
encoder=encoder.Sink(request=encoder_request),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
def test_get(self):
client_thread = None
response = None
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
sink = httpd.Sink(control=Subject())
def setup(sink):
sink.control.on_next(httpd.Initialize()),
sink.control.on_next(
httpd.AddRoute(methods=['GET'], path='/foo', id='foo'))
sink.control.on_next(httpd.StartServer(host='localhost',
port=8080)),
def do_get():
nonlocal response
req = urllib.request.urlopen('http://localhost:8080/foo')
response = req.read()
# todo rxpy threadpool
client_thread = threading.Thread(target=do_get)
def on_server_item(i):
if type(i) is httpd.ServerStarted:
nonlocal client_thread
client_thread.start()
elif type(i) == httpd.ServerStopped:
asyncio.get_event_loop().stop()
def on_route_item(i):
sink.control.on_next(httpd.Response(context=i.context,
data=b'foo'))
loop.call_soon(sink.control.on_next, httpd.StopServer())
loop.call_soon(setup, sink)
source = httpd.make_driver(loop).call(sink)
source.route.pipe(
ops.filter(lambda i: i.id == 'foo'),
ops.flat_map(lambda i: i.request),
).subscribe(on_route_item)
source.server \
.subscribe(on_server_item)
loop.run_forever()
loop.close()
client_thread.join()
self.assertEqual(b'foo', response)
def echo_server(source):
init = rx.from_([
httpd.Initialize(),
httpd.AddRoute(methods=['GET'], path='/echo/{what}', id='echo'),
httpd.StartServer(host='localhost', port=8080),
])
echo = source.httpd.route.pipe(
ops.flat_map(lambda i: i.request),
ops.map(lambda i: httpd.Response(
context=i.context, data=i.match_info['what'].encode('utf-8'))),
)
control = rx.merge(init, echo)
return EchoSink(httpd=httpd.Sink(control=control))
def test_add_route(self):
routes = [
httpd.AddRoute(methods=['GET'], path='/foo', id='foo'),
httpd.AddRoute(methods=['POST'], path='/bar', id='bar'),
httpd.AddRoute(methods=['PUT'], path='/biz', id='biz'),
]
actual_routes = []
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
sink = httpd.Sink(control=Subject())
def setup(sink):
sink.control.on_next(httpd.Initialize()),
sink.control.on_next(httpd.StartServer(host='localhost',
port=9999)),
for route in routes:
sink.control.on_next(route)
def on_route_item(i):
if type(i) is httpd.RouteAdded:
actual_routes.append(i)
# stop mainloop when last route is created
if i.id == routes[-1].id:
asyncio.get_event_loop().stop()
loop.call_soon(setup, sink)
source = httpd.make_driver(loop).call(sink)
source.route.subscribe(on_route_item)
loop.run_forever()
loop.close()
self.assertEqual(len(routes), len(actual_routes))
for index, route in enumerate(actual_routes):
self.assertEqual(routes[index].path, route.path)
self.assertEqual(routes[index].id, route.id)
self.assertIsInstance(route.request, rx.Observable)
def test_start_server(self):
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
sink = httpd.Sink(control=Subject())
def on_httpd_item(i):
if type(i) is httpd.ServerStarted:
sink.control.on_next(httpd.StopServer())
elif type(i) == httpd.ServerStopped:
asyncio.get_event_loop().stop()
def control_stream(sink):
sink.control.on_next(httpd.Initialize())
sink.control.on_next(httpd.StartServer(host='localhost',
port=9999))
loop.call_soon(control_stream, sink)
source = httpd.make_driver(loop).call(sink)
source.server.subscribe(on_httpd_item)
loop.run_forever()
loop.close()
def deepspeech_server(aio_scheduler, sources):
argv = sources.argv.argv
stt = sources.httpd.route
stt_response = sources.deepspeech.text
ds_logs = sources.deepspeech.log
http_ds_error, route_ds_error = make_error_router()
args = parse_arguments(argv)
read_request, read_response = args.pipe(
ops.map(lambda i: file.Read(id='config', path=i.value)),
file.read(sources.file.response),
)
read_request = read_request.pipe(
ops.subscribe_on(aio_scheduler),
)
config = parse_config(read_response)
logs_config = config.pipe(
ops.flat_map(lambda i: rx.from_(i.log.level, scheduler=ImmediateScheduler())),
ops.map(lambda i: logging.SetLevel(logger=i.logger, level=i.level)),
)
logs = rx.merge(logs_config, ds_logs)
ds_stt = stt.pipe(
ops.flat_map(lambda i: i.request),
ops.map(lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)),
)
# config is hot, the combine operator allows to keep its last value
# until logging is initialized
ds_arg = config.pipe(
ops.map(lambda i: deepspeech.Initialize(
model=i.deepspeech.model,
scorer=deepspeech.Scorer(
scorer=getattr(i.deepspeech, 'scorer', None),
lm_alpha=getattr(i.deepspeech, 'lm_alpha', None),
lm_beta=getattr(i.deepspeech, 'lm_beta', None),
),
beam_width=getattr(i.deepspeech, 'beam_width', None),
)),
)
ds = rx.merge(ds_stt, ds_arg)
http_init = config.pipe(
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=i.server.http.request_max_size),
httpd.AddRoute(
methods=['POST'],
path='/stt',
id='stt',
headers=MultiDict([('Content-Type', 'text/plain')]),
),
httpd.StartServer(
host=i.server.http.host,
port=i.server.http.port),
])),
)
http_response = stt_response.pipe(
route_ds_error(
error_map=lambda e: httpd.Response(
data="Speech to text error".encode('utf-8'),
context=e.args[0].context,
status=500
)),
ops.map(lambda i: httpd.Response(
data=i.text.encode('utf-8'),
context=i.context,
)),
)
http = rx.merge(http_init, http_response, http_ds_error)
return DeepspeechSink(
file=file.Sink(request=read_request),
logging=logging.Sink(request=logs),
deepspeech=deepspeech.Sink(speech=ds),
httpd=httpd.Sink(control=http)
)
def audio_encoder(sources):
# Parse configuration
parser = create_arg_parser()
read_request, read_response = sources.argv.argv.pipe(
ops.skip(1),
argparse.parse(parser),
ops.filter(lambda i: i.key == 'config'),
ops.map(lambda i: file.Read(id='config', path=i.value)),
file.read(sources.file.response),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config,
)
# Transcode request handling
encode_init = config.pipe(
ops.map(
lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))
encode_request = sources.httpd.route.pipe(
ops.filter(lambda i: i.id == 'flac_transcode'),
ops.flat_map(lambda i: i.request),
ops.do_action(lambda i: print("[{}]http req: {}".format(
datetime.datetime.now(), threading.get_ident()))),
#.observe_on(encode_scheduler)
ops.flat_map(lambda i: Observable.just(i, encode_scheduler)),
ops.do_action(lambda i: print("[{}]encode req: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])),
)
encoder_request = rx.merge(encode_init, encode_request)
# store encoded file
store_requests = sources.encoder.response.pipe(
ops.do_action(lambda i: print("[{}]encode res: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.observe_on(s3_scheduler),
ops.do_action(lambda i: print("[{}]s3 req: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
)),
)
# acknowledge http request
http_response = sources.s3.response.pipe(
ops.do_action(lambda i: print("[{}]s3 res: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.do_action(
lambda i: print("httpd res: {}".format(threading.get_ident()))),
ops.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)),
)
# http server
http_init = config.pipe(
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port
),
])))
http = rx.merge(http_init, http_response)
# s3 database
s3_init = config.pipe(
ops.map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
)))
# merge sink requests
file_requests = read_request
s3_requests = rx.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
def audio_encoder(sources):
# Parse configuration
parser = create_arg_parser()
parsed_argv = sources.argv.argv.pipe(
ops.skip(1),
argparse.parse(parser),
ops.filter(lambda i: i.key == 'config'),
ops.subscribe_on(aio_scheduler),
ops.share(),
)
# monitor and parse config file
monitor_init = parsed_argv.pipe(
ops.flat_map(lambda i: rx.from_([
inotify.AddWatch(
id='config', path=i.value, flags=aionotify.Flags.MODIFY),
inotify.Start(),
])))
config_update = sources.inotify.response.pipe(
ops.debounce(5.0, scheduler=aio_scheduler),
ops.map(lambda i: True),
ops.start_with(True),
)
read_request, read_response = rx.combine_latest(
parsed_argv, config_update).pipe(
ops.starmap(
lambda config, _: file.Read(id='config', path=config.value)),
file.read(sources.file.response),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config,
)
# Transcode request handling
encode_init = config.pipe(
ops.map(lambda i: i.encode),
ops.distinct_until_changed(),
ops.map(lambda i: encoder.Configure(samplerate=i.samplerate,
bitdepth=i.bitdepth)),
)
encode_request = sources.httpd.route.pipe(
ops.filter(lambda i: i.id == 'flac_transcode'),
ops.flat_map(lambda i: i.request),
ops.flat_map(lambda i: rx.just(i, encode_scheduler)),
ops.map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])),
)
encoder_request = rx.merge(encode_init, encode_request)
# store encoded file
store_requests = sources.encoder.response.pipe(
ops.observe_on(s3_scheduler),
ops.map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
)),
)
# acknowledge http request
http_response = sources.s3.response.pipe(
ops.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
# http server
http_init = config.pipe(
ops.take(1),
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port
),
])),
)
http = rx.merge(http_init, http_response)
# s3 database
s3_init = config.pipe(
ops.take(1),
ops.map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
)),
)
# merge sink requests
file_requests = read_request
s3_requests = rx.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
inotify=inotify.Sink(request=monitor_init),
)