def test_start_server(self):
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
sink = httpd.Sink(
control = Subject()
)
def on_httpd_item(i):
if type(i) is httpd.ServerStarted:
sink.control.on_next(httpd.StopServer())
elif type(i) == httpd.ServerStopped:
asyncio.get_event_loop().stop()
def control_stream(sink):
sink.control.on_next(httpd.Initialize())
sink.control.on_next(httpd.StartServer(host='localhost', port=9999))
loop.call_soon(control_stream, sink)
source = httpd.make_driver(loop).call(sink)
source.server.subscribe(on_httpd_item)
loop.run_forever()
loop.close()
def echo_server(source):
init = Observable.from_([
httpd.Initialize(),
httpd.AddRoute(methods=['GET'], path='/echo/{what}', id='echo'),
httpd.StartServer(host='localhost', port=8080),
])
echo = (source.httpd.route.filter(lambda i: i.id == 'echo').flat_map(
lambda i: i.request).map(lambda i: httpd.Response(
context=i.context, data=i.match_info['what'].encode('utf-8'))))
control = Observable.merge(init, echo)
return EchoSink(httpd=httpd.Sink(control=control))
def test_get(self):
client_thread = None
response = None
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
sink = httpd.Sink(control=Subject())
def setup(sink):
sink.control.on_next(httpd.Initialize()),
sink.control.on_next(httpd.AddRoute(
methods=['GET'], path='/foo', id='foo'))
sink.control.on_next(httpd.StartServer(host='localhost', port=8080)),
def do_get():
nonlocal response
req = urllib.request.urlopen('http://localhost:8080/foo')
response = req.read()
# todo rxpy threadpool
client_thread = threading.Thread(target=do_get)
def on_server_item(i):
if type(i) is httpd.ServerStarted:
nonlocal client_thread
client_thread.start()
elif type(i) == httpd.ServerStopped:
asyncio.get_event_loop().stop()
def on_route_item(i):
sink.control.on_next(httpd.Response(context=i.context, data=b'foo'))
loop.call_soon(sink.control.on_next, httpd.StopServer())
loop.call_soon(setup, sink)
source = httpd.make_driver(loop).call(sink)
source.route \
.filter(lambda i : i.id == 'foo') \
.flat_map(lambda i: i.request) \
.subscribe(on_route_item)
source.server \
.subscribe(on_server_item)
loop.run_forever()
loop.close()
client_thread.join()
self.assertEqual(b'foo', response)
def audio_encoder(sources):
# Parse configuration
read_config_file = (sources.argv.argv.skip(1).let(
argparse.argparse,
parser=Observable.just(
argparse.Parser(description="audio encode server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
])).filter(lambda i: i.key == 'config').map(
lambda i: file.Read(id='config', path=i.value)))
config = sources.file.response.let(parse_config)
# Transcode request handling
encode_init = (config.map(
lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))
encode_request = (
sources.httpd.route.filter(lambda i: i.id == 'flac_transcode').
flat_map(lambda i: i.request).map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])))
encoder_request = Observable.merge(encode_init, encode_request)
# http server
http_init = (config.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http_response = (sources.encoder.response.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
http = Observable.merge(http_init, http_response)
# merge sink requests
file_requests = read_config_file
return Sink(
encoder=encoder.Sink(request=encoder_request),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
def test_add_route(self):
routes = [
httpd.AddRoute(methods=['GET'], path='/foo', id='foo'),
httpd.AddRoute(methods=['POST'], path='/bar', id='bar'),
httpd.AddRoute(methods=['PUT'], path='/biz', id='biz'),
]
actual_routes = []
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
sink = httpd.Sink(control=Subject())
def setup(sink):
sink.control.on_next(httpd.Initialize()),
sink.control.on_next(httpd.StartServer(host='localhost', port=9999)),
for route in routes:
sink.control.on_next(route)
def on_route_item(i):
if type(i) is httpd.RouteAdded:
actual_routes.append(i)
# stop mainloop when last route is created
if i.id == routes[-1].id:
asyncio.get_event_loop().stop()
loop.call_soon(setup, sink)
source = httpd.make_driver(loop).call(sink)
source.route.subscribe(on_route_item)
loop.run_forever()
loop.close()
self.assertEqual(len(routes), len(actual_routes))
for index,route in enumerate(actual_routes):
self.assertEqual(routes[index].path, route.path)
self.assertEqual(routes[index].id, route.id)
self.assertIsInstance(route.request, Observable)
def deepspeech_server(sources):
argv = sources.argv.argv
stt = sources.httpd.route
stt_response = sources.deepspeech.text.share()
ds_logs = sources.deepspeech.log
config_data = sources.file.response
http_ds_error, route_ds_error = make_error_router()
args = argparse.argparse(
argv=argv.skip(1).subscribe_on(aio_scheduler),
parser=Observable.just(
argparse.Parser(description="deepspeech server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
]))
config_file = (args.filter(lambda i: i.key == 'config').map(
lambda i: file.Read(id='config', path=i.value)))
config = parse_config(config_data).subscribe_on(aio_scheduler)
logs_config = (config.flat_map(lambda i: Observable.from_(i.log.level).map(
lambda i: logging.SetLevel(logger=i.logger, level=i.level)).concat(
Observable.just(logging.SetLevelDone()))))
logs = Observable.merge(logs_config, ds_logs)
log_ready = sources.logging.response.take(1)
ds_stt = (stt.flat_map(lambda i: i.request).map(
lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)))
ds_arg = (
# config is hot, the combine operator allows to keep its last value
# until logging is initialized
log_ready.combine_latest(
config, lambda _, i: i).map(lambda i: deepspeech.Initialize(
model=i.deepspeech.model,
alphabet=i.deepspeech.alphabet,
lm=i.deepspeech.lm,
trie=i.deepspeech.trie,
features=deepspeech.FeaturesParameters(
n_features=i.deepspeech.features.n_features,
n_context=i.deepspeech.features.n_context,
beam_width=i.deepspeech.features.beam_width,
lm_alpha=i.deepspeech.features.lm_alpha,
lm_beta=i.deepspeech.features.lm_beta,
) if i.deepspeech.features is not None else None)))
ds = ds_stt.merge(ds_arg)
http_init = (config.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=i.server.http.request_max_size),
httpd.AddRoute(
methods=['POST'],
path='/stt',
id='stt',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http_response = (stt_response.let(lambda x: route_ds_error(
x,
error_map=lambda e: httpd.Response(
data="Speech to text error".encode('utf-8'),
context=e.args[0].context,
status=500))).map(lambda i: httpd.Response(
data=i.text.encode('utf-8'),
context=i.context,
)))
http = Observable.merge(http_init, http_response, http_ds_error)
return DeepspeechSink(file=file.Sink(request=config_file),
logging=logging.Sink(request=logs),
deepspeech=deepspeech.Sink(speech=ds),
httpd=httpd.Sink(control=http))
def audio_encoder(sources):
http_s3_error, route_s3_error = make_error_router()
http_encode_error, route_encode_error = make_error_router()
# Parse configuration
parsed_argv = (sources.argv.argv.skip(1).let(
argparse.argparse,
parser=Observable.just(
argparse.Parser(description="audio encode server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
])).filter(lambda i: i.key == 'config').subscribe_on(
aio_scheduler).share())
# monitor and parse config file
monitor_init = (parsed_argv.flat_map(lambda i: Observable.from_([
inotify.AddWatch(
id='config', path=i.value, flags=aionotify.Flags.MODIFY),
inotify.Start(),
])))
config_update = (sources.inotify.response.debounce(5000).map(
lambda i: True).start_with(True))
read_config_file = (Observable.combine_latest(
parsed_argv, config_update,
lambda config, _: file.Read(id='config', path=config.value)))
config = sources.file.response.let(parse_config)
# Transcode request handling
encode_init = (config.map(lambda i: i.encode).distinct_until_changed().map(
lambda i: encoder.Configure(samplerate=i.samplerate,
bitdepth=i.bitdepth)))
encode_request = (
sources.httpd.route.filter(lambda i: i.id == 'flac_transcode').
flat_map(lambda i: i.request).flat_map(lambda i: Observable.just(
i, encode_scheduler)).map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])))
encoder_request = Observable.merge(encode_init, encode_request)
# store encoded file
store_requests = (sources.encoder.response.let(
catch_or_flat_map,
error_router=route_encode_error,
error_map=lambda i: httpd.Response(data='encode error'.encode('utf-8'),
context=i.args[0].id,
status=500)).
observe_on(s3_scheduler).map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
)))
# acknowledge http request
http_response = (sources.s3.response.let(
catch_or_flat_map,
error_router=route_s3_error,
error_map=lambda i: httpd.Response(data='upload error'.encode('utf-8'),
context=i.args[0].id,
status=500)).map(
lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
# http server
http_init = (config.take(1).flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http = Observable.merge(http_init, http_response, http_s3_error,
http_encode_error)
# s3 database
s3_init = (config.take(1).map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
)))
# merge sink requests
file_requests = read_config_file
s3_requests = Observable.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
inotify=inotify.Sink(request=monitor_init),
)
def audio_encoder(sources):
# Parse configuration
read_config_file = (
sources.argv.argv.skip(1)
.let(argparse.argparse,
parser=Observable.just(
argparse.Parser(description="audio encode server")),
arguments=Observable.from_([
argparse.ArgumentDef(
name='--config', help="Path of the server configuration file")
]))
.filter(lambda i: i.key == 'config')
.map(lambda i: file.Read(id='config', path=i.value))
)
config = sources.file.response.let(parse_config)
# Transcode request handling
encode_init = (
config
.map(lambda i: encoder.Initialize(storage_path=i.encode.storage_path))
)
encode_request = (
sources.httpd.route
.filter(lambda i: i.id == 'flac_transcode')
.flat_map(lambda i: i.request)
.do_action(lambda i: print("[{}]http req: {}".format(datetime.datetime.now(), threading.get_ident())))
#.observe_on(encode_scheduler)
.flat_map(lambda i: Observable.just(i, encode_scheduler))
.do_action(lambda i: print("[{}]encode req: {}".format(datetime.datetime.now(), threading.get_ident())))
.map(lambda i: encoder.EncodeMp3(
id=i.context,
data=i.data,
key=i.match_info['key']))
)
encoder_request = Observable.merge(encode_init, encode_request)
# store encoded file
store_requests = (
sources.encoder.response
.do_action(lambda i: print("[{}]encode res: {}".format(datetime.datetime.now(), threading.get_ident())))
.observe_on(s3_scheduler)
.do_action(lambda i: print("[{}]s3 req: {}".format(datetime.datetime.now(), threading.get_ident())))
.map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
))
)
# acknowledge http request
http_response = (
sources.s3.response
.do_action(lambda i: print("[{}]s3 res: {}".format(datetime.datetime.now(), threading.get_ident())))
.do_action(lambda i: print("httpd res: {}".format(threading.get_ident())))
.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
))
)
# http server
http_init = (
config
.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(
host=i.server.http.host,
port=i.server.http.port),
]))
)
http = Observable.merge(http_init, http_response)
# s3 database
s3_init = (
config
.map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
))
)
# merge sink requests
file_requests = read_config_file
s3_requests = Observable.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)