def test_read(self):
driver_response = Subject()
file_adapter = file.adapter(driver_response)
file_adapter.sink.subscribe(
on_next=functools.partial(self.on_next, 'driver_request'),
on_error=functools.partial(self.on_error, 'driver_request'),
on_completed=functools.partial(self.on_completed, 'driver_request'))
response = file_adapter.api.read('/foo.txt')
response.subscribe(
on_next=functools.partial(self.on_next, 'response'),
on_error=functools.partial(self.on_error, 'response'),
on_completed=functools.partial(self.on_completed, 'response'))
self.assertEqual(
file.Read(id=response,
path='/foo.txt',
size=-1, mode='r',
),
self.actual['driver_request']['next'][0])
result = file.ReadResponse(id=response, path='a', data=Observable.just(b'bar'))
driver_response.on_next(result)
self.assertIs(
b'bar',
self.actual['response']['next'][0])
def audio_encoder(sources):
# Parse configuration
parser = create_arg_parser()
read_request, read_response = sources.argv.argv.pipe(
ops.skip(1),
argparse.parse(parser),
ops.filter(lambda i: i.key == 'config'),
ops.map(lambda i: file.Read(id='config', path=i.value)),
file.read(sources.file.response),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config,
)
# Transcode request handling
encode_init = config.pipe(
ops.map(
lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))
encode_request = sources.httpd.route.pipe(
ops.filter(lambda i: i.id == 'flac_transcode'),
ops.flat_map(lambda i: i.request),
ops.map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])),
)
encoder_request = rx.merge(encode_init, encode_request)
# http server
http_init = config.pipe(
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port
),
])))
http_response = sources.encoder.response.pipe(
ops.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
http = rx.merge(http_init, http_response)
# merge sink requests
file_requests = read_request
return Sink(
encoder=encoder.Sink(request=encoder_request),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
def test_bad_json(self):
# test expectations
expected_file_request = file.Read(id='config',
path='/foo/config.json',
size=-1,
mode='r')
# test setup
file_response = Subject()
argv = Subject()
sink = config.read_configuration(
config.Source(file_response=file_response, argv=argv))
configuration = None
configuration_error = None
file_request = None
def config_on_next(i):
nonlocal configuration
configuration = i
def config_on_error(e):
nonlocal configuration_error
configuration_error = e
def file_request_on_next(i):
nonlocal file_request
file_request = i
sink.configuration.subscribe(on_next=config_on_next,
on_error=config_on_error)
sink.file_request.subscribe(on_next=file_request_on_next)
# feed source observables
argv.on_next('theexe')
argv.on_next('--config')
argv.on_next('/foo/config.json')
argv.on_completed()
self.assertEqual(expected_file_request, file_request)
file_response.on_next(
file.ReadResponse(id='config',
path='/foo/config.json',
data=Observable.just('{ "foo": bar}')))
self.assertIsInstance(configuration_error,
json.decoder.JSONDecodeError)
def audio_encoder(sources):
# Parse configuration
read_config_file = (sources.argv.argv.skip(1).let(
argparse.argparse,
parser=Observable.just(
argparse.Parser(description="audio encode server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
])).filter(lambda i: i.key == 'config').map(
lambda i: file.Read(id='config', path=i.value)))
config = sources.file.response.let(parse_config)
# Transcode request handling
encode_init = (config.map(
lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))
encode_request = (
sources.httpd.route.filter(lambda i: i.id == 'flac_transcode').
flat_map(lambda i: i.request).map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])))
encoder_request = Observable.merge(encode_init, encode_request)
# http server
http_init = (config.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http_response = (sources.encoder.response.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
http = Observable.merge(http_init, http_response)
# merge sink requests
file_requests = read_config_file
return Sink(
encoder=encoder.Sink(request=encoder_request),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
def read_config_from_file(filename, file_response, scheduler=None):
read_request, read_response = filename.pipe(
ops.map(lambda i: file.Read(id='config', path=i)),
file.read(file_response),
)
if scheduler is not None:
read_request = read_request.pipe(
ops.subscribe_on(scheduler),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config(),
)
return config, read_request
def read_configuration(source):
args = argparse.argparse(
argv=source.argv.skip(1),
parser=Observable.just(argparse.Parser(description="deepspeaker")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the configuration file")
]))
read_config_file = Observable.just(
file.Context(id='config',
observable=args.filter(lambda i: i.key == 'config').map(
lambda i: file.Read(id='config', path=i.value))))
config = (source.file_response.filter(lambda i: i.id == "config").flat_map(
lambda i: i.observable
).flat_map(lambda i: i.data).map(lambda i: json.loads(
i, object_hook=lambda d: namedtuple('x', d.keys())(*d.values()))))
return Sink(file_request=read_config_file, configuration=config)
def test_base(self):
# test expectations
expected_file_request = file.Read(id='config',
path='/foo/config.json',
size=-1,
mode='r')
# test setup
file_response = Subject()
argv = Subject()
sink = config.read_configuration(
config.Source(file_response=file_response, argv=argv))
configuration = None
file_request = None
def config_on_next(i):
nonlocal configuration
configuration = i
def file_request_on_next(i):
nonlocal file_request
file_request = i
sink.configuration.subscribe(on_next=config_on_next)
sink.file_request.subscribe(on_next=file_request_on_next)
# feed source observables
argv.on_next('theexe')
argv.on_next('--config')
argv.on_next('/foo/config.json')
argv.on_completed()
self.assertEqual(expected_file_request, file_request)
file_response.on_next(
file.ReadResponse(id='config',
path='/foo/config.json',
data=Observable.just('{ "foo": "bar"}')))
self.assertEqual(dict_to_namedtuple({"foo": "bar"}), configuration)
def deepspeech_server(sources):
argv = sources.argv.argv
stt = sources.httpd.route
stt_response = sources.deepspeech.text.share()
ds_logs = sources.deepspeech.log
config_data = sources.file.response
http_ds_error, route_ds_error = make_error_router()
args = argparse.argparse(
argv=argv.skip(1).subscribe_on(aio_scheduler),
parser=Observable.just(
argparse.Parser(description="deepspeech server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
]))
config_file = (args.filter(lambda i: i.key == 'config').map(
lambda i: file.Read(id='config', path=i.value)))
config = parse_config(config_data).subscribe_on(aio_scheduler)
logs_config = (config.flat_map(lambda i: Observable.from_(i.log.level).map(
lambda i: logging.SetLevel(logger=i.logger, level=i.level)).concat(
Observable.just(logging.SetLevelDone()))))
logs = Observable.merge(logs_config, ds_logs)
log_ready = sources.logging.response.take(1)
ds_stt = (stt.flat_map(lambda i: i.request).map(
lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)))
ds_arg = (
# config is hot, the combine operator allows to keep its last value
# until logging is initialized
log_ready.combine_latest(
config, lambda _, i: i).map(lambda i: deepspeech.Initialize(
model=i.deepspeech.model,
alphabet=i.deepspeech.alphabet,
lm=i.deepspeech.lm,
trie=i.deepspeech.trie,
features=deepspeech.FeaturesParameters(
n_features=i.deepspeech.features.n_features,
n_context=i.deepspeech.features.n_context,
beam_width=i.deepspeech.features.beam_width,
lm_alpha=i.deepspeech.features.lm_alpha,
lm_beta=i.deepspeech.features.lm_beta,
) if i.deepspeech.features is not None else None)))
ds = ds_stt.merge(ds_arg)
http_init = (config.flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=i.server.http.request_max_size),
httpd.AddRoute(
methods=['POST'],
path='/stt',
id='stt',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http_response = (stt_response.let(lambda x: route_ds_error(
x,
error_map=lambda e: httpd.Response(
data="Speech to text error".encode('utf-8'),
context=e.args[0].context,
status=500))).map(lambda i: httpd.Response(
data=i.text.encode('utf-8'),
context=i.context,
)))
http = Observable.merge(http_init, http_response, http_ds_error)
return DeepspeechSink(file=file.Sink(request=config_file),
logging=logging.Sink(request=logs),
deepspeech=deepspeech.Sink(speech=ds),
httpd=httpd.Sink(control=http))
def deepspeech_server(aio_scheduler, sources):
argv = sources.argv.argv
stt = sources.httpd.route
stt_response = sources.deepspeech.text
ds_logs = sources.deepspeech.log
http_ds_error, route_ds_error = make_error_router()
args = parse_arguments(argv)
read_request, read_response = args.pipe(
ops.map(lambda i: file.Read(id='config', path=i.value)),
file.read(sources.file.response),
)
read_request = read_request.pipe(
ops.subscribe_on(aio_scheduler),
)
config = parse_config(read_response)
logs_config = config.pipe(
ops.flat_map(lambda i: rx.from_(i.log.level, scheduler=ImmediateScheduler())),
ops.map(lambda i: logging.SetLevel(logger=i.logger, level=i.level)),
)
logs = rx.merge(logs_config, ds_logs)
ds_stt = stt.pipe(
ops.flat_map(lambda i: i.request),
ops.map(lambda i: deepspeech.SpeechToText(data=i.data, context=i.context)),
)
# config is hot, the combine operator allows to keep its last value
# until logging is initialized
ds_arg = config.pipe(
ops.map(lambda i: deepspeech.Initialize(
model=i.deepspeech.model,
scorer=deepspeech.Scorer(
scorer=getattr(i.deepspeech, 'scorer', None),
lm_alpha=getattr(i.deepspeech, 'lm_alpha', None),
lm_beta=getattr(i.deepspeech, 'lm_beta', None),
),
beam_width=getattr(i.deepspeech, 'beam_width', None),
)),
)
ds = rx.merge(ds_stt, ds_arg)
http_init = config.pipe(
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=i.server.http.request_max_size),
httpd.AddRoute(
methods=['POST'],
path='/stt',
id='stt',
headers=MultiDict([('Content-Type', 'text/plain')]),
),
httpd.StartServer(
host=i.server.http.host,
port=i.server.http.port),
])),
)
http_response = stt_response.pipe(
route_ds_error(
error_map=lambda e: httpd.Response(
data="Speech to text error".encode('utf-8'),
context=e.args[0].context,
status=500
)),
ops.map(lambda i: httpd.Response(
data=i.text.encode('utf-8'),
context=i.context,
)),
)
http = rx.merge(http_init, http_response, http_ds_error)
return DeepspeechSink(
file=file.Sink(request=read_request),
logging=logging.Sink(request=logs),
deepspeech=deepspeech.Sink(speech=ds),
httpd=httpd.Sink(control=http)
)
def audio_encoder(sources):
# Parse configuration
parser = create_arg_parser()
read_request, read_response = sources.argv.argv.pipe(
ops.skip(1),
argparse.parse(parser),
ops.filter(lambda i: i.key == 'config'),
ops.map(lambda i: file.Read(id='config', path=i.value)),
file.read(sources.file.response),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config,
)
# Transcode request handling
encode_init = config.pipe(
ops.map(
lambda i: encoder.Initialize(storage_path=i.encode.storage_path)))
encode_request = sources.httpd.route.pipe(
ops.filter(lambda i: i.id == 'flac_transcode'),
ops.flat_map(lambda i: i.request),
ops.do_action(lambda i: print("[{}]http req: {}".format(
datetime.datetime.now(), threading.get_ident()))),
#.observe_on(encode_scheduler)
ops.flat_map(lambda i: Observable.just(i, encode_scheduler)),
ops.do_action(lambda i: print("[{}]encode req: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])),
)
encoder_request = rx.merge(encode_init, encode_request)
# store encoded file
store_requests = sources.encoder.response.pipe(
ops.do_action(lambda i: print("[{}]encode res: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.observe_on(s3_scheduler),
ops.do_action(lambda i: print("[{}]s3 req: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
)),
)
# acknowledge http request
http_response = sources.s3.response.pipe(
ops.do_action(lambda i: print("[{}]s3 res: {}".format(
datetime.datetime.now(), threading.get_ident()))),
ops.do_action(
lambda i: print("httpd res: {}".format(threading.get_ident()))),
ops.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)),
)
# http server
http_init = config.pipe(
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port
),
])))
http = rx.merge(http_init, http_response)
# s3 database
s3_init = config.pipe(
ops.map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
)))
# merge sink requests
file_requests = read_request
s3_requests = rx.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
)
def audio_encoder(sources):
http_s3_error, route_s3_error = make_error_router()
http_encode_error, route_encode_error = make_error_router()
# Parse configuration
parsed_argv = (sources.argv.argv.skip(1).let(
argparse.argparse,
parser=Observable.just(
argparse.Parser(description="audio encode server")),
arguments=Observable.from_([
argparse.ArgumentDef(name='--config',
help="Path of the server configuration file")
])).filter(lambda i: i.key == 'config').subscribe_on(
aio_scheduler).share())
# monitor and parse config file
monitor_init = (parsed_argv.flat_map(lambda i: Observable.from_([
inotify.AddWatch(
id='config', path=i.value, flags=aionotify.Flags.MODIFY),
inotify.Start(),
])))
config_update = (sources.inotify.response.debounce(5000).map(
lambda i: True).start_with(True))
read_config_file = (Observable.combine_latest(
parsed_argv, config_update,
lambda config, _: file.Read(id='config', path=config.value)))
config = sources.file.response.let(parse_config)
# Transcode request handling
encode_init = (config.map(lambda i: i.encode).distinct_until_changed().map(
lambda i: encoder.Configure(samplerate=i.samplerate,
bitdepth=i.bitdepth)))
encode_request = (
sources.httpd.route.filter(lambda i: i.id == 'flac_transcode').
flat_map(lambda i: i.request).flat_map(lambda i: Observable.just(
i, encode_scheduler)).map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])))
encoder_request = Observable.merge(encode_init, encode_request)
# store encoded file
store_requests = (sources.encoder.response.let(
catch_or_flat_map,
error_router=route_encode_error,
error_map=lambda i: httpd.Response(data='encode error'.encode('utf-8'),
context=i.args[0].id,
status=500)).
observe_on(s3_scheduler).map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
)))
# acknowledge http request
http_response = (sources.s3.response.let(
catch_or_flat_map,
error_router=route_s3_error,
error_map=lambda i: httpd.Response(data='upload error'.encode('utf-8'),
context=i.args[0].id,
status=500)).map(
lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
# http server
http_init = (config.take(1).flat_map(lambda i: Observable.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port),
])))
http = Observable.merge(http_init, http_response, http_s3_error,
http_encode_error)
# s3 database
s3_init = (config.take(1).map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
)))
# merge sink requests
file_requests = read_config_file
s3_requests = Observable.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
inotify=inotify.Sink(request=monitor_init),
)
def audio_encoder(sources):
# Parse configuration
parser = create_arg_parser()
parsed_argv = sources.argv.argv.pipe(
ops.skip(1),
argparse.parse(parser),
ops.filter(lambda i: i.key == 'config'),
ops.subscribe_on(aio_scheduler),
ops.share(),
)
# monitor and parse config file
monitor_init = parsed_argv.pipe(
ops.flat_map(lambda i: rx.from_([
inotify.AddWatch(
id='config', path=i.value, flags=aionotify.Flags.MODIFY),
inotify.Start(),
])))
config_update = sources.inotify.response.pipe(
ops.debounce(5.0, scheduler=aio_scheduler),
ops.map(lambda i: True),
ops.start_with(True),
)
read_request, read_response = rx.combine_latest(
parsed_argv, config_update).pipe(
ops.starmap(
lambda config, _: file.Read(id='config', path=config.value)),
file.read(sources.file.response),
)
config = read_response.pipe(
ops.filter(lambda i: i.id == "config"),
ops.flat_map(lambda i: i.data),
parse_config,
)
# Transcode request handling
encode_init = config.pipe(
ops.map(lambda i: i.encode),
ops.distinct_until_changed(),
ops.map(lambda i: encoder.Configure(samplerate=i.samplerate,
bitdepth=i.bitdepth)),
)
encode_request = sources.httpd.route.pipe(
ops.filter(lambda i: i.id == 'flac_transcode'),
ops.flat_map(lambda i: i.request),
ops.flat_map(lambda i: rx.just(i, encode_scheduler)),
ops.map(lambda i: encoder.EncodeMp3(
id=i.context, data=i.data, key=i.match_info['key'])),
)
encoder_request = rx.merge(encode_init, encode_request)
# store encoded file
store_requests = sources.encoder.response.pipe(
ops.observe_on(s3_scheduler),
ops.map(lambda i: s3.UploadObject(
key=i.key + '.flac',
data=i.data,
id=i.id,
)),
)
# acknowledge http request
http_response = sources.s3.response.pipe(
ops.map(lambda i: httpd.Response(
data='ok'.encode('utf-8'),
context=i.id,
)))
# http server
http_init = config.pipe(
ops.take(1),
ops.flat_map(lambda i: rx.from_([
httpd.Initialize(request_max_size=0),
httpd.AddRoute(
methods=['POST'],
path='/api/transcode/v1/flac/{key:[a-zA-Z0-9-\._]*}',
id='flac_transcode',
),
httpd.StartServer(host=i.server.http.host, port=i.server.http.port
),
])),
)
http = rx.merge(http_init, http_response)
# s3 database
s3_init = config.pipe(
ops.take(1),
ops.map(lambda i: s3.Configure(
access_key=i.s3.access_key,
secret_key=i.s3.secret_key,
bucket=i.s3.bucket,
endpoint_url=i.s3.endpoint_url,
region_name=i.s3.region_name,
)),
)
# merge sink requests
file_requests = read_request
s3_requests = rx.merge(s3_init, store_requests)
return Sink(
encoder=encoder.Sink(request=encoder_request),
s3=s3.Sink(request=s3_requests),
file=file.Sink(request=file_requests),
httpd=httpd.Sink(control=http),
inotify=inotify.Sink(request=monitor_init),
)
def process(path, configuration):
''' compute features for all files in path
This lettable operator processes all files present in the path observable.
It reads each file, process them, and writes the result. The processing
is multithreaded via a thread pool. The resulting observable is still
scheduled in the asyncio event loop.
'''
aio_ts_scheduler = AsyncIOScheduler(threadsafe=True)
aio_scheduler = AsyncIOScheduler()
thread_scheduler = ThreadPoolScheduler(
max_workers=configuration.data_preparation.cpu_core_count)
# prepare file routing : read media and write features
media_file_response = file_response
feature_write_request, write_feature_file = router.make_crossroad_router(
feature_response)
media_read_request, read_media_file = router.make_crossroad_router(
media_file_response)
media_read_request \
.flat_map(lambda i: Observable.just(i, scheduler=thread_scheduler)) \
.subscribe(sink_request)
feature_write_request.subscribe(feature_request)
# feature engineering
return (
path
#.do_action(TraceObserver(prefix='write1', trace_next_payload=False))
.flat_map(
lambda i: i.files
.map(lambda path: file.Read(
id='read_media',
path=path,
mode='rb',
))
.let(read_media_file)
.filter(lambda i: i.id == 'read_media')
.flat_map(lambda media: media.data
#.do_action(lambda i: print('write20-{}'.format(threading.get_ident()))) \
#.do_action(TraceObserver(prefix='write20', trace_next_payload=False))
.let(process_audio, configuration=configuration.features)
.map(lambda i: data_to_feature(i, media.path, configuration))
#.do_action(TraceObserver(prefix='write21-{}'.format(threading.get_ident()), trace_next_payload=False))
# .do_action(lambda i: print(i.path))
)
#)
#.do_action(lambda i: print('write2-{}'.format(threading.get_ident()))) \
#.do_action(TraceObserver(prefix='write2', trace_next_payload=False))
)
# write feature file to disk
.map(lambda i: file.Write(
id='write_feature',
path=i.path, data=i.data,
mode='wb', mkdirs=True))
.let(write_feature_file)
.filter(lambda i: i.id == 'write_feature')
.map(lambda i: i.path)
#.do_action(lambda i: print('write2-2-{}'.format(threading.get_ident()))) \
#.do_action(TraceObserver(prefix='write2-2', trace_next_payload=False))
.observe_on(aio_ts_scheduler)
#.map(lambda i: "/foo/bar/i/4.mp3")
#.do_action(lambda i: print('write3-{}'.format(threading.get_ident())))
#.do_action(TraceObserver(prefix='write3', trace_next_payload=False))
)
