def test_run_1snk(self):
''' Creates a cycle with one sink driver.
'''
MainDrivers = namedtuple('MainDrivers', ['drv1'])
MainSource = namedtuple('MainSource', [])
MainSink = namedtuple('MainSink', ['drv1'])
test_values = []
def drv1(sink):
sink.values.subscribe(lambda i: test_values.append(i))
return None
Drv1Sink = namedtuple('Drv1Sink', ['values'])
Drv1Driver = Component(call=drv1, input=Drv1Sink)
def main(sources):
val = Observable.from_([1, 2, 3])
return MainSink(drv1=Drv1Sink(values=val))
drivers = MainDrivers(drv1=Drv1Driver)
dispose = run(Component(call=main, input=MainSource), drivers)
dispose()
self.assertEqual(3, len(test_values))
self.assertEqual(1, test_values[0])
self.assertEqual(2, test_values[1])
self.assertEqual(3, test_values[2])
def test_constructor(self):
def test_main():
pass
component = Component(call=test_main, input=int)
self.assertEqual(test_main, component.call)
self.assertEqual(int, component.input)
self.assertEqual(None, component.output)
component = Component(call=test_main, output=float)
self.assertEqual(test_main, component.call)
self.assertEqual(None, component.input)
self.assertEqual(float, component.output)
self.assertRaises(TypeError, Component)
def make_driver():
def encoder(sink):
def on_subscribe(observer):
samplerate = None
bitdepth = None
def on_next(item):
nonlocal samplerate
nonlocal bitdepth
if type(item) is Configure:
print("configure: {}".format(item))
samplerate = item.samplerate
bitdepth = item.bitdepth
elif type(item) is EncodeMp3:
encoded_data = mp3_to_flac(item.data, samplerate, bitdepth)
observer.on_next(
EncodeResult(id=item.id,
key=item.key,
data=encoded_data))
else:
observer.on_error("unknown item: {}".format(type(item)))
sink.request.subscribe(
on_next=on_next,
on_error=lambda e: observer.on_error(e),
on_completed=lambda: observer.on_completed(),
)
return Source(response=Observable.create(on_subscribe))
return Component(call=encoder, input=Sink)
def make_driver(loop=None):
loop = asyncio.get_event_loop() if loop is None else loop
def driver(sink):
def on_subscribe(observer, scheduler):
watcher = aionotify.Watcher()
async def read_events():
nonlocal observer
await watcher.setup(loop)
while True:
event = await watcher.get_event()
loop.call_soon(observer.on_next,
Event(id=event.alias, path=event.name))
watcher.close()
def on_next(item):
if type(item) is AddWatch:
watcher.watch(alias=item.id,
path=item.path,
flags=item.flags)
elif type(item) is Start:
asyncio.ensure_future(read_events())
else:
observer.on_error("unknown item: {}".format(type(item)))
sink.request.subscribe(on_next=on_next,
on_error=lambda e: observer.on_error(e))
return Source(response=rx.create(on_subscribe))
return Component(call=driver, input=Sink)
def make_driver():
def driver(sink):
def on_subscribe(observer):
def on_next(i):
if type(i) is SetSeed:
random.seed(i.value)
elif type(i) is Shuffle:
data = list(i.data)
random.shuffle(data)
observer.on_next(ShuffleResponse(
id=i.id,
data=data
))
else:
observer.on_error("unknown item: {}".format(i))
sink.request.subscribe(
on_next=on_next,
on_error=observer.on_error,
on_completed=observer.on_completed,
)
return Source(
response=Observable.create(on_subscribe)
)
return Component(call=driver, input=Sink)
def make_driver(loop=None):
def driver(sink):
'''
Routes must be configured before starting the server.
'''
session = None
def on_response_subscribe(observer, scheduler):
async def _request(request):
nonlocal session
if session is None:
session = aiohttp.ClientSession()
try:
response = await session.request(
request.method,
request.url,
params=request.params,
data=request.data,
headers=request.headers,
allow_redirects=request.allow_redirects,
max_redirects=request.max_redirects)
data = await response.read()
observer.on_next(
Response(id=request.id,
response=rx.just(
HttpResponse(
status=response.status,
reason=response.status,
method=response.method,
url=response.url,
data=data,
cookies=response.cookies,
headers=response.headers,
content_type=response.content_type))))
except Exception as e:
#print("exception: {}, {}".format(e, traceback.format_exc()))
observer.on_next(
Response(id=request.id, response=rx.throw(e)))
pass
def on_request_item(i):
if type(i) is Request:
asyncio.ensure_future(_request(i), loop=loop)
else:
print("received unknown item: {}".format(type(i)))
def on_request_error(e):
print("http sink error: {}, {}".format(e,
traceback.format_exc()))
return sink.request.subscribe(on_next=on_request_item,
on_error=on_request_error)
return Source(response=rx.create(on_response_subscribe))
return Component(call=driver, input=Sink)
def make_driver():
handlers = {}
observer = None
def driver(sink):
def on_subscribe(o):
nonlocal observer
observer = o
def on_request_item(i):
nonlocal handlers
nonlocal observer
if type(i) is Log:
logging.getLogger(i.logger).log(i.level, i.message)
elif type(i) is SetLevel:
level = level_from_string(i.level)
logger = logging.getLogger(i.logger)
logger.setLevel(level)
if i.logger in handlers:
logger.removeHandler(handlers[i.logger])
handlers[i.logger] = logging.StreamHandler()
handlers[i.logger].setLevel(level)
logger.addHandler(handlers[i.logger])
elif type(i) is SetLevelDone:
if observer is not None:
observer.on_next(i)
else:
if observer is not None:
observer.on_error("invalid item: {}".format(i))
sink.request.subscribe(on_request_item)
return Source(response=Observable.create(on_subscribe))
return Component(call=driver, input=Sink)
def make_driver(loop=None):
''' Returns a stop driver.
The optional loop argument can be provided to use the driver in another
loop than the default one.
Parameters
-----------
loop: BaseEventLoop
The event loop to use instead of the default one.
'''
loop = loop or asyncio.get_event_loop()
def stop(i = None):
loop.stop()
def driver(sink):
''' The stop driver stops the asyncio event loop.
The event loop is stopped as soon as an event is received on the
control observable or when it completes (both in case of success or
error).
Parameters
----------
sink: Sink
'''
sink.control.subscribe(
on_next=stop,
on_error=stop,
on_completed=stop)
return None
return Component(call=driver, input=Sink)
def make_driver():
def encoder(sink):
def on_subscribe(observer, scheduler):
storage_path = None
def on_next(item):
if type(item) is Initialize:
nonlocal storage_path
storage_path = item.storage_path
elif type(item) is EncodeMp3:
encoded_file = mp3_to_flac(item.data,
storage_path,
name=item.key)
observer.on_next(
EncodeResult(id=item.id, file=encoded_file))
else:
observer.on_error("unknown item: {}".format(type(item)))
sink.request.subscribe(
on_next=on_next,
on_error=lambda e: observer.on_error(e),
on_completed=lambda: observer.on_completed(),
)
return Source(response=rx.create(on_subscribe))
return Component(call=encoder, input=Sink)
def main():
dispose = run(entry_point=Component(call=rmux_client, input=Source),
drivers=Drivers(
tcp_client=tcp_client.make_driver(),
stdout=stdout.make_driver(),
))
dispose()
def make_driver():
def driver(sink):
def on_subscribe(observer, scheduler):
client = None
bucket = None
def on_next(item):
nonlocal client
nonlocal bucket
if type(item) is Configure:
session = Session(aws_access_key_id=item.access_key,
aws_secret_access_key=item.secret_key)
client = session.client('s3',
endpoint_url=item.endpoint_url,
region_name=item.region_name)
bucket = item.bucket
elif type(item) is UploadObject:
data = BytesIO(item.data)
client.upload_fileobj(data, bucket, item.key)
observer.on_next(UploadReponse(key=item.key, id=item.id))
else:
observer.on_error("unknown item: {}".format(type(item)))
sink.request.subscribe(
on_next=on_next,
on_error=lambda e: observer.on_error(e),
on_completed=lambda: observer.on_completed())
return Source(response=rx.create(on_subscribe))
return Component(call=driver, input=Sink)
def main():
run(
Component(call=deepspeech_server, input=DeepspeechSource),
DeepspeechDrivers(deepspeech=deepspeech.make_driver(),
httpd=httpd.make_driver(),
argv=argv.make_driver(),
logging=logging.make_driver(),
file=file.make_driver()))
def main():
dispose = run(entry_point=Component(call=audio_encoder, input=Source),
drivers=Drivers(
encoder=encoder.make_driver(),
httpd=httpd.make_driver(),
file=file.make_driver(),
argv=argv.make_driver(),
))
dispose()
def main():
loop = asyncio.get_event_loop()
aio_scheduler = AsyncIOScheduler(loop=loop)
run(
Component(call=functools.partial(model_publisher, aio_scheduler),
input=ModelPublisherSource),
ModelPublisherDrivers(
kafka=kafka.make_driver(),
http=http.make_driver(),
file=file.make_driver(),
argv=argv.make_driver(),
))
def make_driver(loop=None):
loop = loop or asyncio.get_event_loop()
def driver(sink):
def on_subscribe(observer, scheduler):
async def tcp_client(host, port):
def on_connection_subscribe(observer, reader, writer):
async def handle_connection(observer, reader, writer):
while True:
try:
data = await reader.read(100)
if data == b'':
break
loop.call_soon(observer.on_next,
Data(data=data))
except Exception as e:
loop.call_soon(observer.on_error(e))
break
loop.call_soon(observer.on_completed)
writer.close()
asyncio.ensure_future(
handle_connection(observer, reader, writer))
try:
reader, writer = await asyncio.open_connection(host,
port,
loop=loop)
connection = rx.create(
lambda o, s: on_connection_subscribe(
o, reader, writer))
observer.on_next(
Connection(id=writer, observable=connection))
except Exception as e:
loop.call_soon(observer.on_error(e))
async def write(writer, data):
writer.write(data)
def on_next(i):
if type(i) is Connect:
asyncio.ensure_future(tcp_client(i.host, i.port))
elif type(i) is Write:
asyncio.ensure_future(write(i.id, i.data))
sink.request.subscribe(on_next=on_next,
on_completed=observer.on_completed,
on_error=observer.on_error)
return Source(response=rx.create(on_subscribe))
return Component(call=driver, input=Sink)
def make_driver(loop=None):
if loop is None:
loop = asyncio.get_event_loop()
def driver(sink):
def on_subscribe(observer):
client = None
bucket = None
def on_next(item):
nonlocal client
nonlocal bucket
if type(item) is Configure:
session = Session(aws_access_key_id=item.access_key,
aws_secret_access_key=item.secret_key)
client = session.client('s3',
endpoint_url=item.endpoint_url,
region_name=item.region_name)
bucket = item.bucket
elif type(item) is UploadObject:
try:
data = BytesIO(item.data)
client.upload_fileobj(data, bucket, item.key)
loop.call_soon_threadsafe(
observer.on_next(
Observable.just(
UploadReponse(key=item.key, id=item.id))))
except:
loop.call_soon_threadsafe(
observer.on_next(
Observable.throw(
Exception(
UploadError(key=item.key,
id=item.id)))))
else:
loop.call_soon_threadsafe(
observer.on_error,
"unknown item: {}".format(type(item)))
sink.request.subscribe(on_next=on_next,
on_error=lambda e: loop.
call_soon_threadsafe(observer.on_error, e),
on_completed=lambda: loop.
call_soon_threadsafe(observer.on_completed))
return Source(response=Observable.create(on_subscribe))
return Component(call=driver, input=Sink)
def main():
loop = asyncio.get_event_loop()
# loop.set_debug(True)
aio_scheduler = AsyncIOScheduler(loop=loop)
run(
Component(call=partial(deepspeech_server, aio_scheduler),
input=DeepspeechSource),
DeepspeechDrivers(deepspeech=deepspeech.make_driver(),
httpd=httpd.make_driver(),
argv=argv.make_driver(),
logging=logging.make_driver(),
file=file.make_driver()),
loop=loop,
)
def make_driver():
def driver(sink):
def subscribe_response(observer, scheduler):
def on_request_item(i):
if type(i) is Walk:
content = _walk(i.top, i.recursive, scheduler)
observer.on_next(
WalkResponse(top=i.top, id=i.id, content=content))
sink.request.subscribe(on_request_item)
return Source(response=rx.create(subscribe_response))
return Component(call=driver, input=Sink)
def make_driver(loop=None):
def driver(sink):
def subscribe_response(observer):
def on_request_item(i):
if type(i) is Walk:
content = walk(i.top, i.recursive)
observer.on_next(
WalkResponse(top=i.top, id=i.id, content=content))
sink.request.subscribe(on_request_item)
return Source(response=AnonymousObservable(subscribe_response))
return Component(call=driver, input=Sink)
def test_run_1src_1snk(self):
''' Creates a cycle with one sink driver and one source driver.
'''
test_values = []
Drv1Sink = namedtuple('Drv1Sink', ['values'])
def drv1(sink):
sink.values.subscribe(lambda i: test_values.append(i))
return None
Drv1Driver = Component(call=drv1, input=Drv1Sink)
Drv2Source = namedtuple('Drv2Source', ['counter'])
def drv2():
counter_stream = Observable.from_([1, 2, 3])
return Drv2Source(counter=counter_stream)
Drv2Driver = Component(call=drv2, input=None)
MainDrivers = namedtuple('MainDrivers', ['drv1', 'drv2'])
MainSource = namedtuple('MainSource', ['drv2'])
MainSink = namedtuple('MainSink', ['drv1'])
def main(sources):
val = sources.drv2.counter
return MainSink(drv1=Drv1Sink(values=val))
drivers = MainDrivers(drv1=Drv1Driver, drv2=Drv2Driver)
dispose = run(Component(call=main, input=MainSource), drivers)
dispose()
self.assertEqual(3, len(test_values))
self.assertEqual(1, test_values[0])
self.assertEqual(2, test_values[1])
self.assertEqual(3, test_values[2])
def make_driver(loop=None):
loop = loop or asyncio.get_event_loop()
def driver(sink):
feedback_observer = None
def get_feedback_observer():
return feedback_observer
def on_feedback_subscribe(observer, scheduler):
def dispose():
nonlocal feedback_observer
feedback_observer = None
nonlocal feedback_observer
feedback_observer = observer
return Disposable(dispose)
def on_subscribe(observer, scheduler):
consumer_tasks = []
def on_next(i):
if type(i) is Consumer:
print("starting consumer: {}".format(i))
task = run_consumer(loop, observer, i.server, i.group,
i.max_partition_fetch_bytes, i.topics,
i.source_type, i.feed_mode)
consumer_tasks.append(task)
elif type(i) is Producer:
run_producer(loop, observer, i.server, i.topics, i.acks,
i.max_request_size, get_feedback_observer)
else:
e = "Unknown item type: {}".format(i)
print(e)
observer.on_error(TypeError(e))
print("driver kafka subscribe")
return sink.request.subscribe(
on_next=on_next,
on_error=observer.on_error,
)
return Source(
response=rx.create(on_subscribe),
feedback=rx.create(on_feedback_subscribe),
)
return Component(call=driver, input=Sink)
def main():
loop = asyncio.get_event_loop()
# loop.set_debug(True)
aio_scheduler = AsyncIOScheduler(loop=loop)
run(
Component(
call=partial(makinage, aio_scheduler),
input=MakiNageSource),
MakiNageDrivers(
kafka=kafka.make_driver(),
http=http.make_driver(),
file=file.make_driver(),
argv=argv.make_driver(),
),
loop=loop,
)
def make_driver():
def driver(sink):
def on_data_error(e):
sys.stdout.write("error: {}".format(e))
sys.stdout.flush()
def on_data_completed():
sys.stdout.flush()
sink.data.subscribe(on_next=lambda data: sys.stdout.write(
bytes(data, 'utf-8').decode('unicode_escape')),
on_error=on_data_error,
on_completed=on_data_completed)
return None
return Component(call=driver, input=Sink)
def make_driver():
def driver(sink):
handlers = {}
def on_next(i):
nonlocal handlers
if type(i) is Log:
logging.getLogger(i.logger).log(i.level, i.message)
elif type(i) is SetLevel:
level = level_from_string(i.level)
logger = logging.getLogger(i.logger)
logger.setLevel(level)
if i.logger in handlers:
logger.removeHandler(handlers[i.logger])
handlers[i.logger] = logging.StreamHandler()
handlers[i.logger].setLevel(level)
logger.addHandler(handlers[i.logger])
sink.request.subscribe(on_next)
return None
return Component(call=driver, input=Sink)
def make_driver():
def driver(sink):
def on_subscribe(observer):
def on_next(i):
if type(i) is training.TrainRequest:
print('Train')
elif type(i) is training.TestRequest:
print('Test')
elif type(i) is training.EvalRequest:
print('Eval')
elif type(i) is Initialize:
initialize(i)
sink.request.subscribe(
on_next=on_next,
on_error=observer.on_error,
on_completed=observer.on_completed,
)
return Source(
response=Observable.create(on_subscribe)
)
return Component(call=driver, input=Sink)
def make_driver():
def driver(sink):
""" File driver.
Reads content of files provided in sink stream and outputs it in the
source stream.
warning: implementation is synchronous.
sink stream structure:
- name: identifier of the file
- path: path of the file to read
source stream structure;
- name: identifier of the file
- data: content of the file
"""
def on_context_subscribe(sink, observer, scheduler):
def on_next(i):
if type(i) is Read:
try:
with open(i.path, i.mode) as content_file:
content = content_file.read(i.size)
data = rx.just(content)
except Exception as e:
data = rx.throw(e)
observer.on_next(
ReadResponse(id=i.id, path=i.path, data=data))
elif type(i) is ReadLine:
try:
with open(i.path) as content_file:
''' from_ does not work with ImmediateScheduler
def on_data_subscribe(data_observer):
for line in content_file:
data_observer.on_next(line)
data_observer.on_completed()
data = Observable.create(on_data_subscribe)
'''
data = rx.from_(content_file)
except Exception as e:
data = rx.throw(e)
observer.on_next(
ReadResponse(id=i.id, path=i.path, data=data))
elif type(i) is Write:
try:
if i.mkdirs is True:
os.makedirs(os.path.split(i.path)[0],
exist_ok=True)
with open(i.path, i.mode) as content_file:
size = content_file.write(i.data)
status = 0 if size == len(i.data) else -1
except Exception as e:
status = e
observer.on_next(
WriteResponse(id=i.id, path=i.path, status=status))
else:
observer.on_error("file unknown command: {}".format(i))
sink.subscribe(on_next=on_next,
on_completed=observer.on_completed,
on_error=observer.on_error)
def on_subscribe(observer, scheduler):
def on_request_item(i):
if type(i) is Context:
observer.on_next(
Context(
i.id,
rx.create(
functools.partial(on_context_subscribe,
i.observable))))
else:
observer.on_error("file unknown command: {}".format(i))
dispose = sink.request.subscribe(
on_next=on_request_item,
on_error=observer.on_error,
on_completed=observer.on_completed)
return dispose
return Source(response=rx.create(on_subscribe), )
return Component(call=driver, input=Sink)
def make_driver(loop=None):
def driver(sink):
model = None
log_observer = None
def on_log_subscribe(observer, scheduler):
nonlocal log_observer
log_observer = observer
def log(message, level=logging.DEBUG):
if log_observer is not None:
log_observer.on_next(Log(
logger=__name__,
level=level,
message=message,
))
def setup_model(model_path, scorer, beam_width):
log("creating model {} with scorer {}...".format(model_path, scorer))
model = Model(model_path)
if scorer.scorer is not None:
model.enableExternalScorer(scorer.scorer)
if scorer.lm_alpha is not None and scorer.lm_beta is not None:
if model.setScorerAlphaBeta(scorer.lm_alpha, scorer.lm_beta) != 0:
raise RuntimeError("Unable to set scorer parameters")
if beam_width is not None:
if model.setBeamWidth(beam_width) != 0:
raise RuntimeError("Unable to set beam width")
log("model is ready.")
return model
def subscribe(observer, scheduler):
def on_deepspeech_request(item):
nonlocal model
if type(item) is SpeechToText:
if model is not None:
try:
_, audio = wav.read(io.BytesIO(item.data))
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
text = model.stt(audio)
log("STT result: {}".format(text))
observer.on_next(rx.just(TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e), level=logging.ERROR)
observer.on_next(rx.throw(TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
model = setup_model(
item.model, item.scorer, item.beam_width)
else:
log("unknown item: {}".format(item), level=logging.CRITICAL)
observer.on_error(
"Unknown item type: {}".format(type(item)))
sink.speech.subscribe(lambda item: on_deepspeech_request(item))
return Source(
text=rx.create(subscribe),
log=rx.create(on_log_subscribe),
)
return Component(call=driver, input=Sink)
def make_driver(loop=None):
def driver(sink):
ds_model = None
log_observer = None
def on_log_subscribe(observer):
nonlocal log_observer
log_observer = observer
def log(message, level=logging.DEBUG):
if log_observer is not None:
log_observer.on_next(
Log(
logger=__name__,
level=level,
message=message,
))
def setup_model(model_path, alphabet, lm, trie, features):
log("creating model {} {} with features {}...".format(
model_path, alphabet, features))
ds_model = Model(model_path, features.beam_width)
if lm and trie:
ds_model.enableDecoderWithLM(lm, trie, features.lm_alpha,
features.lm_beta)
log("model is ready.")
return ds_model
def subscribe(observer):
def on_deepspeech_request(item):
nonlocal ds_model
if type(item) is SpeechToText:
if ds_model is not None:
try:
fs, audio = wav.read(io.BytesIO(item.data))
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
text = ds_model.stt(audio) # , fs)
log("STT result: {}".format(text))
observer.on_next(
Observable.just(
TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e))
observer.on_next(
Observable.throw(
TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
ds_model = setup_model(
item.model, item.alphabet, item.lm, item.trie,
item.features or FeaturesParameters())
else:
log("unknown item: {}".format(item),
level=logging.CRITICAL)
observer.on_error("Unknown item type: {}".format(
type(item)))
sink.speech.subscribe(lambda item: on_deepspeech_request(item))
return Source(
text=Observable.create(subscribe),
log=Observable.create(on_log_subscribe),
)
return Component(call=driver, input=Sink)
def main():
run(Component(call=echo_server, input=EchoSource),
EchoDrivers(httpd=httpd.make_driver()))
def make_driver(scheduler=None):
def driver():
return Source(argv=rx.from_(sys.argv, scheduler=scheduler))
return Component(call=driver)
