def test_run_1snk(self): ''' Creates a cycle with one sink driver. ''' MainDrivers = namedtuple('MainDrivers', ['drv1']) MainSource = namedtuple('MainSource', []) MainSink = namedtuple('MainSink', ['drv1']) test_values = [] def drv1(sink): sink.values.subscribe(lambda i: test_values.append(i)) return None Drv1Sink = namedtuple('Drv1Sink', ['values']) Drv1Driver = Component(call=drv1, input=Drv1Sink) def main(sources): val = Observable.from_([1, 2, 3]) return MainSink(drv1=Drv1Sink(values=val)) drivers = MainDrivers(drv1=Drv1Driver) dispose = run(Component(call=main, input=MainSource), drivers) dispose() self.assertEqual(3, len(test_values)) self.assertEqual(1, test_values[0]) self.assertEqual(2, test_values[1]) self.assertEqual(3, test_values[2])
def test_constructor(self): def test_main(): pass component = Component(call=test_main, input=int) self.assertEqual(test_main, component.call) self.assertEqual(int, component.input) self.assertEqual(None, component.output) component = Component(call=test_main, output=float) self.assertEqual(test_main, component.call) self.assertEqual(None, component.input) self.assertEqual(float, component.output) self.assertRaises(TypeError, Component)
def make_driver(): def encoder(sink): def on_subscribe(observer): samplerate = None bitdepth = None def on_next(item): nonlocal samplerate nonlocal bitdepth if type(item) is Configure: print("configure: {}".format(item)) samplerate = item.samplerate bitdepth = item.bitdepth elif type(item) is EncodeMp3: encoded_data = mp3_to_flac(item.data, samplerate, bitdepth) observer.on_next( EncodeResult(id=item.id, key=item.key, data=encoded_data)) else: observer.on_error("unknown item: {}".format(type(item))) sink.request.subscribe( on_next=on_next, on_error=lambda e: observer.on_error(e), on_completed=lambda: observer.on_completed(), ) return Source(response=Observable.create(on_subscribe)) return Component(call=encoder, input=Sink)
def make_driver(loop=None): loop = asyncio.get_event_loop() if loop is None else loop def driver(sink): def on_subscribe(observer, scheduler): watcher = aionotify.Watcher() async def read_events(): nonlocal observer await watcher.setup(loop) while True: event = await watcher.get_event() loop.call_soon(observer.on_next, Event(id=event.alias, path=event.name)) watcher.close() def on_next(item): if type(item) is AddWatch: watcher.watch(alias=item.id, path=item.path, flags=item.flags) elif type(item) is Start: asyncio.ensure_future(read_events()) else: observer.on_error("unknown item: {}".format(type(item))) sink.request.subscribe(on_next=on_next, on_error=lambda e: observer.on_error(e)) return Source(response=rx.create(on_subscribe)) return Component(call=driver, input=Sink)
def make_driver(): def driver(sink): def on_subscribe(observer): def on_next(i): if type(i) is SetSeed: random.seed(i.value) elif type(i) is Shuffle: data = list(i.data) random.shuffle(data) observer.on_next(ShuffleResponse( id=i.id, data=data )) else: observer.on_error("unknown item: {}".format(i)) sink.request.subscribe( on_next=on_next, on_error=observer.on_error, on_completed=observer.on_completed, ) return Source( response=Observable.create(on_subscribe) ) return Component(call=driver, input=Sink)
def make_driver(loop=None): def driver(sink): ''' Routes must be configured before starting the server. ''' session = None def on_response_subscribe(observer, scheduler): async def _request(request): nonlocal session if session is None: session = aiohttp.ClientSession() try: response = await session.request( request.method, request.url, params=request.params, data=request.data, headers=request.headers, allow_redirects=request.allow_redirects, max_redirects=request.max_redirects) data = await response.read() observer.on_next( Response(id=request.id, response=rx.just( HttpResponse( status=response.status, reason=response.status, method=response.method, url=response.url, data=data, cookies=response.cookies, headers=response.headers, content_type=response.content_type)))) except Exception as e: #print("exception: {}, {}".format(e, traceback.format_exc())) observer.on_next( Response(id=request.id, response=rx.throw(e))) pass def on_request_item(i): if type(i) is Request: asyncio.ensure_future(_request(i), loop=loop) else: print("received unknown item: {}".format(type(i))) def on_request_error(e): print("http sink error: {}, {}".format(e, traceback.format_exc())) return sink.request.subscribe(on_next=on_request_item, on_error=on_request_error) return Source(response=rx.create(on_response_subscribe)) return Component(call=driver, input=Sink)
def make_driver(): handlers = {} observer = None def driver(sink): def on_subscribe(o): nonlocal observer observer = o def on_request_item(i): nonlocal handlers nonlocal observer if type(i) is Log: logging.getLogger(i.logger).log(i.level, i.message) elif type(i) is SetLevel: level = level_from_string(i.level) logger = logging.getLogger(i.logger) logger.setLevel(level) if i.logger in handlers: logger.removeHandler(handlers[i.logger]) handlers[i.logger] = logging.StreamHandler() handlers[i.logger].setLevel(level) logger.addHandler(handlers[i.logger]) elif type(i) is SetLevelDone: if observer is not None: observer.on_next(i) else: if observer is not None: observer.on_error("invalid item: {}".format(i)) sink.request.subscribe(on_request_item) return Source(response=Observable.create(on_subscribe)) return Component(call=driver, input=Sink)
def make_driver(loop=None): ''' Returns a stop driver. The optional loop argument can be provided to use the driver in another loop than the default one. Parameters ----------- loop: BaseEventLoop The event loop to use instead of the default one. ''' loop = loop or asyncio.get_event_loop() def stop(i = None): loop.stop() def driver(sink): ''' The stop driver stops the asyncio event loop. The event loop is stopped as soon as an event is received on the control observable or when it completes (both in case of success or error). Parameters ---------- sink: Sink ''' sink.control.subscribe( on_next=stop, on_error=stop, on_completed=stop) return None return Component(call=driver, input=Sink)
def make_driver(): def encoder(sink): def on_subscribe(observer, scheduler): storage_path = None def on_next(item): if type(item) is Initialize: nonlocal storage_path storage_path = item.storage_path elif type(item) is EncodeMp3: encoded_file = mp3_to_flac(item.data, storage_path, name=item.key) observer.on_next( EncodeResult(id=item.id, file=encoded_file)) else: observer.on_error("unknown item: {}".format(type(item))) sink.request.subscribe( on_next=on_next, on_error=lambda e: observer.on_error(e), on_completed=lambda: observer.on_completed(), ) return Source(response=rx.create(on_subscribe)) return Component(call=encoder, input=Sink)
def main(): dispose = run(entry_point=Component(call=rmux_client, input=Source), drivers=Drivers( tcp_client=tcp_client.make_driver(), stdout=stdout.make_driver(), )) dispose()
def make_driver(): def driver(sink): def on_subscribe(observer, scheduler): client = None bucket = None def on_next(item): nonlocal client nonlocal bucket if type(item) is Configure: session = Session(aws_access_key_id=item.access_key, aws_secret_access_key=item.secret_key) client = session.client('s3', endpoint_url=item.endpoint_url, region_name=item.region_name) bucket = item.bucket elif type(item) is UploadObject: data = BytesIO(item.data) client.upload_fileobj(data, bucket, item.key) observer.on_next(UploadReponse(key=item.key, id=item.id)) else: observer.on_error("unknown item: {}".format(type(item))) sink.request.subscribe( on_next=on_next, on_error=lambda e: observer.on_error(e), on_completed=lambda: observer.on_completed()) return Source(response=rx.create(on_subscribe)) return Component(call=driver, input=Sink)
def main(): run( Component(call=deepspeech_server, input=DeepspeechSource), DeepspeechDrivers(deepspeech=deepspeech.make_driver(), httpd=httpd.make_driver(), argv=argv.make_driver(), logging=logging.make_driver(), file=file.make_driver()))
def main(): dispose = run(entry_point=Component(call=audio_encoder, input=Source), drivers=Drivers( encoder=encoder.make_driver(), httpd=httpd.make_driver(), file=file.make_driver(), argv=argv.make_driver(), )) dispose()
def main(): loop = asyncio.get_event_loop() aio_scheduler = AsyncIOScheduler(loop=loop) run( Component(call=functools.partial(model_publisher, aio_scheduler), input=ModelPublisherSource), ModelPublisherDrivers( kafka=kafka.make_driver(), http=http.make_driver(), file=file.make_driver(), argv=argv.make_driver(), ))
def make_driver(loop=None): loop = loop or asyncio.get_event_loop() def driver(sink): def on_subscribe(observer, scheduler): async def tcp_client(host, port): def on_connection_subscribe(observer, reader, writer): async def handle_connection(observer, reader, writer): while True: try: data = await reader.read(100) if data == b'': break loop.call_soon(observer.on_next, Data(data=data)) except Exception as e: loop.call_soon(observer.on_error(e)) break loop.call_soon(observer.on_completed) writer.close() asyncio.ensure_future( handle_connection(observer, reader, writer)) try: reader, writer = await asyncio.open_connection(host, port, loop=loop) connection = rx.create( lambda o, s: on_connection_subscribe( o, reader, writer)) observer.on_next( Connection(id=writer, observable=connection)) except Exception as e: loop.call_soon(observer.on_error(e)) async def write(writer, data): writer.write(data) def on_next(i): if type(i) is Connect: asyncio.ensure_future(tcp_client(i.host, i.port)) elif type(i) is Write: asyncio.ensure_future(write(i.id, i.data)) sink.request.subscribe(on_next=on_next, on_completed=observer.on_completed, on_error=observer.on_error) return Source(response=rx.create(on_subscribe)) return Component(call=driver, input=Sink)
def make_driver(loop=None): if loop is None: loop = asyncio.get_event_loop() def driver(sink): def on_subscribe(observer): client = None bucket = None def on_next(item): nonlocal client nonlocal bucket if type(item) is Configure: session = Session(aws_access_key_id=item.access_key, aws_secret_access_key=item.secret_key) client = session.client('s3', endpoint_url=item.endpoint_url, region_name=item.region_name) bucket = item.bucket elif type(item) is UploadObject: try: data = BytesIO(item.data) client.upload_fileobj(data, bucket, item.key) loop.call_soon_threadsafe( observer.on_next( Observable.just( UploadReponse(key=item.key, id=item.id)))) except: loop.call_soon_threadsafe( observer.on_next( Observable.throw( Exception( UploadError(key=item.key, id=item.id))))) else: loop.call_soon_threadsafe( observer.on_error, "unknown item: {}".format(type(item))) sink.request.subscribe(on_next=on_next, on_error=lambda e: loop. call_soon_threadsafe(observer.on_error, e), on_completed=lambda: loop. call_soon_threadsafe(observer.on_completed)) return Source(response=Observable.create(on_subscribe)) return Component(call=driver, input=Sink)
def main(): loop = asyncio.get_event_loop() # loop.set_debug(True) aio_scheduler = AsyncIOScheduler(loop=loop) run( Component(call=partial(deepspeech_server, aio_scheduler), input=DeepspeechSource), DeepspeechDrivers(deepspeech=deepspeech.make_driver(), httpd=httpd.make_driver(), argv=argv.make_driver(), logging=logging.make_driver(), file=file.make_driver()), loop=loop, )
def make_driver(): def driver(sink): def subscribe_response(observer, scheduler): def on_request_item(i): if type(i) is Walk: content = _walk(i.top, i.recursive, scheduler) observer.on_next( WalkResponse(top=i.top, id=i.id, content=content)) sink.request.subscribe(on_request_item) return Source(response=rx.create(subscribe_response)) return Component(call=driver, input=Sink)
def make_driver(loop=None): def driver(sink): def subscribe_response(observer): def on_request_item(i): if type(i) is Walk: content = walk(i.top, i.recursive) observer.on_next( WalkResponse(top=i.top, id=i.id, content=content)) sink.request.subscribe(on_request_item) return Source(response=AnonymousObservable(subscribe_response)) return Component(call=driver, input=Sink)
def test_run_1src_1snk(self): ''' Creates a cycle with one sink driver and one source driver. ''' test_values = [] Drv1Sink = namedtuple('Drv1Sink', ['values']) def drv1(sink): sink.values.subscribe(lambda i: test_values.append(i)) return None Drv1Driver = Component(call=drv1, input=Drv1Sink) Drv2Source = namedtuple('Drv2Source', ['counter']) def drv2(): counter_stream = Observable.from_([1, 2, 3]) return Drv2Source(counter=counter_stream) Drv2Driver = Component(call=drv2, input=None) MainDrivers = namedtuple('MainDrivers', ['drv1', 'drv2']) MainSource = namedtuple('MainSource', ['drv2']) MainSink = namedtuple('MainSink', ['drv1']) def main(sources): val = sources.drv2.counter return MainSink(drv1=Drv1Sink(values=val)) drivers = MainDrivers(drv1=Drv1Driver, drv2=Drv2Driver) dispose = run(Component(call=main, input=MainSource), drivers) dispose() self.assertEqual(3, len(test_values)) self.assertEqual(1, test_values[0]) self.assertEqual(2, test_values[1]) self.assertEqual(3, test_values[2])
def make_driver(loop=None): loop = loop or asyncio.get_event_loop() def driver(sink): feedback_observer = None def get_feedback_observer(): return feedback_observer def on_feedback_subscribe(observer, scheduler): def dispose(): nonlocal feedback_observer feedback_observer = None nonlocal feedback_observer feedback_observer = observer return Disposable(dispose) def on_subscribe(observer, scheduler): consumer_tasks = [] def on_next(i): if type(i) is Consumer: print("starting consumer: {}".format(i)) task = run_consumer(loop, observer, i.server, i.group, i.max_partition_fetch_bytes, i.topics, i.source_type, i.feed_mode) consumer_tasks.append(task) elif type(i) is Producer: run_producer(loop, observer, i.server, i.topics, i.acks, i.max_request_size, get_feedback_observer) else: e = "Unknown item type: {}".format(i) print(e) observer.on_error(TypeError(e)) print("driver kafka subscribe") return sink.request.subscribe( on_next=on_next, on_error=observer.on_error, ) return Source( response=rx.create(on_subscribe), feedback=rx.create(on_feedback_subscribe), ) return Component(call=driver, input=Sink)
def main(): loop = asyncio.get_event_loop() # loop.set_debug(True) aio_scheduler = AsyncIOScheduler(loop=loop) run( Component( call=partial(makinage, aio_scheduler), input=MakiNageSource), MakiNageDrivers( kafka=kafka.make_driver(), http=http.make_driver(), file=file.make_driver(), argv=argv.make_driver(), ), loop=loop, )
def make_driver(): def driver(sink): def on_data_error(e): sys.stdout.write("error: {}".format(e)) sys.stdout.flush() def on_data_completed(): sys.stdout.flush() sink.data.subscribe(on_next=lambda data: sys.stdout.write( bytes(data, 'utf-8').decode('unicode_escape')), on_error=on_data_error, on_completed=on_data_completed) return None return Component(call=driver, input=Sink)
def make_driver(): def driver(sink): handlers = {} def on_next(i): nonlocal handlers if type(i) is Log: logging.getLogger(i.logger).log(i.level, i.message) elif type(i) is SetLevel: level = level_from_string(i.level) logger = logging.getLogger(i.logger) logger.setLevel(level) if i.logger in handlers: logger.removeHandler(handlers[i.logger]) handlers[i.logger] = logging.StreamHandler() handlers[i.logger].setLevel(level) logger.addHandler(handlers[i.logger]) sink.request.subscribe(on_next) return None return Component(call=driver, input=Sink)
def make_driver(): def driver(sink): def on_subscribe(observer): def on_next(i): if type(i) is training.TrainRequest: print('Train') elif type(i) is training.TestRequest: print('Test') elif type(i) is training.EvalRequest: print('Eval') elif type(i) is Initialize: initialize(i) sink.request.subscribe( on_next=on_next, on_error=observer.on_error, on_completed=observer.on_completed, ) return Source( response=Observable.create(on_subscribe) ) return Component(call=driver, input=Sink)
def make_driver(): def driver(sink): """ File driver. Reads content of files provided in sink stream and outputs it in the source stream. warning: implementation is synchronous. sink stream structure: - name: identifier of the file - path: path of the file to read source stream structure; - name: identifier of the file - data: content of the file """ def on_context_subscribe(sink, observer, scheduler): def on_next(i): if type(i) is Read: try: with open(i.path, i.mode) as content_file: content = content_file.read(i.size) data = rx.just(content) except Exception as e: data = rx.throw(e) observer.on_next( ReadResponse(id=i.id, path=i.path, data=data)) elif type(i) is ReadLine: try: with open(i.path) as content_file: ''' from_ does not work with ImmediateScheduler def on_data_subscribe(data_observer): for line in content_file: data_observer.on_next(line) data_observer.on_completed() data = Observable.create(on_data_subscribe) ''' data = rx.from_(content_file) except Exception as e: data = rx.throw(e) observer.on_next( ReadResponse(id=i.id, path=i.path, data=data)) elif type(i) is Write: try: if i.mkdirs is True: os.makedirs(os.path.split(i.path)[0], exist_ok=True) with open(i.path, i.mode) as content_file: size = content_file.write(i.data) status = 0 if size == len(i.data) else -1 except Exception as e: status = e observer.on_next( WriteResponse(id=i.id, path=i.path, status=status)) else: observer.on_error("file unknown command: {}".format(i)) sink.subscribe(on_next=on_next, on_completed=observer.on_completed, on_error=observer.on_error) def on_subscribe(observer, scheduler): def on_request_item(i): if type(i) is Context: observer.on_next( Context( i.id, rx.create( functools.partial(on_context_subscribe, i.observable)))) else: observer.on_error("file unknown command: {}".format(i)) dispose = sink.request.subscribe( on_next=on_request_item, on_error=observer.on_error, on_completed=observer.on_completed) return dispose return Source(response=rx.create(on_subscribe), ) return Component(call=driver, input=Sink)
def make_driver(loop=None): def driver(sink): model = None log_observer = None def on_log_subscribe(observer, scheduler): nonlocal log_observer log_observer = observer def log(message, level=logging.DEBUG): if log_observer is not None: log_observer.on_next(Log( logger=__name__, level=level, message=message, )) def setup_model(model_path, scorer, beam_width): log("creating model {} with scorer {}...".format(model_path, scorer)) model = Model(model_path) if scorer.scorer is not None: model.enableExternalScorer(scorer.scorer) if scorer.lm_alpha is not None and scorer.lm_beta is not None: if model.setScorerAlphaBeta(scorer.lm_alpha, scorer.lm_beta) != 0: raise RuntimeError("Unable to set scorer parameters") if beam_width is not None: if model.setBeamWidth(beam_width) != 0: raise RuntimeError("Unable to set beam width") log("model is ready.") return model def subscribe(observer, scheduler): def on_deepspeech_request(item): nonlocal model if type(item) is SpeechToText: if model is not None: try: _, audio = wav.read(io.BytesIO(item.data)) # convert to mono. # todo: move to a component or just a function here if len(audio.shape) > 1: audio = audio[:, 0] text = model.stt(audio) log("STT result: {}".format(text)) observer.on_next(rx.just(TextResult( text=text, context=item.context, ))) except Exception as e: log("STT error: {}".format(e), level=logging.ERROR) observer.on_next(rx.throw(TextError( error=e, context=item.context, ))) elif type(item) is Initialize: log("initialize: {}".format(item)) model = setup_model( item.model, item.scorer, item.beam_width) else: log("unknown item: {}".format(item), level=logging.CRITICAL) observer.on_error( "Unknown item type: {}".format(type(item))) sink.speech.subscribe(lambda item: on_deepspeech_request(item)) return Source( text=rx.create(subscribe), log=rx.create(on_log_subscribe), ) return Component(call=driver, input=Sink)
def make_driver(loop=None): def driver(sink): ds_model = None log_observer = None def on_log_subscribe(observer): nonlocal log_observer log_observer = observer def log(message, level=logging.DEBUG): if log_observer is not None: log_observer.on_next( Log( logger=__name__, level=level, message=message, )) def setup_model(model_path, alphabet, lm, trie, features): log("creating model {} {} with features {}...".format( model_path, alphabet, features)) ds_model = Model(model_path, features.beam_width) if lm and trie: ds_model.enableDecoderWithLM(lm, trie, features.lm_alpha, features.lm_beta) log("model is ready.") return ds_model def subscribe(observer): def on_deepspeech_request(item): nonlocal ds_model if type(item) is SpeechToText: if ds_model is not None: try: fs, audio = wav.read(io.BytesIO(item.data)) # convert to mono. # todo: move to a component or just a function here if len(audio.shape) > 1: audio = audio[:, 0] text = ds_model.stt(audio) # , fs) log("STT result: {}".format(text)) observer.on_next( Observable.just( TextResult( text=text, context=item.context, ))) except Exception as e: log("STT error: {}".format(e)) observer.on_next( Observable.throw( TextError( error=e, context=item.context, ))) elif type(item) is Initialize: log("initialize: {}".format(item)) ds_model = setup_model( item.model, item.alphabet, item.lm, item.trie, item.features or FeaturesParameters()) else: log("unknown item: {}".format(item), level=logging.CRITICAL) observer.on_error("Unknown item type: {}".format( type(item))) sink.speech.subscribe(lambda item: on_deepspeech_request(item)) return Source( text=Observable.create(subscribe), log=Observable.create(on_log_subscribe), ) return Component(call=driver, input=Sink)
def main(): run(Component(call=echo_server, input=EchoSource), EchoDrivers(httpd=httpd.make_driver()))
def make_driver(scheduler=None): def driver(): return Source(argv=rx.from_(sys.argv, scheduler=scheduler)) return Component(call=driver)