def rmux_client(sources): response = sources.tcp_client.response.pipe(ops.share()) tcp_connect = rx.just(tcp_client.Connect( host='127.0.0.1', port='8080' )) create_observable = response.pipe( ops.flat_map(lambda connection: rx.just({'what': 'subscribe', 'id':42, 'name': '1234'}).pipe( ops.map(lambda i: json.dumps(i)), frame, ops.map(lambda j: tcp_client.Write(id=connection.id, data=j.encode())) )) ) console = response.pipe( ops.flat_map(lambda connection: connection.observable.pipe( ops.map(lambda i: i.data.decode('utf-8')), unframe, ops.map(lambda i: json.loads(i)), ops.group_by(lambda i: i['id']), ops.flat_map(lambda subscription: subscription.pipe( ops.map(notification), ops.dematerialize(), )) )), ops.map(lambda i: "item: {}\n".format(i)) ) tcp_sink = rx.merge(tcp_connect, create_observable) return Sink( tcp_client=tcp_client.Sink(request=tcp_sink), stdout=stdout.Sink(data=console), )
def test_flat_map_list(): source = [1, 2, 3, 4] actual_result = [] rx.just(source).pipe( rs.ops.flat_map(), ).subscribe(on_next=actual_result.append) assert actual_result == [1, 2, 3, 4]
def execute(self, channel_value: str, profile_data: List[Tuple[int, int]]) -> Observable: _LOG.debug("SetSpeedProfileInteractor.execute()") # pylint: disable=not-callable return rx.defer(lambda _: rx.just( self._kraken_repository.set_speed_profile(channel_value, profile_data)))
def set_led_status(self, driver: X52Driver, led_status: Union[X52ColoredLedStatus, X52LedStatus], attr_name: str) -> Observable: _LOG.debug("X52DriverInteractor.set_led_status()") return rx.defer(lambda _: rx.just( self._x52_repository.set_led_status(driver, led_status, attr_name)) )
def on_deepspeech_request(item): nonlocal model if type(item) is SpeechToText: if model is not None: try: audio = decoding.decode_audio(io.BytesIO(item.data)) text = model.stt(audio) log("STT result: {}".format(text)) observer.on_next(rx.just(TextResult( text=text, context=item.context, ))) except Exception as e: log("STT error: {}".format(e), level=logging.ERROR) observer.on_next(rx.throw(TextError( error=e, context=item.context, ))) elif type(item) is Initialize: log("initialize: {}".format(item)) model = setup_model( item.model, item.scorer, item.beam_width) else: log("unknown item: {}".format(item), level=logging.CRITICAL) observer.on_error( "Unknown item type: {}".format(type(item)))
def on_deepspeech_request(item): nonlocal model if type(item) is SpeechToText: if model is not None: try: _, audio = wav.read(io.BytesIO(item.data)) # convert to mono. # todo: move to a component or just a function here if len(audio.shape) > 1: audio = audio[:, 0] text = model.stt(audio) log("STT result: {}".format(text)) observer.on_next(rx.just(TextResult( text=text, context=item.context, ))) except Exception as e: log("STT error: {}".format(e), level=logging.ERROR) observer.on_next(rx.throw(TextError( error=e, context=item.context, ))) elif type(item) is Initialize: log("initialize: {}".format(item)) model = setup_model( item.model, item.scorer, item.beam_width) else: log("unknown item: {}".format(item), level=logging.CRITICAL) observer.on_error( "Unknown item type: {}".format(type(item)))
def test_walk_file_and_dir(self): expected_files = [ os.path.join('dfoo', 'foo'), os.path.join('dbar', 'bar'), os.path.join('dbiz', 'biz'), ] expected_dirs = ['dfoo', 'dbar', 'dbiz'] actual_files = [] create_file_tree(self.wordkir, expected_dirs, expected_files) source = walk.make_driver().call( walk.Sink(request=rx.just( walk.Walk(top=self.wordkir, id='test', recursive=True)))) class TestObserver(Observer): def on_next(self, i): actual_files.append(i) def on_completed(self): return def on_error(self, e): raise Exception(e) source.response.pipe( ops.filter(lambda i: type(i) is walk.WalkResponse), ops.flat_map(lambda i: i.content.directories), ops.flat_map(lambda i: i.files), ).subscribe(TestObserver()) for f in expected_files: self.assertIn(os.path.join(self.wordkir, f), actual_files)
def on_next(item): nonlocal client nonlocal bucket if type(item) is Configure: session = Session(aws_access_key_id=item.access_key, aws_secret_access_key=item.secret_key) client = session.client('s3', endpoint_url=item.endpoint_url, region_name=item.region_name) bucket = item.bucket elif type(item) is UploadObject: try: data = BytesIO(item.data) client.upload_fileobj(data, bucket, item.key) loop.call_soon_threadsafe( observer.on_next, rx.just(UploadReponse(key=item.key, id=item.id))) except: loop.call_soon_threadsafe( observer.on_next, rx.throw( Exception(UploadError(key=item.key, id=item.id)))) else: loop.call_soon_threadsafe( observer.on_error, "unknown item: {}".format(type(item)))
def _log_exception_return_empty_observable(self, ex: Exception, _: Observable) -> Observable: LOG.exception(f"Err = {ex}") self.main_view.set_statusbar_text(str(ex)) observable = rx.just(None) assert isinstance(operators, Observable) return observable
async def _request(request): nonlocal session if session is None: session = aiohttp.ClientSession() try: response = await session.request( request.method, request.url, params=request.params, data=request.data, headers=request.headers, allow_redirects=request.allow_redirects, max_redirects=request.max_redirects) data = await response.read() observer.on_next( Response(id=request.id, response=rx.just( HttpResponse( status=response.status, reason=response.status, method=response.method, url=response.url, data=data, cookies=response.cookies, headers=response.headers, content_type=response.content_type)))) except Exception as e: #print("exception: {}, {}".format(e, traceback.format_exc())) observer.on_next( Response(id=request.id, response=rx.throw(e))) pass
def create_model_readers(config): readers = {} for model in config['models']: readers[model['topic']] = rx.just(model['path']).pipe( ops.map(read_file), ) return readers
def set_mfd_profile_name_line(self, driver: X52Driver, name: str, clear_mfd: bool = False) -> Observable: _LOG.debug("X52DriverInteractor.set_mfd_line3()") return rx.defer(lambda _: rx.just( self._x52_repository.set_mfd_line3(driver, name[:_X52_MFD_LINE_SIZE ], clear_mfd)))
def _retry_handler(self, exception, source, data): if isinstance(exception, ApiException): if exception.status == 429 or exception.status == 503: _delay = self._jitter_delay() + timedelta(milliseconds=self._write_options.retry_interval) return self._retryable(data, delay=_delay) return rx.just(_BatchResponse(exception=exception, data=data))
def _read(read_request): driver_request = rx.just( Context(id=read_request, observable=read_request)) read_response = driver_response.pipe( ops.filter(lambda i: i.id is read_request), ops.flat_map(lambda i: i.observable)) return driver_request, read_response
def _log_exception_return_empty_observable(self, ex: Exception, _: Observable) -> Observable: _LOG.exception("Err = %s", ex) self.main_view.set_statusbar_text(str(ex)) if isinstance(ex, OSError): raise ex observable = rx.just(None) assert isinstance(operators, Observable) return observable
def select_flows(request: OIDCRequest) -> Observable: observables: List[Observable] = [] for response_type in sorted( request.response_type, key=lambda x: x == 'id_token' and 1 or 0): if response_type == 'code': observables.append( just(request).pipe( op.flat_map(call_async(oauth2_authorization_code_grant)))) elif response_type == 'token': observables.append( just(request).pipe( op.flat_map(call_async(oauth2_implicit_grant)))) elif response_type == 'id_token': observables.append( just(request).pipe(op.flat_map(call_async(implicit_grant)))) else: raise InvalidRequest('Invalid response_type parameter') return from_list(observables)
def list_files(config, dataset_key='voxceleb2_test_path'): files = rx.just(None).pipe( rs.with_latest_from(config), ops.starmap(lambda _, c: walk(c['config']['dataset'][dataset_key])), ops.merge_all(), ) return files,
def set_date_time(self, driver: X52Driver, use_local_time: bool, use_24h: Tuple[bool, bool, bool], clock2_offset: datetime.timedelta, clock3_offset: datetime.timedelta, date_format: X52DateFormat) -> Observable: _LOG.debug("X52DriverInteractor.set_date_time()") return rx.defer(lambda _: rx.just( self._x52_repository. set_date_time(driver, use_local_time, use_24h, clock2_offset, clock3_offset, date_format)))
def _retry_handler(self, exception, source, data): print('retry_handler: {}, source: {}'.format(exception, source)) if "server is temporarily" in str(exception): print("RETRY!!!: {}".format(datetime.datetime.now())) return self._retryable(data, delay=datetime.timedelta(seconds=2)) notification = _Notification(exception=exception, data=data) return rx.just(notification)
def get_authorization_stream(request: OIDCRequest) -> Observable: response_params: Dict[str, Any] = {} # yapf: disable return just(request).pipe( op.flat_map(call_async(validate_redirect_uri)), op.flat_map(call_async(validate_response_type)), op.flat_map(call_async(validate_scope)), op.flat_map(select_flows), op.merge_all(), op.do_action(lambda x: response_params.update(asdict(x))), op.last(), op.map(lambda x: AuthorizationResponse(**response_params)))
def insert_new_customer(customer_name, region, street_address, city, state, zip_code): stmt = text( "INSERT INTO CUSTOMER (NAME, REGION, STREET_ADDRESS, CITY, STATE, ZIP) VALUES (" ":customer_name, :region, :street_address, :city, :state, :zip_code)") result = conn.execute(stmt, customer_name=customer_name, region=region, street_address=street_address, city=city, state=state, zip_code=zip_code) return just(result.lastrowid)
def configure(): def error_report(x): tb.print_stack() sc = QtScheduler.QtScheduler(QtCore) rx.just(1).pipe( operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_1", "series_1").add_new_point(1.2))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_1", "series_1").add_new_point(-1))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_2", "series_1").add_new_point(1.2))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_2", "series_1").add_new_point(-12))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(1.2))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(-12))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(1))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_2").add_new_point(3))), operators.delay(1.0, sc), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(-6))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_4", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_5", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_6", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_7", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_8", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_9", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_10", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_11", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_12", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_13", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_14", "series_2").add_new_point(-5))), operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_15", "series_2").add_new_point(-5))), ).subscribe()
def rmux_server(sources): tcp_listen = rx.just(tcp_server.Listen(host='127.0.0.1', port='8080')) beat = sources.tcp_server.response.pipe( ops.flat_map(lambda connection: connection.observable.pipe( ops.map(lambda i: i.data.decode('utf-8')), unframe, ops.map(lambda i: json.loads(i)), ops.flat_map(lambda subscription: create_observable[subscription[ 'name']]().pipe( ops.materialize(), ops.map(lambda i: materialize_repr(i, subscription['id'])), )), ops.map(lambda i: json.dumps(i)), frame, ops.map(lambda j: tcp_server.Write(id=connection.id, data=j.encode()))))) tcp_sink = rx.merge(tcp_listen, beat) return Sink(tcp_server=tcp_server.Sink(request=tcp_sink), )
def on_next(i): if type(i) is Read: try: with open(i.path, i.mode) as content_file: content = content_file.read(i.size) data = rx.just(content) except Exception as e: data = rx.throw(e) observer.on_next( ReadResponse(id=i.id, path=i.path, data=data)) elif type(i) is ReadLine: try: with open(i.path) as content_file: ''' from_ does not work with ImmediateScheduler def on_data_subscribe(data_observer): for line in content_file: data_observer.on_next(line) data_observer.on_completed() data = Observable.create(on_data_subscribe) ''' data = rx.from_(content_file) except Exception as e: data = rx.throw(e) observer.on_next( ReadResponse(id=i.id, path=i.path, data=data)) elif type(i) is Write: try: if i.mkdirs is True: os.makedirs(os.path.split(i.path)[0], exist_ok=True) with open(i.path, i.mode) as content_file: size = content_file.write(i.data) status = 0 if size == len(i.data) else -1 except Exception as e: status = e observer.on_next( WriteResponse(id=i.id, path=i.path, status=status)) else: observer.on_error("file unknown command: {}".format(i))
def play_step(self, step, cancel): interval = rx.interval(0.1) interval_steps = rx.just(step).pipe( ops.flat_map(lambda step: interval.pipe(ops.map(lambda _: step)))) step_done = interval_steps.pipe( ops.filter(lambda step: self.player.position() >= step.step_end), ops.do_action( lambda step: self.player.set_position(step.loop_start)), ops.take(1)) loop_done = interval_steps.pipe( ops.filter(lambda step: self.player.position() >= step.loop_end), ops.do_action( lambda step: self.player.set_position(step.loop_start)), ops.take_until(cancel.pipe(ops.skip(1)))) return step_done.pipe(ops.merge(loop_done))
def identity_or_redirect(config, kv_adapter): '''Returns the configuration data as is, or the content from a redirection. Args: config: Dict containing the configuration content. kv_adapter: Consul adapter, used to monitor the configuration. Returns: Observable of configuration items Observable of http requests ''' if 'redirect' in config and config['redirect']['connector'] == 'consul': return read_config_from_consul( kv_adapter, config['redirect']['endpoint'], config['redirect']['key']) else: return rx.just(config)
def initialize_regulators(config, kafka_feedback): regulators = {} for regulator in config: control = kafka_feedback.pipe( trace_observable("regulator feedback"), ops.filter(lambda i: i[0] == regulator['feedback']), ops.map(lambda i: i[1] / 1000), pid(rx.concat(rx.just(1.0), rx.never()), -0.001, -0.001, 0.0), #ops.map(lambda i: 1/i if i != 0 else 1.0), ops.map(lambda i: max(min(i, 0.01), 0.0)), trace_observable("regulator"), ) regulators[regulator['control']] = control return regulators
def test_route_error(self): actual_sequence = [] def on_chain_item(i): nonlocal actual_sequence actual_sequence.append(i) sink, route_error = make_error_router() origin = rx.from_([rx.just(1), rx.throw(-1)]).pipe( route_error(error_map=lambda e: e.args[0] * 100), ) result = rx.merge(origin, sink) disposable = result.subscribe(on_chain_item, scheduler=CurrentThreadScheduler()) disposable.dispose() expected_sequence = [1, -100] self.assertEqual(actual_sequence, expected_sequence)
def on_deepspeech_request(item): nonlocal ds_model if type(item) is SpeechToText: if ds_model is not None: out = None try: # _, audio = wav.read(io.BytesIO(item.data)) unique_filename = str(uuid.uuid4()) out='/tmp/' + unique_filename + '.wav' convert(io.BytesIO(item.data),out,'s16','pcm_s16le',"mono",16000) _, audio = wav.read(out) # convert to mono. # todo: move to a component or just a function here if len(audio.shape) > 1: audio = audio[:, 0] text = ds_model.stt(audio) log("STT result: {}".format(text)) observer.on_next(rx.just(TextResult( text=text, context=item.context, ))) except Exception as e: log("STT error: {}".format(e), level=logging.ERROR) observer.on_next(rx.throw(TextError( error=e, context=item.context, ))) finally: if os.path.exists(out): os.remove(out) elif type(item) is Initialize: log("initialize: {}".format(item)) ds_model = setup_model( item.model, item.lm, item.trie, item.features or FeaturesParameters()) else: log("unknown item: {}".format(item), level=logging.CRITICAL) observer.on_error( "Unknown item type: {}".format(type(item)))
def on_next(item): nonlocal samplerate nonlocal bitdepth if type(item) is Configure: print("configure: {}".format(item)) samplerate = item.samplerate bitdepth = item.bitdepth elif type(item) is EncodeMp3: try: encoded_data = mp3_to_flac(item.data, samplerate, bitdepth) observer.on_next( rx.just( EncodeResult(id=item.id, key=item.key, data=encoded_data))) except: observer.on_next( rx.throw( Exception(EncodeError(key=item.key, id=item.id)))) else: observer.on_error("unknown item: {}".format(type(item)))
def test_run_just(self): result = rx.just(42).run() assert result == 42