def rmux_client(sources):
response = sources.tcp_client.response.pipe(ops.share())
tcp_connect = rx.just(tcp_client.Connect(
host='127.0.0.1', port='8080'
))
create_observable = response.pipe(
ops.flat_map(lambda connection:
rx.just({'what': 'subscribe', 'id':42, 'name': '1234'}).pipe(
ops.map(lambda i: json.dumps(i)),
frame,
ops.map(lambda j: tcp_client.Write(id=connection.id, data=j.encode()))
))
)
console = response.pipe(
ops.flat_map(lambda connection: connection.observable.pipe(
ops.map(lambda i: i.data.decode('utf-8')),
unframe,
ops.map(lambda i: json.loads(i)),
ops.group_by(lambda i: i['id']),
ops.flat_map(lambda subscription: subscription.pipe(
ops.map(notification),
ops.dematerialize(),
))
)),
ops.map(lambda i: "item: {}\n".format(i))
)
tcp_sink = rx.merge(tcp_connect, create_observable)
return Sink(
tcp_client=tcp_client.Sink(request=tcp_sink),
stdout=stdout.Sink(data=console),
)
def test_flat_map_list():
source = [1, 2, 3, 4]
actual_result = []
rx.just(source).pipe(
rs.ops.flat_map(), ).subscribe(on_next=actual_result.append)
assert actual_result == [1, 2, 3, 4]
def execute(self, channel_value: str,
profile_data: List[Tuple[int, int]]) -> Observable:
_LOG.debug("SetSpeedProfileInteractor.execute()")
# pylint: disable=not-callable
return rx.defer(lambda _: rx.just(
self._kraken_repository.set_speed_profile(channel_value,
profile_data)))
def set_led_status(self, driver: X52Driver,
led_status: Union[X52ColoredLedStatus, X52LedStatus],
attr_name: str) -> Observable:
_LOG.debug("X52DriverInteractor.set_led_status()")
return rx.defer(lambda _: rx.just(
self._x52_repository.set_led_status(driver, led_status, attr_name))
)
def on_deepspeech_request(item):
nonlocal model
if type(item) is SpeechToText:
if model is not None:
try:
audio = decoding.decode_audio(io.BytesIO(item.data))
text = model.stt(audio)
log("STT result: {}".format(text))
observer.on_next(rx.just(TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e), level=logging.ERROR)
observer.on_next(rx.throw(TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
model = setup_model(
item.model, item.scorer, item.beam_width)
else:
log("unknown item: {}".format(item), level=logging.CRITICAL)
observer.on_error(
"Unknown item type: {}".format(type(item)))
def on_deepspeech_request(item):
nonlocal model
if type(item) is SpeechToText:
if model is not None:
try:
_, audio = wav.read(io.BytesIO(item.data))
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
text = model.stt(audio)
log("STT result: {}".format(text))
observer.on_next(rx.just(TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e), level=logging.ERROR)
observer.on_next(rx.throw(TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
model = setup_model(
item.model, item.scorer, item.beam_width)
else:
log("unknown item: {}".format(item), level=logging.CRITICAL)
observer.on_error(
"Unknown item type: {}".format(type(item)))
def test_walk_file_and_dir(self):
expected_files = [
os.path.join('dfoo', 'foo'),
os.path.join('dbar', 'bar'),
os.path.join('dbiz', 'biz'),
]
expected_dirs = ['dfoo', 'dbar', 'dbiz']
actual_files = []
create_file_tree(self.wordkir, expected_dirs, expected_files)
source = walk.make_driver().call(
walk.Sink(request=rx.just(
walk.Walk(top=self.wordkir, id='test', recursive=True))))
class TestObserver(Observer):
def on_next(self, i):
actual_files.append(i)
def on_completed(self):
return
def on_error(self, e):
raise Exception(e)
source.response.pipe(
ops.filter(lambda i: type(i) is walk.WalkResponse),
ops.flat_map(lambda i: i.content.directories),
ops.flat_map(lambda i: i.files),
).subscribe(TestObserver())
for f in expected_files:
self.assertIn(os.path.join(self.wordkir, f), actual_files)
def on_next(item):
nonlocal client
nonlocal bucket
if type(item) is Configure:
session = Session(aws_access_key_id=item.access_key,
aws_secret_access_key=item.secret_key)
client = session.client('s3',
endpoint_url=item.endpoint_url,
region_name=item.region_name)
bucket = item.bucket
elif type(item) is UploadObject:
try:
data = BytesIO(item.data)
client.upload_fileobj(data, bucket, item.key)
loop.call_soon_threadsafe(
observer.on_next,
rx.just(UploadReponse(key=item.key, id=item.id)))
except:
loop.call_soon_threadsafe(
observer.on_next,
rx.throw(
Exception(UploadError(key=item.key,
id=item.id))))
else:
loop.call_soon_threadsafe(
observer.on_error,
"unknown item: {}".format(type(item)))
def _log_exception_return_empty_observable(self, ex: Exception,
_: Observable) -> Observable:
LOG.exception(f"Err = {ex}")
self.main_view.set_statusbar_text(str(ex))
observable = rx.just(None)
assert isinstance(operators, Observable)
return observable
async def _request(request):
nonlocal session
if session is None:
session = aiohttp.ClientSession()
try:
response = await session.request(
request.method,
request.url,
params=request.params,
data=request.data,
headers=request.headers,
allow_redirects=request.allow_redirects,
max_redirects=request.max_redirects)
data = await response.read()
observer.on_next(
Response(id=request.id,
response=rx.just(
HttpResponse(
status=response.status,
reason=response.status,
method=response.method,
url=response.url,
data=data,
cookies=response.cookies,
headers=response.headers,
content_type=response.content_type))))
except Exception as e:
#print("exception: {}, {}".format(e, traceback.format_exc()))
observer.on_next(
Response(id=request.id, response=rx.throw(e)))
pass
def create_model_readers(config):
readers = {}
for model in config['models']:
readers[model['topic']] = rx.just(model['path']).pipe(
ops.map(read_file), )
return readers
def set_mfd_profile_name_line(self,
driver: X52Driver,
name: str,
clear_mfd: bool = False) -> Observable:
_LOG.debug("X52DriverInteractor.set_mfd_line3()")
return rx.defer(lambda _: rx.just(
self._x52_repository.set_mfd_line3(driver, name[:_X52_MFD_LINE_SIZE
], clear_mfd)))
def _retry_handler(self, exception, source, data):
if isinstance(exception, ApiException):
if exception.status == 429 or exception.status == 503:
_delay = self._jitter_delay() + timedelta(milliseconds=self._write_options.retry_interval)
return self._retryable(data, delay=_delay)
return rx.just(_BatchResponse(exception=exception, data=data))
def _read(read_request):
driver_request = rx.just(
Context(id=read_request, observable=read_request))
read_response = driver_response.pipe(
ops.filter(lambda i: i.id is read_request),
ops.flat_map(lambda i: i.observable))
return driver_request, read_response
def _log_exception_return_empty_observable(self, ex: Exception,
_: Observable) -> Observable:
_LOG.exception("Err = %s", ex)
self.main_view.set_statusbar_text(str(ex))
if isinstance(ex, OSError):
raise ex
observable = rx.just(None)
assert isinstance(operators, Observable)
return observable
def select_flows(request: OIDCRequest) -> Observable:
observables: List[Observable] = []
for response_type in sorted(
request.response_type, key=lambda x: x == 'id_token' and 1 or 0):
if response_type == 'code':
observables.append(
just(request).pipe(
op.flat_map(call_async(oauth2_authorization_code_grant))))
elif response_type == 'token':
observables.append(
just(request).pipe(
op.flat_map(call_async(oauth2_implicit_grant))))
elif response_type == 'id_token':
observables.append(
just(request).pipe(op.flat_map(call_async(implicit_grant))))
else:
raise InvalidRequest('Invalid response_type parameter')
return from_list(observables)
def list_files(config, dataset_key='voxceleb2_test_path'):
files = rx.just(None).pipe(
rs.with_latest_from(config),
ops.starmap(lambda _, c: walk(c['config']['dataset'][dataset_key])),
ops.merge_all(),
)
return files,
def set_date_time(self, driver: X52Driver, use_local_time: bool,
use_24h: Tuple[bool, bool,
bool], clock2_offset: datetime.timedelta,
clock3_offset: datetime.timedelta,
date_format: X52DateFormat) -> Observable:
_LOG.debug("X52DriverInteractor.set_date_time()")
return rx.defer(lambda _: rx.just(
self._x52_repository.
set_date_time(driver, use_local_time, use_24h, clock2_offset,
clock3_offset, date_format)))
def _retry_handler(self, exception, source, data):
print('retry_handler: {}, source: {}'.format(exception, source))
if "server is temporarily" in str(exception):
print("RETRY!!!: {}".format(datetime.datetime.now()))
return self._retryable(data, delay=datetime.timedelta(seconds=2))
notification = _Notification(exception=exception, data=data)
return rx.just(notification)
def get_authorization_stream(request: OIDCRequest) -> Observable:
response_params: Dict[str, Any] = {}
# yapf: disable
return just(request).pipe(
op.flat_map(call_async(validate_redirect_uri)),
op.flat_map(call_async(validate_response_type)),
op.flat_map(call_async(validate_scope)),
op.flat_map(select_flows),
op.merge_all(),
op.do_action(lambda x: response_params.update(asdict(x))),
op.last(),
op.map(lambda x: AuthorizationResponse(**response_params)))
def insert_new_customer(customer_name, region, street_address, city, state,
zip_code):
stmt = text(
"INSERT INTO CUSTOMER (NAME, REGION, STREET_ADDRESS, CITY, STATE, ZIP) VALUES ("
":customer_name, :region, :street_address, :city, :state, :zip_code)")
result = conn.execute(stmt,
customer_name=customer_name,
region=region,
street_address=street_address,
city=city,
state=state,
zip_code=zip_code)
return just(result.lastrowid)
def configure():
def error_report(x):
tb.print_stack()
sc = QtScheduler.QtScheduler(QtCore)
rx.just(1).pipe(
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_1", "series_1").add_new_point(1.2))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_1", "series_1").add_new_point(-1))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_2", "series_1").add_new_point(1.2))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_2", "series_1").add_new_point(-12))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(1.2))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(-12))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(1))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_2").add_new_point(3))),
operators.delay(1.0, sc),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_1").add_new_point(-6))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_3", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_4", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_5", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_6", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_7", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_8", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_9", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_10", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_11", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_12", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_13", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_14", "series_2").add_new_point(-5))),
operators.map(lambda x: timeline_widget.update_plot(TimelineDataPoint("plot_15", "series_2").add_new_point(-5))),
).subscribe()
def rmux_server(sources):
tcp_listen = rx.just(tcp_server.Listen(host='127.0.0.1', port='8080'))
beat = sources.tcp_server.response.pipe(
ops.flat_map(lambda connection: connection.observable.pipe(
ops.map(lambda i: i.data.decode('utf-8')), unframe,
ops.map(lambda i: json.loads(i)),
ops.flat_map(lambda subscription: create_observable[subscription[
'name']]().pipe(
ops.materialize(),
ops.map(lambda i: materialize_repr(i, subscription['id'])),
)), ops.map(lambda i: json.dumps(i)), frame,
ops.map(lambda j: tcp_server.Write(id=connection.id,
data=j.encode())))))
tcp_sink = rx.merge(tcp_listen, beat)
return Sink(tcp_server=tcp_server.Sink(request=tcp_sink), )
def on_next(i):
if type(i) is Read:
try:
with open(i.path, i.mode) as content_file:
content = content_file.read(i.size)
data = rx.just(content)
except Exception as e:
data = rx.throw(e)
observer.on_next(
ReadResponse(id=i.id, path=i.path, data=data))
elif type(i) is ReadLine:
try:
with open(i.path) as content_file:
''' from_ does not work with ImmediateScheduler
def on_data_subscribe(data_observer):
for line in content_file:
data_observer.on_next(line)
data_observer.on_completed()
data = Observable.create(on_data_subscribe)
'''
data = rx.from_(content_file)
except Exception as e:
data = rx.throw(e)
observer.on_next(
ReadResponse(id=i.id, path=i.path, data=data))
elif type(i) is Write:
try:
if i.mkdirs is True:
os.makedirs(os.path.split(i.path)[0],
exist_ok=True)
with open(i.path, i.mode) as content_file:
size = content_file.write(i.data)
status = 0 if size == len(i.data) else -1
except Exception as e:
status = e
observer.on_next(
WriteResponse(id=i.id, path=i.path, status=status))
else:
observer.on_error("file unknown command: {}".format(i))
def play_step(self, step, cancel):
interval = rx.interval(0.1)
interval_steps = rx.just(step).pipe(
ops.flat_map(lambda step: interval.pipe(ops.map(lambda _: step))))
step_done = interval_steps.pipe(
ops.filter(lambda step: self.player.position() >= step.step_end),
ops.do_action(
lambda step: self.player.set_position(step.loop_start)),
ops.take(1))
loop_done = interval_steps.pipe(
ops.filter(lambda step: self.player.position() >= step.loop_end),
ops.do_action(
lambda step: self.player.set_position(step.loop_start)),
ops.take_until(cancel.pipe(ops.skip(1))))
return step_done.pipe(ops.merge(loop_done))
def identity_or_redirect(config, kv_adapter):
'''Returns the configuration data as is, or the content from a redirection.
Args:
config: Dict containing the configuration content.
kv_adapter: Consul adapter, used to monitor the configuration.
Returns:
Observable of configuration items
Observable of http requests
'''
if 'redirect' in config and config['redirect']['connector'] == 'consul':
return read_config_from_consul(
kv_adapter,
config['redirect']['endpoint'],
config['redirect']['key'])
else:
return rx.just(config)
def initialize_regulators(config, kafka_feedback):
regulators = {}
for regulator in config:
control = kafka_feedback.pipe(
trace_observable("regulator feedback"),
ops.filter(lambda i: i[0] == regulator['feedback']),
ops.map(lambda i: i[1] / 1000),
pid(rx.concat(rx.just(1.0), rx.never()),
-0.001, -0.001, 0.0),
#ops.map(lambda i: 1/i if i != 0 else 1.0),
ops.map(lambda i: max(min(i, 0.01), 0.0)),
trace_observable("regulator"),
)
regulators[regulator['control']] = control
return regulators
def test_route_error(self):
actual_sequence = []
def on_chain_item(i):
nonlocal actual_sequence
actual_sequence.append(i)
sink, route_error = make_error_router()
origin = rx.from_([rx.just(1), rx.throw(-1)]).pipe(
route_error(error_map=lambda e: e.args[0] * 100), )
result = rx.merge(origin, sink)
disposable = result.subscribe(on_chain_item,
scheduler=CurrentThreadScheduler())
disposable.dispose()
expected_sequence = [1, -100]
self.assertEqual(actual_sequence, expected_sequence)
def on_deepspeech_request(item):
nonlocal ds_model
if type(item) is SpeechToText:
if ds_model is not None:
out = None
try:
# _, audio = wav.read(io.BytesIO(item.data))
unique_filename = str(uuid.uuid4())
out='/tmp/' + unique_filename + '.wav'
convert(io.BytesIO(item.data),out,'s16','pcm_s16le',"mono",16000)
_, audio = wav.read(out)
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
text = ds_model.stt(audio)
log("STT result: {}".format(text))
observer.on_next(rx.just(TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e), level=logging.ERROR)
observer.on_next(rx.throw(TextError(
error=e,
context=item.context,
)))
finally:
if os.path.exists(out):
os.remove(out)
elif type(item) is Initialize:
log("initialize: {}".format(item))
ds_model = setup_model(
item.model, item.lm, item.trie, item.features or FeaturesParameters())
else:
log("unknown item: {}".format(item), level=logging.CRITICAL)
observer.on_error(
"Unknown item type: {}".format(type(item)))
def on_next(item):
nonlocal samplerate
nonlocal bitdepth
if type(item) is Configure:
print("configure: {}".format(item))
samplerate = item.samplerate
bitdepth = item.bitdepth
elif type(item) is EncodeMp3:
try:
encoded_data = mp3_to_flac(item.data, samplerate,
bitdepth)
observer.on_next(
rx.just(
EncodeResult(id=item.id,
key=item.key,
data=encoded_data)))
except:
observer.on_next(
rx.throw(
Exception(EncodeError(key=item.key,
id=item.id))))
else:
observer.on_error("unknown item: {}".format(type(item)))
def test_run_just(self):
result = rx.just(42).run()
assert result == 42
