def discover(self, thing_filter, dnssd_find_kwargs=None):
"""Starts the discovery process that will provide ThingDescriptions
that match the optional argument filter of type ThingFilter."""
supported_methods = [
DiscoveryMethod.ANY, DiscoveryMethod.LOCAL,
DiscoveryMethod.MULTICAST
]
if thing_filter.method not in supported_methods:
err = NotImplementedError("Unsupported discovery method")
# noinspection PyUnresolvedReferences
return Observable.throw(err)
if thing_filter.query:
err = NotImplementedError(
"Queries are not supported yet (please use filter.fragment)")
# noinspection PyUnresolvedReferences
return Observable.throw(err)
observables = []
if thing_filter.method in [DiscoveryMethod.ANY, DiscoveryMethod.LOCAL]:
observables.append(
self._build_local_discover_observable(thing_filter))
if thing_filter.method in [
DiscoveryMethod.ANY, DiscoveryMethod.MULTICAST
]:
observables.append(
self._build_dnssd_discover_observable(thing_filter,
dnssd_find_kwargs))
# noinspection PyUnresolvedReferences
return Observable.merge(*observables)
def subscribe(observer):
active_plans = []
external_subscriptions = {}
def on_error(err):
for v in external_subscriptions.values():
v.on_error(err)
observer.on_error(err)
out_observer = AnonymousObserver(observer.on_next, on_error, observer.on_completed)
def deactivate(active_plan):
active_plans.remove(active_plan)
if not len(active_plans):
observer.on_completed()
try:
for plan in plans:
active_plans.append(plan.activate(external_subscriptions,
out_observer, deactivate))
except Exception as ex:
Observable.throw(ex).subscribe(observer)
group = CompositeDisposable()
for join_observer in external_subscriptions.values():
join_observer.subscribe()
group.add(join_observer)
return group
def subscribe(observer):
active_plans = []
external_subscriptions = {}
def on_error(err):
for v in external_subscriptions.values():
v.on_error(err)
observer.on_error(err)
out_observer = AnonymousObserver(observer.on_next, on_error,
observer.on_completed)
def deactivate(active_plan):
active_plans.remove(active_plan)
if not len(active_plans):
observer.on_completed()
try:
for plan in plans:
active_plans.append(
plan.activate(external_subscriptions, out_observer,
deactivate))
except Exception as ex:
Observable.throw(ex).subscribe(observer)
group = CompositeDisposable()
for join_observer in external_subscriptions.values():
join_observer.subscribe()
group.add(join_observer)
return group
def on_next(i):
if type(i) is Read:
try:
with open(i.path, i.mode) as content_file:
content = content_file.read(i.size)
data = Observable.just(content)
observer.on_next(
ReadResponse(id=i.id, path=i.path, data=data))
except Exception as e:
observer.on_next(Observable.throw(e))
elif type(i) is ReadLine:
try:
with open(i.path) as content_file:
data = Observable.from_(content_file)
observer.on_next(
ReadResponse(id=i.id, path=i.path, data=data))
except Exception as e:
observer.on_next(Observable.throw(e))
elif type(i) is Write:
try:
if i.mkdirs is True:
os.makedirs(os.path.split(i.path)[0],
exist_ok=True)
with open(i.path, i.mode) as content_file:
size = content_file.write(i.data)
status = 0 if size == len(i.data) else -1
observer.on_next(
WriteResponse(id=i.id,
path=i.path,
status=status))
except Exception as e:
observer.on_next(Observable.throw(e))
else:
observer.on_error("file unknown command: {}".format(i))
def on_event(self, td, name):
"""Subscribes to an event on a remote Thing.
Returns an Observable."""
if name not in td.events:
# noinspection PyUnresolvedReferences
return Observable.throw(FormNotFoundException())
form = pick_form(td, td.get_event_forms(name), WebsocketSchemes.list())
if not form:
# noinspection PyUnresolvedReferences
return Observable.throw(FormNotFoundException())
ws_url = form.resolve_uri(td.base)
msg_req = WebsocketMessageRequest(method=WebsocketMethods.ON_EVENT,
params={"name": name},
msg_id=uuid.uuid4().hex)
def on_next(observer, msg_item):
observer.on_next(EmittedEvent(init=msg_item.data, name=name))
subscribe = self._build_subscribe(ws_url, msg_req, on_next)
# noinspection PyUnresolvedReferences
return Observable.create(subscribe)
def on_property_change(self, td, name):
"""Subscribes to property changes on a remote Thing.
Returns an Observable."""
if name not in td.properties:
# noinspection PyUnresolvedReferences
return Observable.throw(FormNotFoundException())
form = pick_form(td, td.get_property_forms(name),
WebsocketSchemes.list())
if not form:
# noinspection PyUnresolvedReferences
return Observable.throw(FormNotFoundException())
ws_url = form.resolve_uri(td.base)
msg_req = WebsocketMessageRequest(
method=WebsocketMethods.ON_PROPERTY_CHANGE,
params={"name": name},
msg_id=uuid.uuid4().hex)
def on_next(observer, msg_item):
init_name = msg_item.data["name"]
init_value = msg_item.data["value"]
init = PropertyChangeEventInit(name=init_name, value=init_value)
observer.on_next(PropertyChangeEmittedEvent(init=init))
subscribe = self._build_subscribe(ws_url, msg_req, on_next)
# noinspection PyUnresolvedReferences
return Observable.create(subscribe)
def create():
return xs.group_join(
ys, lambda x: Observable.timer(x.interval).flat_map(
Observable.throw(ex)
if x.value == 6 else Observable.empty()),
lambda y: Observable.timer(y.interval), lambda x, yy: yy.map(
lambda y: str(x.value) + y.value)).merge_all()
async def _request(request):
nonlocal session
if session is None:
session = aiohttp.ClientSession()
try:
response = await session.request(
request.method,
request.url,
params=request.params,
data=request.data,
headers=request.headers,
allow_redirects=request.allow_redirects,
max_redirects=request.max_redirects
)
data = await response.read()
observer.on_next(Response(
id=request.id,
response=Observable.just(HttpResponse(
status=response.status, reason=response.status,
method=response.method, url=response.url,
data=data, cookies=response.cookies,
headers=response.headers,
content_type=response.content_type
))
))
except Exception as e:
print("exception: {}".format(e))
observer.on_next(Response(
id=request.id,
response=Observable.throw(e)))
def on_next(item):
nonlocal client
nonlocal bucket
if type(item) is Configure:
session = Session(aws_access_key_id=item.access_key,
aws_secret_access_key=item.secret_key)
client = session.client('s3',
endpoint_url=item.endpoint_url,
region_name=item.region_name)
bucket = item.bucket
elif type(item) is UploadObject:
try:
data = BytesIO(item.data)
client.upload_fileobj(data, bucket, item.key)
loop.call_soon_threadsafe(
observer.on_next(
Observable.just(
UploadReponse(key=item.key, id=item.id))))
except:
loop.call_soon_threadsafe(
observer.on_next(
Observable.throw(
Exception(
UploadError(key=item.key,
id=item.id)))))
else:
loop.call_soon_threadsafe(
observer.on_error,
"unknown item: {}".format(type(item)))
def test_route_error(self):
actual_sequence = []
def on_chain_item(i):
nonlocal actual_sequence
actual_sequence.append(i)
sink, route_error = make_error_router()
origin = (
Observable.from_([
Observable.just(1),
Observable.throw(-1)
])
.do_action(lambda i: print)
.let(route_error,
error_map=lambda e: e.args[0] * 100,
)
.do_action(lambda i: print)
)
result = Observable.merge(origin, sink)
disposable = result.subscribe(on_chain_item)
disposable.dispose()
expected_sequence = [1, -100]
self.assertEqual(actual_sequence, expected_sequence)
def create():
return xs.join(ys,
lambda x: Observable.timer(x.interval),
lambda y: Observable.timer(y.interval).flat_map(
Observable.throw(ex) if y.value == "tin" else Observable.empty()),
lambda x, y: str(x.value) + y.value
)
def test_throw_observer_throws(self):
scheduler = TestScheduler()
xs = Observable.throw('ex')
xs.subscribe_(lambda x: None,
lambda ex: _raise('ex'),
lambda: None,
scheduler=scheduler)
self.assertRaises(RxException, scheduler.start)
def on_property_change(self, name):
"""Returns an Observable for the Property specified in the name argument,
allowing subscribing to and unsubscribing from notifications."""
try:
interaction = self._find_interaction(name=name)
except ValueError:
# noinspection PyUnresolvedReferences
return Observable.throw(Exception("Unknown property"))
if not interaction.observable:
# noinspection PyUnresolvedReferences
return Observable.throw(Exception("Property is not observable"))
def property_change_filter(item):
return item.name == DefaultThingEvent.PROPERTY_CHANGE and \
item.data.name == name
# noinspection PyUnresolvedReferences
return self._events_stream.filter(property_change_filter)
def observe_file(url):
text_request = requests.get(url)
if text_request.ok == False:
return Observable.throw("request failed: {}\n{}".format(
text_request.reason, text_request.url))
temp = NamedTemporaryFile(delete=False)
temp.file.write(text_request.content)
temp.close()
return Observable.from_(open(temp.name))\
.map(lambda line: line.strip())\
.filter(lambda l: l != "")
def on_deepspeech_request(item):
nonlocal ds_model
if type(item) is SpeechToText:
if ds_model is not None:
try:
# use ffmpeg to convert to raw
out, _ = (ffmpeg.input(
'-',
format='ogg',
).output('-',
format='s16le',
acodec='pcm_s16le',
ac=1,
ar='16k').run(capture_stdout=True,
input=item.data))
# to numpy array
audio = np.frombuffer(out, dtype=np.int16)
fs = 16000
# fs, audio = wav.read(io.BytesIO(item.data))
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
text = ds_model.stt(audio, fs)
log("STT result: {}".format(text))
observer.on_next(
Observable.just(
TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e))
observer.on_next(
Observable.throw(
TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
ds_model = setup_model(
item.model, item.alphabet, item.lm, item.trie,
item.features or FeaturesParameters())
else:
log("unknown item: {}".format(item),
level=logging.CRITICAL)
observer.on_error("Unknown item type: {}".format(
type(item)))
def on_event(self, name):
"""Returns an Observable for the Event specified in the name argument,
allowing subscribing to and unsubscribing from notifications."""
if name not in self.thing.events:
# noinspection PyUnresolvedReferences
return Observable.throw(Exception("Unknown event"))
def event_filter(item):
return item.name == name
# noinspection PyUnresolvedReferences
return self._events_stream.filter(event_filter)
def test_forward_error(self):
numbers = Observable.throw(ValueError())
expected_numbers = []
actual_numbers = []
actual_error = None
def on_next(i):
actual_numbers.append(i)
def on_error(e):
nonlocal actual_error
actual_error = e
integer_passthrough(numbers).subscribe(on_next=on_next,
on_error=on_error)
self.assertIsInstance(actual_error, ValueError)
self.assertEqual(expected_numbers, actual_numbers)
def on_deepspeech_request(item):
nonlocal ds_model
if type(item) is SpeechToText:
if ds_model is not None:
try:
audio, orig_sr = sf.read(io.BytesIO(item.data),
always_2d=True)
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
# convert to 16K
if orig_sr != 16000:
audio = librosa.resample(y=audio,
orig_sr=orig_sr,
target_sr=16000,
scale=False)
maxv = np.iinfo(np.int16).max
audio_int16 = (audio * maxv).astype(np.int16)
text = ds_model.stt(audio_int16, aSampleRate=16000)
log("STT result: {}".format(text))
observer.on_next(Observable.just(TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e))
observer.on_next(Observable.throw(TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
ds_model = setup_model(
item.model, item.alphabet, item.lm, item.trie, item.features or FeaturesParameters())
else:
log("unknown item: {}".format(item), level=logging.CRITICAL)
observer.on_error(
"Unknown item type: {}".format(type(item)))
def on_next(item):
nonlocal samplerate
nonlocal bitdepth
if type(item) is Configure:
print("configure: {}".format(item))
samplerate = item.samplerate
bitdepth = item.bitdepth
elif type(item) is EncodeMp3:
try:
encoded_data = mp3_to_flac(item.data, samplerate,
bitdepth)
observer.on_next(
Observable.just(
EncodeResult(id=item.id,
key=item.key,
data=encoded_data)))
except:
observer.on_next(
Observable.throw(
Exception(EncodeError(key=item.key,
id=item.id))))
else:
observer.on_error("unknown item: {}".format(type(item)))
def on_deepspeech_request(item):
nonlocal ds_model
if type(item) is SpeechToText:
if ds_model is not None:
try:
fs, audio = wav.read(io.BytesIO(item.data))
# convert to mono.
# todo: move to a component or just a function here
if len(audio.shape) > 1:
audio = audio[:, 0]
text = ds_model.stt(audio) # , fs)
log("STT result: {}".format(text))
observer.on_next(
Observable.just(
TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e))
observer.on_next(
Observable.throw(
TextError(
error=e,
context=item.context,
)))
elif type(item) is Initialize:
log("initialize: {}".format(item))
ds_model = setup_model(
item.model, item.alphabet, item.lm, item.trie,
item.features or FeaturesParameters())
else:
log("unknown item: {}".format(item),
level=logging.CRITICAL)
observer.on_error("Unknown item type: {}".format(
type(item)))
def create():
return Observable.throw(ex).sample(0)
def factory():
return Observable.throw('ex')
from rx import Observable
error = Observable.throw("Something wrong happened")
error.subscribe(on_next=lambda i: print("item: {}".format(i)),
on_error=lambda e: print("error: {}".format(e)),
on_completed=lambda: print("completed"))
def create():
Observable.throw(RxException()).transduce(
compose(filtering(even),
mapping(mul10))).subscribe_(noop, throw_error)
from rx import Observable
exception = Observable.throw(NotImplementedError("I do nothing"))
exception.subscribe(on_next=lambda i: print("item: {}".format(i)),
on_error=lambda e: print("error: {}".format(e)),
on_completed=lambda: print("completed"))
def on_deepspeech_request(item):
nonlocal ds_model
if type(item) is SpeechToText:
if ds_model is not None:
temp_dir = tempfile.mkdtemp()
noise_temp_filepath = temp_dir + "/noise.wav"
noise_prof_temp_filepath = temp_dir + "/noise.prof"
input_temp_filepath = temp_dir + "/input.wav"
output_temp_filepath = temp_dir + "/output.wav"
try:
# ffmpeg -i input.wav -acodec pcm_s16le -ac 1 -ar 16000 -af lowpass=3000,highpass=200 ...
# output.wav
# sox input.wav -b 16 output.wav channels 1 rate 16k sinc 200-3k -
fs, audio = wav.read(io.BytesIO(item.data))
wav.write(input_temp_filepath, fs, audio)
# Get noise profile
noise = sox.Transformer()
noise.trim(0.0, 0.1)
noise.build(input_temp_filepath,
noise_temp_filepath)
noise_prof = sox.Transformer()
noise_prof.noiseprof(noise_temp_filepath,
noise_prof_temp_filepath)
# Convert WAV file to a cleaner representation that will be better for inference.
cbn = sox.Transformer()
cbn.set_output_format(rate=16000,
channels=1,
bits=16,
encoding="signed-integer")
cbn.noisered(noise_prof_temp_filepath, 0.05)
cbn.build(input_temp_filepath,
output_temp_filepath)
fs, audio = wav.read(
open(output_temp_filepath, 'rb'))
if len(audio.shape) > 1:
audio = audio[:, 0]
text = ds_model.stt(audio, fs)
log("STT result: {}".format(text))
observer.on_next(
Observable.just(
TextResult(
text=text,
context=item.context,
)))
except Exception as e:
log("STT error: {}".format(e))
observer.on_next(
Observable.throw(
TextError(
error=e,
context=item.context,
)))
finally:
shutil.rmtree(temp_dir)
elif type(item) is Initialize:
log("initialize: {}".format(item))
ds_model = setup_model(
item.model, item.alphabet, item.lm, item.trie,
item.features or FeaturesParameters())
else:
log("unknown item: {}".format(item),
level=logging.CRITICAL)
observer.on_error("Unknown item type: {}".format(
type(item)))
def create():
return Observable.throw(ex).time_interval()
def create():
def handler(e):
handler_called[0] = True
return o2
return Observable.throw('ex').catch_exception(handler)
def _subscribe(self, observer: Observer) -> Disposable:
if self.is_disposed:
return Observable.throw(DisposedException('Trying to access an already disposed object')) \
.subscribe(observer)
else:
return self._collectionChanges.subscribe(observer)
def create():
return Observable.throw(ex).debounce(10)
def test_for_each_index_throws(self):
ex = Exception()
xs = Observable.throw(ex)
self.assertRaises(Exception, lambda:xs.to_blocking().for_each(lambda x, i: _raise(ex)))
