def model_publisher(scheduler, sources):
file_source = sources.file.response.pipe(ops.share())
# kafka driver bootstrap. fixme
kafka_source = sources.kafka.response.pipe(
ops.do_action(print),
ops.replay(),
ops.ref_count(),
)
kafka_source.subscribe()
config, config_read_request, http_request = read_config_from_args(
sources.argv.argv,
file_source,
sources.http.response,
scheduler=scheduler)
config = config.pipe(ops.first())
kafka_request = config.pipe(ops.map(lambda c: create_model_topics(c)), )
return ModelPublisherSink(
file=file.Sink(request=rx.merge(config_read_request)),
http=http.Sink(request=http_request),
kafka=kafka.Sink(request=kafka_request),
)
def partition(source: Observable) -> List[Observable]:
"""The partially applied `partition` operator.
Returns two observables which partition the observations of the
source by the given function. The first will trigger
observations for those values for which the predicate returns
true. The second will trigger observations for those values
where the predicate returns false. The predicate is executed
once for each subscribed observer. Both also propagate all
error observations arising from the source and each completes
when the source completes.
Args:
source: Source obserable to partition.
Returns:
A list of observables. The first triggers when the
predicate returns True, and the second triggers when the
predicate returns False.
"""
published = source.pipe(ops.publish(), ops.ref_count())
return [
published.pipe(ops.filter(predicate)),
published.pipe(ops.filter(lambda x: not predicate(x)))
]
def partition(source: Observable) -> List[Observable]:
"""The partially applied `partition` operator.
Returns two observables which partition the observations of the
source by the given function. The first will trigger
observations for those values for which the predicate returns
true. The second will trigger observations for those values
where the predicate returns false. The predicate is executed
once for each subscribed observer. Both also propagate all
error observations arising from the source and each completes
when the source completes.
Args:
source: Source obserable to partition.
Returns:
A list of observables. The first triggers when the
predicate returns True, and the second triggers when the
predicate returns False.
"""
published = source.pipe(
ops.publish(),
ops.ref_count()
)
return [
published.pipe(ops.filter(predicate)),
published.pipe(ops.filter(lambda x: not predicate(x)))
]
def _parse_config(config_data):
''' takes a stream with the content of the configuration file as input
and returns a (hot) stream of arguments .
'''
return config_data.pipe(
ops.map(lambda i: yaml.load(i, Loader=yaml.FullLoader)),
ops.replay(buffer_size=1),
ops.ref_count(),
)
def to_hot_observable(
# subject_factory: Callable[[Optional[rx.typing.Scheduler]], rx.typing.Subject[T2, T3]],
base_stream: rx.typing.Observable[T1],
) -> rx.typing.Observable[T1]:
return cast(rx.Observable, base_stream)\
.pipe(
ops.publish(),
ops.ref_count(),
)
def _share() -> Callable[[Observable], Observable]:
"""Share a single subscription among multple observers.
Returns a new Observable that multicasts (shares) the original
Observable. As long as there is at least one Subscriber this
Observable will be subscribed and emitting data. When all
subscribers have unsubscribed it will unsubscribe from the source
Observable.
This is an alias for a composed publish() and ref_count().
"""
return pipe(_publish(), ops.ref_count())
def _share() -> Callable[[Observable], Observable]:
"""Share a single subscription among multple observers.
Returns a new Observable that multicasts (shares) the original
Observable. As long as there is at least one Subscriber this
Observable will be subscribed and emitting data. When all
subscribers have unsubscribed it will unsubscribe from the source
Observable.
This is an alias for a composed publish() and ref_count().
"""
return pipe(_publish(), ops.ref_count())
def _init_signals(self, incoming_msgs):
def _is_signal(msg: Message) -> bool:
return msg.type == DBUS_CONSTANTS.MESSAGE_TYPE_SIGNAL
def _parse_signal(msg: Message) -> UnresolvedSignal:
id_ = msg.sender
service_names = self._get_service_names_of_id(id_)
return UnresolvedSignal(service_names, id_,
str(msg.path), msg.interface, msg.member,
list(msg.objects))
return incoming_msgs.pipe(op.filter(_is_signal), op.map(_parse_signal),
op.publish(), op.ref_count())
def __init__(self, scheduler=None):
self._observerable = rx.interval(
ObserveConfig.interval,
scheduler).pipe(ops.map(lambda dummy: get_merge_requests()),
ops.retry(), ops.publish(), ops.ref_count())
self._ready_to_merge = self._observerable.pipe(
ops.map(lambda requests: next((request for request in requests if
is_ready_to_merge(request)), None)),
ops.start_with(None), ops.distinct_until_changed())
self._ready_to_merge.subscribe(lambda ready_to_merge: logging.info(
'Ready to merge: ' + str(ready_to_merge)))
voted_merge_requests = self._observerable.pipe(
ops.map(_to_voted_merge_requests_set))
self._new_votes_merge_requests = voted_merge_requests.pipe(
ops.skip(1), ops.zip(voted_merge_requests),
ops.map(lambda zipped: zipped[0] - zipped[1]), ops.filter(len),
ops.map(_to_merge_requests))
self._new_votes_merge_requests.pipe(
ops.map(lambda diff_set:
[merge_request.get_iid() for merge_request in diff_set])
).subscribe(
lambda ids: logging.info(f'New votes for merge requests: {ids}'))
awards = self._new_votes_merge_requests.pipe(ops.map(_to_awards_set),
ops.publish(),
ops.ref_count(),
ops.start_with(set()))
self._new_awards = awards.pipe(
ops.skip(1), ops.zip(awards),
ops.map(lambda zipped: zipped[0] - zipped[1]), ops.filter(len),
ops.flat_map(lambda diff_set: rx.from_iterable(diff_set)),
ops.map(lambda award_key: award_key.award))
self._new_awards.subscribe(
lambda new_award: logging.info('New award: ' + str(new_award)))
def test_ref_count():
print('rx.interval(1).pipe(ops.publish(), ops.ref_count())')
obs = rx.interval(1).pipe(ops.publish(), ops.ref_count())
input('Press Enter to subscribe...\n')
unsub = obs.subscribe(lambda x: print(x))
input('Press Enter to dispose...\n')
unsub.dispose()
input('Press Enter to subscribe again...\n')
unsub = obs.subscribe(lambda x: print(x))
input('Press Enter to finish...\n')
unsub.dispose()
def select(selector: Mapper[T1, T2]
) -> Callable[[Observable], Observable]:
""" Reactive operator that applies a selector
and shares the result across multiple subscribers
Args:
selector: the selector function
Returns:
The reactive operator
"""
return pipe(
op.map(selector),
op.distinct_until_changed(comparer=is_),
op.multicast(subject=ReplaySubject(1)),
op.ref_count(),
)
def makinage(aio_scheduler, sources):
def on_error(e):
raise e
config, read_request, http_request = read_config_from_args(
sources.argv.argv,
sources.file.response,
sources.http.response,
scheduler=aio_scheduler
)
first_config = rx.concat(config.pipe(ops.take(1),), rx.never())
kafka_source = sources.kafka.response.pipe(
trace_observable("kafka source1"),
ops.replay(),
ops.ref_count(),
trace_observable("kafka source2"),
)
kafka_source.subscribe(on_error=on_error)
kafka_request = first_config.pipe(
ops.flat_map(lambda i: create_operators(
i, config,
kafka_source,
sources.kafka.feedback.pipe(ops.share()),
)),
ops.subscribe_on(aio_scheduler),
trace_observable("makinage"),
)
'''
config.pipe(ops.subscribe_on(aio_scheduler)).subscribe(
on_next=print,
on_error=print,
)
'''
return MakiNageSink(
file=file.Sink(request=read_request),
http=http.Sink(request=http_request),
kafka=kafka.Sink(request=kafka_request),
)
def test_ref_count_notconnected(self):
disconnected = [False]
count = [0]
def factory(scheduler):
count[0] += 1
def create(obs, scheduler=None):
def func():
disconnected[0] = True
return func
return rx.create(create)
xs = rx.defer(factory)
subject = MySubject()
conn = ConnectableObservable(xs, subject)
refd = conn.pipe(ops.ref_count())
dis1 = refd.subscribe()
self.assertEqual(1, count[0])
self.assertEqual(1, subject.subscribe_count)
assert not disconnected[0]
dis2 = refd.subscribe()
self.assertEqual(1, count[0])
self.assertEqual(2, subject.subscribe_count)
assert not disconnected[0]
dis1.dispose()
assert not disconnected[0]
dis2.dispose()
assert disconnected[0]
disconnected[0] = False
dis3 = refd.subscribe()
self.assertEqual(2, count[0])
self.assertEqual(3, subject.subscribe_count)
assert not disconnected[0]
dis3.dispose()
assert disconnected[0]
def test_ref_count_notconnected(self):
disconnected = [False]
count = [0]
def factory(scheduler):
count[0] += 1
def create(obs, scheduler=None):
def func():
disconnected[0] = True
return func
return rx.create(create)
xs = rx.defer(factory)
subject = MySubject()
conn = ConnectableObservable(xs, subject)
refd = conn.pipe(ops.ref_count())
dis1 = refd.subscribe()
self.assertEqual(1, count[0])
self.assertEqual(1, subject.subscribe_count)
assert not disconnected[0]
dis2 = refd.subscribe()
self.assertEqual(1, count[0])
self.assertEqual(2, subject.subscribe_count)
assert not disconnected[0]
dis1.dispose()
assert not disconnected[0]
dis2.dispose()
assert disconnected[0]
disconnected[0] = False
dis3 = refd.subscribe()
self.assertEqual(2, count[0])
self.assertEqual(3, subject.subscribe_count)
assert not disconnected[0]
dis3.dispose()
assert disconnected[0]
def _init_bus_item_calls(self, method_calls):
def bus_item(msg: Message):
return msg.interface == 'com.victronenergy.BusItem'
bus_item_calls = method_calls.pipe(op.filter(bus_item), op.publish(),
op.ref_count())
def is_set_value(msg: Message):
return msg.member == 'SetValue'
def is_get_value(msg: Message):
return msg.member == 'GetValue'
def is_get_text(msg: Message):
return msg.member == 'GetText'
# TODO: simplify, mb merge with getvalue
def on_get_text_called(msg: Message):
object_path = msg.path
reply = None
if object_path == '/':
# tree export
prop_dict = {
path: prop.text
for ((_service_name, path),
prop) in self._ve_properties.items()
}
reply = msg.new_method_return()
reply.append_objects('v', ('a{ss}', prop_dict))
print(prop_dict)
else:
text = next((convert_python_value_to_dbus(prop.text)
for ((_service_name, path),
prop) in self._ve_properties.items()
if path == object_path), None)
if text is not None:
reply = msg.new_method_return()
reply.append_objects(*text)
else:
reply = msg.new_error(DBUS_CONSTANTS.ERROR_UNKNOWN_OBJECT,
object_path)
self._connection.send(reply)
# TODO: simplify, mb merge with gettext
def on_get_value_called(msg: Message):
object_path = msg.path
reply = None
if object_path == '/':
# tree export
prop_dict = {
path: convert_python_value_to_dbus(prop.value)
for ((_service_name, path),
prop) in self._ve_properties.items()
}
reply = msg.new_method_return()
reply.append_objects('v', ('a{sv}', prop_dict))
print(prop_dict)
else:
value = next((convert_python_value_to_dbus(prop.value)
for ((_service_name, path),
prop) in self._ve_properties.items()
if path == object_path), None)
if value is not None:
reply = msg.new_method_return()
reply.append_objects(*value)
else:
reply = msg.new_error(DBUS_CONSTANTS.ERROR_UNKNOWN_OBJECT,
object_path)
self._connection.send(reply)
def on_set_value_called(msg: Message):
object_path = msg.path
reply = None
prop = next((prop for ((_service_name, path),
prop) in self._ve_properties.items()
if path == object_path), None)
if prop is None:
reply = msg.new_error(DBUS_CONSTANTS.ERROR_UNKNOWN_OBJECT,
object_path)
else:
# noinspection PyBroadException,PyBroadException
try:
new_value = list(
msg.objects)[0][1] # TODO: respect data type
if prop.accept_value_change(new_value):
reply = msg.new_method_return()
reply.append_objects('i', 0)
def update():
self.publish_ve_property(
prop.service_name,
prop.object_path,
new_value,
str(new_value
), # TODO: render text of new value
accept_change=prop.accept_value_change)
self.schedule(update) # do this after we replied
else:
reply = msg.new_method_return()
reply.append_objects('i', 2) # TODO: constants?
#reply = msg.new_error(DBUS_CONSTANTS.ERROR_INVALID_ARGS, object_path)
except Exception as e:
reply = msg.new_error(DBUS_CONSTANTS.ERROR_FAILED, str(e))
self._connection.send(reply)
s_set_value = bus_item_calls.pipe(op.filter(is_set_value)).subscribe(
on_set_value_called, on_error=self._on_error)
s_get_value = bus_item_calls.pipe(op.filter(is_get_value)).subscribe(
on_get_value_called, on_error=self._on_error)
s_get_text = bus_item_calls.pipe(op.filter(is_get_text)).subscribe(
on_get_text_called, on_error=self._on_error)
self._disposables.append(s_set_value)
self._disposables.append(s_get_value)
self._disposables.append(s_get_text)
import rx
import rx.operators as ops
numbers = rx.from_([1, 2, 3])
pub_numbers = numbers.pipe(ops.publish(), ops.ref_count())
pub_numbers.subscribe(on_next=lambda i: print("item: {}".format(i)),
on_error=lambda e: print("error: {}".format(e)),
on_completed=lambda: print("completed"))
pub_numbers.subscribe(on_next=lambda i: print("item: {}".format(i)),
on_error=lambda e: print("error: {}".format(e)),
on_completed=lambda: print("completed"))
def create():
return conn.pipe(ops.ref_count())
from rx import interval, operators as ops
import time
source = interval(1).pipe(ops.publish(), ops.ref_count())
source.subscribe(lambda s: print("Subscriber 1: {0}".format(s)))
# sleep 5 seconds, then add another subscriber
time.sleep(5)
source.subscribe(lambda s: print("Subscriber 2: {0}".format(s)))
input("Press any key to exit\n")
#emit modified date every second, if the date has changed since last time.
#use a connectable observable so that multiple downstreams
#all use the same file polling.
file_changed_observable = file_check_interval_observable.pipe(
#Uncomment this to see what happens if file takes 10 seconds to load.
#ops.map(lambda i: intense_calculation(myinput_path, i)),
ops.map(lambda i: get_modified_date(myinput_path)),
ops.distinct_until_changed(),
#Without this each subscriber would create a new stream
#So we'd have one modification date check for the debug statement below
#and one for the debug, which is unnecessary.
#Replay ensures late subscribers
#will read the file.
ops.replay(buffer_size=1),
ops.ref_count()
)
#Debug subscription to show when modified date gets printed out.
file_changed_observable.subscribe(
on_next = lambda i: print("Got date modified - {0}".format(i)),
on_error = lambda e: print("Error : {0}".format(e)),
on_completed = lambda: print("Job Done!"),
)
#sleep 15 seconds, to test if a late subscriber will get an emission before a file change
#(it should).
#time.sleep(15)
#When a emission is produced upstream, read contents of the new file.
file_changed_observable.subscribe(lambda ignore: read_file_contents(myinput_path))
op.catch() op.retry() """Utility""" op.delay() op.materialize() op.time_interval() op.timeout() op.timestamp() """Conditional and Boolean""" op.all() op.contains() op.default_if_empty() op.sequence_equal() op.skip_until() op.skip_while() op.take_until() op.take_while() """Connectable""" op.publish() op.ref_count() op.replay() """Combining""" op.combine_latest() op.merge() op.start_with() op.zip()
def create():
return conn.pipe(ops.ref_count())
import rx
from rx import operators as ops
import time
source = rx.interval(1.0).pipe(ops.publish(), ops.ref_count())
source.subscribe(lambda s: print("Subscriber 1: {0}".format(s)))
# sleep 5 seconds, then add another subscriber
time.sleep(5)
source.subscribe(lambda s: print("Subscriber 2: {0}".format(s)))
input("Press any key to exit\n")
def _init_method_calls(incoming_msgs):
def _is_method_call(msg: Message) -> bool:
return msg.type == DBUS_CONSTANTS.MESSAGE_TYPE_METHOD_CALL
return incoming_msgs.pipe(op.filter(_is_method_call), op.publish(),
op.ref_count())
