def _window_to_group(self, value):
return value.pipe(
ops.to_iterable(),
ops.map(
lambda x: rx.from_iterable(x).pipe(ops.group_by(
_group_by), ops.map(_group_to_batch), ops.merge_all())),
ops.merge_all())
def _buffer_with_time(timespan: typing.RelativeTime, timeshift: typing.RelativeTime = None,
scheduler: typing.Scheduler = None) -> Callable[[Observable], Observable]:
if not timeshift:
timeshift = timespan
return pipe(
ops.window_with_time(timespan, timeshift, scheduler),
ops.flat_map(lambda x: x.pipe(ops.to_iterable()))
)
def _window_to_group(value):
return value.pipe(
ops.to_iterable(),
ops.map(lambda x: rx.from_iterable(x).pipe(
# Group window by 'organization', 'bucket' and 'precision'
ops.group_by(_group_by),
# Create batch (concatenation line protocols by \n)
ops.map(_group_to_batch),
ops.merge_all())),
ops.merge_all())
def _buffer_with_time(
timespan: typing.RelativeTime,
timeshift: Optional[typing.RelativeTime] = None,
scheduler: Optional[typing.Scheduler] = None
) -> Callable[[Observable], Observable]:
if not timeshift:
timeshift = timespan
return pipe(ops.window_with_time(timespan, timeshift, scheduler),
ops.flat_map(lambda x: x.pipe(ops.to_iterable())))
def demo_group_by():
'''tuple unpacking'''
a = rx.of([
{'id':1 , 'name': 'aaa'},
{'id':2 , 'name': 'bbb'},
{'id':1 , 'name': 'aaa'},
{'id':1 , 'name': 'aaa'},
{'id':2 , 'name': 'aaa'},
])
a.pipe(
ops.group_by(lambda x: x['id'], lambda x: x['name'], subject_mapper = rx.subject.ReplaySubject()),
ops.to_iterable(),
).subscribe(print)
def test_create_and_run_all_supported_algorithms(self):
sys = ActorSystem("multiprocTCPBase", logDefs=log_helper.EVOGIL_LOG_CONFIG)
test_cases = run_config.algorithms
for test_case in test_cases:
with self.subTest(algorithm=test_case):
algo_factory, _ = prepare(test_case, "ZDT1")
algorithm = algo_factory()
simple_simulation = StepsRun(1)
result = list(
simple_simulation.create_job(algorithm)
.pipe(ops.subscribe_on(NewThreadScheduler()), ops.to_iterable())
.run()
)
self.assertEqual(1, len(result))
self.assertIsInstance(result[0], ProgressMessage)
sys.shutdown()
def _buffer(buffer_openings=None,
buffer_closing_mapper=None) -> Callable[[Observable], Observable]:
"""Projects each element of an observable sequence into zero or more
buffers.
Args:
buffer_openings -- Observable sequence whose elements denote the
creation of windows.
buffer_closing_mapper -- [optional] A function invoked to define
the closing of each produced window. If a closing mapper
function is specified for the first parameter, this parameter is
ignored.
Returns:
A function that takes an observable source and retuerns an
observable sequence of windows.
"""
return pipe(ops.window(buffer_openings, buffer_closing_mapper),
ops.flat_map(pipe(ops.to_iterable(), ops.map(list))))
def _buffer(buffer_openings=None, buffer_closing_mapper=None) -> Callable[[Observable], Observable]:
"""Projects each element of an observable sequence into zero or more
buffers.
Args:
buffer_openings -- Observable sequence whose elements denote the
creation of windows.
buffer_closing_mapper -- [optional] A function invoked to define
the closing of each produced window. If a closing mapper
function is specified for the first parameter, this parameter is
ignored.
Returns:
A function that takes an observable source and retuerns an
observable sequence of windows.
"""
return pipe(
ops.window(buffer_openings, buffer_closing_mapper),
ops.flat_map(pipe(ops.to_iterable(), ops.map(list)))
)
def __init__(
self,
influxdb_client,
write_options: WriteOptions = WriteOptions(),
point_settings: PointSettings = PointSettings()
) -> None:
self._influxdb_client = influxdb_client
self._write_service = WriteService(influxdb_client.api_client)
self._write_options = write_options
self._point_settings = point_settings
if influxdb_client.default_tags:
for key, value in influxdb_client.default_tags.items():
self._point_settings.add_default_tag(key, value)
if self._write_options.write_type is WriteType.batching:
# Define Subject that listen incoming data and produces writes into InfluxDB
self._subject = Subject()
self._disposable = self._subject.pipe(
# Split incoming data to windows by batch_size or flush_interval
ops.window_with_time_or_count(count=write_options.batch_size,
timespan=timedelta(milliseconds=write_options.flush_interval)),
# Map window into groups defined by 'organization', 'bucket' and 'precision'
ops.flat_map(lambda window: window.pipe(
# Group window by 'organization', 'bucket' and 'precision'
ops.group_by(lambda batch_item: batch_item.key),
# Create batch (concatenation line protocols by \n)
ops.map(lambda group: group.pipe(
ops.to_iterable(),
ops.map(lambda xs: _BatchItem(key=group.key, data=_body_reduce(xs), size=len(xs))))),
ops.merge_all())),
# Write data into InfluxDB (possibility to retry if its fail)
ops.map(mapper=lambda batch: self._retryable(data=batch, delay=self._jitter_delay())), #
ops.merge_all())\
.subscribe(self._on_next, self._on_error, self._on_complete)
else:
self._subject = None
self._disposable = None
def mapper(value):
return value.pipe(ops.to_iterable(), ops.map(list))
import rx
import rx.operators as ops
"""
Use a dictionnary to convert elements declared in the marbles diagram to
the specified values.
"""
lookup0 = {'a': 1, 'b': 3, 'c': 5}
lookup1 = {'x': 2, 'y': 4, 'z': 6}
source0 = rx.cold('a---b----c----|', timespan=0.01, lookup=lookup0)
source1 = rx.cold('---x---y---z--|', timespan=0.01, lookup=lookup1)
observable = rx.merge(source0, source1).pipe(ops.to_iterable())
elements = observable.run()
print('received {}'.format(list(elements)))
def create():
return xs.pipe(
ops.to_iterable(),
ops.map(list),
)
def _buffer(boundaries: Observable) -> Callable[[Observable], Observable]:
return pipe(ops.window(boundaries),
ops.flat_map(pipe(ops.to_iterable(), ops.map(list))))
def _buffer_toggle(
openings: Observable, closing_mapper: Callable[[Any], Observable]
) -> Callable[[Observable], Observable]:
return pipe(ops.window_toggle(openings, closing_mapper),
ops.flat_map(pipe(ops.to_iterable(), ops.map(list))))
def create():
return xs.pipe(ops.to_iterable())
def action1(scheduler, state):
xs[0] = rx.from_iterable(["alpha", "apple", "beta", "bat", "gamma"]) \
.pipe(ops.group_by(lambda s: s[0]),
ops.map(lambda xs: xs.pipe(ops.to_iterable(), ops.map(list))),
ops.merge_all(),
)
def _buffer_with_time_or_count(timespan, count, scheduler = None) -> Callable[[Observable], Observable]:
return pipe(
ops.window_with_time_or_count(timespan, count, scheduler),
ops.flat_map(lambda x: x.pipe(ops.to_iterable()))
)
def test_run_range_to_iterable(self):
result = rx.range(42).pipe(ops.to_iterable()).run()
assert list(result) == list(range(42))
def test_run_range_to_iterable(self):
result = rx.range(42).pipe(ops.to_iterable()).run()
assert list(result) == list(range(42))
def _group_to_batch(group: GroupedObservable):
return group.pipe(ops.to_iterable(), ops.map(list),
ops.map(_create_batch(group)))
import pandas as pd
from context import iter_to_observable, simple_print, do_reactive
from rx import operators as op
my_serie = pd.Series([1, 2, 3])
my_obs = iter_to_observable(my_serie)
my_obs.subscribe(simple_print)
# test list to stream to list
my_obs_from_list = iter_to_observable([4, 5, 6])
my_obs_from_list.\
pipe(
op.map(lambda x: x*2),
op.to_iterable()).\
subscribe(simple_print)
# test DF
base_dict = [{'id': 1, 'name': 'foo'}, {'id': 2, 'name': 'bar'}]
my_df = pd.DataFrame(base_dict)
my_obs_df = iter_to_observable(my_df.iterrows())
# print_iter = unpack_iterrow_and_apply(print_data_iterrow)
# print_id = unpack_iterrow_and_apply(lambda i, d: print(d.get('name')))
my_obs_df.pipe(op.map(lambda x: x[1].get('name'))).subscribe(simple_print)
do_reactive(my_df.iterrows(), simple_print,
[op.map(lambda x: x[1].get('id') * 2)])
def _buffer_when(
closing_mapper: Callable[[], Observable]
) -> Callable[[Observable], Observable]:
return pipe(ops.window_when(closing_mapper),
ops.flat_map(pipe(ops.to_iterable(), ops.map(list))))
def transform(self):
return rx.pipe(
super().transform(),
ops.to_iterable(),
ops.map(lambda xs: [y[1] for y in sorted(xs, key=lambda x: x[0])]),
)
def mapper(value):
return value.pipe(ops.to_iterable(), ops.map(list))
def create():
return xs.pipe(
ops.to_iterable(),
ops.map(list),
)
def action1(scheduler, state):
xs[0] = rx.from_iterable(["alpha", "apple", "beta", "bat", "gamma"]) \
.pipe(ops.group_by(lambda s: s[0]),
ops.map(lambda xs: xs.pipe(ops.to_iterable(), ops.map(list))),
ops.merge_all(),
)
def create():
return xs.pipe(ops.to_iterable())
