def test_buffering(self):
self.source.observable().pipe(
buffer_with_count(12),
map(lambda items: len(items)),
buffer_with_count(3),
).subscribe(lambda aantal: assert_that(aantal, contains(12, 12, 10)),
lambda e: fail(e))
def monitor_cpu(npoints):
(lines,) = plt.plot([], [])
plt.xlim(0, npoints)
plt.ylim(0, 100)
cpu_data_window = buffer_with_count(npoints, 1)(cpu_data)
def update_plot(cpu_readings):
lines.set_xdata(np.arange(len(cpu_readings)))
lines.set_ydata(np.array(cpu_readings))
plt.draw()
alertpoints = 4
high_cpu = map(lambda readings: all(r > 20 for r in readings))(
buffer_with_count(alertpoints, 1)(cpu_data)
)
label = plt.text(1, 1, "normal")
def update_warning(is_high):
if is_high:
label.set_text("high")
else:
label.set_text("normal")
high_cpu.subscribe(update_warning)
cpu_data_window.subscribe(update_plot)
plt.show()
def step(self):
last_result = (rx.from_iterable(self.islands).pipe(
ops.subscribe_on(NewThreadScheduler()),
ops.flat_map(lambda island: island.epoch(self.epoch_length).pipe(
ops.last())),
ops.buffer_with_count(len(self.islands)),
).run())
self.migration()
self.update_cost(last_result)
def configure_subscriptions(self):
self.subjects.detection_result.pipe(
operators.buffer_with_count(self.config["group_size"]),
operators.take_until(self._stop),
).subscribe(ErrorToConsoleObserver(self.process_distribution_data))
self.subjects.sample_image_data.pipe(operators.take_until(
self._stop), ).subscribe(
ErrorToConsoleObserver(self.render_sample_image))
def sync_stores(self) -> None:
self.fetch_source().observable().pipe(
rx_filter(lambda event: event.is_valid()),
buffer_with_count(200),
rx_map(self.event_repository.upsert_no_slicing)
).subscribe(
on_next=lambda e: self.logger.info('Upserted %d events for %s', len(e), self.venue.venue_id),
on_error=self._log_error,
)
def _simex_configure_subscription(self, x=None):
self.subjects.image_producer.pipe(
operators.observe_on(self.execution_thread),
operators.map(lambda acquired_image: acquired_image.image
), # pluck the image array
operators.map(lambda im: np.median(im)),
operators.buffer_with_count(self.config["buffer_count"]),
operators.map(lambda medians: np.mean(medians)),
operators.take_until(self._stop)
).subscribe(
ErrorToConsoleObserver(lambda t: self.instance.request_port_update(
OutputPorts.BRIGHTNESS.value, np.asarray(t, dtype=np.float64)).
subscribe(ErrorToConsoleObserver())))
def from_rx(
source: rx.Observable,
batch_size: int = None,
overflow_strategy: OverflowStrategy = None,
is_batched: bool = None,
) -> Flowable:
"""
Wrap a rx.Observable and exposes it as a Flowable, relaying signals in a backpressure-aware manner.
:param source: an rx.observable
:param overflow_strategy: define which batches are ignored once the buffer is full
:param batch_size: determines the number of elements that are sent in a batch
:param is_batched: if set to True, the elements emitted by the source rx.Observable are
either of type List or of type Iterator
"""
if is_batched is True:
batched_source = source
else:
if batch_size is None:
batch_size = 1
batched_source = source.pipe(operators.buffer_with_count(batch_size), )
if isinstance(overflow_strategy, DropOld) or isinstance(
overflow_strategy, ClearBuffer):
return init_flowable(
FromRxEvictingFlowable(
batched_source=batched_source,
overflow_strategy=overflow_strategy,
))
else:
if overflow_strategy is None:
buffer_size = math.inf
elif isinstance(overflow_strategy, BackPressure):
buffer_size = overflow_strategy.buffer_size
else:
raise AssertionError(
'only BackPressure is currently supported as overflow strategy'
)
return init_flowable(
FromRxBufferingFlowable(
batched_source=batched_source,
overflow_strategy=overflow_strategy,
buffer_size=buffer_size,
))
def info_generator(self, fullnames):
with ThreadPoolExecutor() as executor:
queue = Queue(FETCHES)
executor.submit(lambda: from_iterable(fullnames).pipe(
buffer_with_count(100),
map(lambda chunk: {"id": ",".join(chunk)}),
map(lambda params: lambda: self.get(API_PATH["info"],
params=params)),
map(lambda task: executor.submit(task)),
map(lambda future: queue.put(future)),
finally_action(lambda: queue.put(None))).subscribe())
while True:
response = queue.get()
if not response:
break
yield from response.result()
def configure_subscriptions(self, connected):
if connected:
self.subjects.image_producer.pipe(
operators.observe_on(self.feed_scheduler),
operators.buffer_with_count(self.batch_size),
bp_operator(BackPressure.DROP, 5),
operators.take_until(self._stop),
).subscribe(ErrorToConsoleObserver(self.feed_image))
self.inference_comm.back_pressure_chan.pipe(
operators.subscribe_on(self.process_scheduler),
operators.take_until(self._stop)).subscribe(
ErrorToConsoleObserver(self.update_back_pressure_status))
# report error when image source is still producing when back pressuring
self.subjects.analyzer_back_pressure_detected.pipe(
operators.combine_latest(self.subjects.image_producer),
operators.filter(lambda x: x[0]),
operators.throttle_first(1.0),
operators.take_until(self._stop),
).subscribe(
ErrorToConsoleObserver(lambda x: self.logger.warning(
"Image is feeding while back-pressure is detected. Please slow down the FPS"
)))
self.inference_comm.result_chan.pipe(
operators.subscribe_on(self.process_scheduler),
operators.take_until(self._stop)).subscribe(
ErrorToConsoleObserver(self.result_processing))
self.inference_comm.error_chan.pipe(
operators.take_until(self._stop)).subscribe(
ErrorToConsoleObserver(lambda err: self.logger.error(err)))
self.inference_comm.connection_chan.pipe(
operators.take_until(self._stop)).subscribe(
ErrorToConsoleObserver(lambda connected: self.logger.info(
"GRPC Remote analyzer connected" if connected else
"GRPC Remote analyzer disconnected")))
self.inference_comm.stats_chan.take_until(self._stop).subscribe(
ErrorToConsoleObserver(lambda x: self.logger.info(
f"Processed {x.frame} frames. Average {x.processTime / x.frame} secs"
)))
def start(self):
# report more image when back pressure
self.subjects.image_producer.pipe(
operators.observe_on(self.scheduler),
operators.combine_latest(
self.subjects.analyzer_back_pressure_detected),
operators.filter(
lambda x: x[1]), # only operate when back pressure
operators.buffer_with_time(1.0), # in 1 sec
operators.filter(lambda x: len(x) > 3), # more than 3 emission
operators.throttle_first(3.0), # report every 3 seconds
operators.take_until(self._stop),
).subscribe(self.report_back_pressure_emission)
self.subjects.image_producer.pipe(
operators.observe_on(
self.scheduler), # prevent blocking the upstream subject
operators.filter(self.back_pressure_barrier),
operators.buffer_with_count(5),
bp_drop_report_full(self.subjects.analyzer_back_pressure_detected,
3, 1),
operators.take_until(self._stop),
).subscribe(ErrorToConsoleObserver(self.produce_fake_analyze_data))
super(TestAnalyzer, self).start()
def create():
return xs.pipe(ops.buffer_with_count(3, 2),
ops.map(lambda x: str(x)))
scheduler = TestScheduler()
ts = 0.1
rx.from_marbles('--(a1)-(b2)---(c3)|', timespan=ts).subscribe(print_value)
rx.from_marbles('(a6)---(b5)(c4)|', timespan=ts).subscribe(print_value)
time.sleep(2)
print('--')
# Interval 을 이용한 옵저버블 생성
rx.interval(0.3).pipe(ops.take_until(rx.timer(3))).subscribe(print_value)
time.sleep(4)
print('--')
# 버퍼
print('-- buffer')
rx.from_(range(2000)).pipe(ops.buffer(
rx.interval(0.001))).subscribe(on_next=lambda buffer: print(
'# of items in buffer {}'.format(len(buffer))))
time.sleep(2)
print('-- buffer with count')
rx.from_(range(10)).pipe(ops.buffer_with_count(3)).subscribe(print_value)
print('-- buffer with time')
rx.interval(1).pipe(
ops.take_until(rx.timer(10)),
ops.buffer_with_time(3),
).subscribe(print_value)
time.sleep(12)
print('--')
def create():
return xs.pipe(ops.buffer_with_count(2))
def create():
return xs.pipe(ops.buffer_with_count(3, 2), ops.map(lambda x: str(x)))
import rx
import rx.operators as ops
from rx.subject import Subject
import time
import threading
numbers = rx.from_([1, 2, 3, 4, 5, 6])
numbers.pipe(ops.buffer_with_count(3)).subscribe(
on_next=lambda i: print("on_next {}".format(i)),
on_error=lambda e: print("on_error: {}".format(e)),
on_completed=lambda: print("on_completed"))
conn.get_database("planning").get_collection("edges2"))
save_nodes_in_db = partial(
save_many_items,
conn.get_database("planning").get_collection("nodes2"))
graph_type_map = {"source": "edge", "node_name": "node", "name": "graph"}
get_dict_type = partial(get_obj_type_from_type_map, graph_type_map)
edge_subject, node_subject, graph_subject = Subject(), Subject(), Subject()
processed_edges = edge_subject.pipe(
op.filter(lambda edge_dic: not exists(edge_dic)),
op.group_by(lambda dic: "".join(
[str(v) for k, v in dic.items() if k not in ['level', 'type']])),
op.map(lambda o: general_edge_grouper(o)), op.merge_all(),
op.buffer_with_count(1000),
op.map(lambda dict_list: save_edges_in_db(dict_list)),
op.buffer_with_count(5), op.map(lambda futures: perform_futures(futures)),
op.map(lambda results: [r.inserted_ids for r in results])).subscribe(dumb)
processed_nodes = node_subject.pipe(
op.filter(lambda node_dic: not exists(node_dic)),
op.group_by(lambda dic: "".join(
[str(v) for k, v in dic.items() if k not in ['level']])),
op.map(lambda o: general_node_grouper(o)), op.merge_all(),
op.buffer_with_count(5000),
op.map(lambda dict_list: save_nodes_in_db(dict_list)),
op.buffer_with_count(5), op.map(lambda futures: perform_futures(futures)),
op.map(lambda results: [r.inserted_ids for r in results])).subscribe(dumb)
graph_subject.pipe(op.filter(lambda graph_dic: not exists(graph_dic)),
"""
retries = WritesRetry(total=3, backoff_factor=1, exponential_base=2)
client = InfluxDBClient(url='http://localhost:8086',
token='my-token',
org='my-org',
retries=retries)
"""
Use synchronous version of WriteApi to strongly depends on result of write
"""
write_api = client.write_api(write_options=SYNCHRONOUS)
"""
Prepare batches from generator
"""
batches = rx \
.from_iterable(csv_to_generator('vix-daily.csv')) \
.pipe(ops.buffer_with_count(500))
def write_batch(batch):
"""
Synchronous write
"""
print(f'Writing... {len(batch)}')
write_api.write(bucket='my-bucket', record=batch)
"""
Write batches
"""
batches.subscribe(on_next=lambda batch: write_batch(batch),
on_error=lambda ex: print(f'Unexpected error: {ex}'),
def create():
return xs.pipe(ops.buffer_with_count(5))
import rx
import rx.operators as op
import threading
# 与えられたものをそのまま
rx.just(3).subscribe(print)
rx.from_list([1, 2, 3, 4]).subscribe(print)
rx.from_iterable([1, 2, 3, 4]).subscribe(print)
# bufferはたまってから処理する
rx.from_iterable(range(10)).pipe(op.map(lambda x: x * 2),
op.buffer_with_count(3)).subscribe(print)
rx.from_iterable(range(10)).pipe(
op.map(lambda x: x * 2),
op.buffer_with_count(3),
# ここではリストになっている
op.map(len)).subscribe(print)
# windowもたまってから処理するがlistではなくObservableになる
rx.from_iterable(range(10)).pipe(
op.map(lambda x: x * 2),
op.window_with_count(3),
).subscribe(print)
# observableの中身を1つ1つ処理したい場合はflatmapが使える
rx.from_iterable(range(10)).pipe(
op.map(lambda x: x * 2),
op.window_with_count(3),
# ただしこの時点でxはObservable
# 複数のObservableを1つにまとめる
