def test_error(self): numbers = Subject() windows = Subject() expected_numbers = [] expected_error = None actual_numbers = [] actual_error = None def on_next(i): actual_numbers.append(i) def on_error(e): nonlocal actual_error actual_error = e numbers.pipe(ops.buffer(windows)).subscribe(on_next=on_next, on_error=on_error) numbers.on_next(1) numbers.on_next(2) numbers.on_error(ValueError()) self.assertIsInstance(actual_error, ValueError) self.assertEqual(expected_numbers, actual_numbers)
def test_nominal(self): numbers = Subject() windows = Subject() expected_numbers = [[1, 2], [3, 4, 5]] expected_error = None actual_numbers = [] actual_error = None def on_next(i): actual_numbers.append(i) def on_error(e): nonlocal actual_error actual_error = e numbers.pipe(ops.buffer(windows)).subscribe(on_next=on_next, on_error=on_error) numbers.on_next(1) numbers.on_next(2) windows.on_next(True) numbers.on_next(3) numbers.on_next(4) numbers.on_next(5) windows.on_next(True) self.assertEqual(None, actual_error) self.assertEqual(expected_numbers, actual_numbers)
def _buffer_until_complete(source: rx.Observable): def do_boundary(): boundary.on_next(0) boundary.on_completed() boundary = subject.Subject() source.subscribe(on_completed=do_boundary) return operators.buffer(boundary)(source)
def __init__(self, func): self.values = Subject() self.boundaries = Subject() self.grouped_values = self.values.pipe( op.buffer(self.boundaries) ) self.sub = self.grouped_values.subscribe(func)
def open(self): print("WebSocket opened") self.write_message("connection opened") def _send_response(x): print(x) self.write_message(json.dumps(x)) def _on_error(ex): print(ex) self.subject = Subject() self.subject.pipe(buffer(rx.interval(5.0)), last(), flat_map(self.get_data)).subscribe( on_next=_send_response, on_error=_on_error)
async def api(): @copy_current_websocket_context async def ws_handler(val): await websocket.send(val) obs: Subject = app.output.port_data obs.pipe(ops.buffer(obs.pipe(ops.filter(b"\r".__eq__))), ops.map(lambda i: b"".join(i).decode())).subscribe( on_next=lambda val: asyncio.get_event_loop().create_task( ws_handler(val)), on_error=lambda val: print(f"on_error {val}"), on_completed=print("on_completed"), ) while True: await waiter(1)
scheduler = TestScheduler() ts = 0.1 rx.from_marbles('--(a1)-(b2)---(c3)|', timespan=ts).subscribe(print_value) rx.from_marbles('(a6)---(b5)(c4)|', timespan=ts).subscribe(print_value) time.sleep(2) print('--') # Interval 을 이용한 옵저버블 생성 rx.interval(0.3).pipe(ops.take_until(rx.timer(3))).subscribe(print_value) time.sleep(4) print('--') # 버퍼 print('-- buffer') rx.from_(range(2000)).pipe(ops.buffer( rx.interval(0.001))).subscribe(on_next=lambda buffer: print( '# of items in buffer {}'.format(len(buffer)))) time.sleep(2) print('-- buffer with count') rx.from_(range(10)).pipe(ops.buffer_with_count(3)).subscribe(print_value) print('-- buffer with time') rx.interval(1).pipe( ops.take_until(rx.timer(10)), ops.buffer_with_time(3), ).subscribe(print_value) time.sleep(12) print('--')
"""this operator will collect all the values, from the source observable and emit them at regular intervals once the given boundary condition is satisfied :parameter boundaries: the input observable that will decide when to stop so that the collected values are emitted :return the return value is observable, that will have all the values collected from source observable based and that is time duration is decided by the observable taken. example) """ from rx import of, interval, operators as op import numpy as np of(np.arange(1, 10000000)) \ .pipe( op.map(lambda n: (n * (n + 1)) / 2), # op.filter(lambda x: x % 3 != 0), op.buffer(interval(0.001)) ).subscribe(lambda x: print("the element is {}".format(x)))
def create(): return xs.pipe(ops.buffer(ys))
"edge": edge_subject, "node": node_subject, "graph": graph_subject } local_subscriber = partial(subscriber, subscribe_map) # base_obs = rx.from_(open("streamTest.txt")) base_obs = rx.from_(sys.stdin) c = ConnectableObservable(base_obs, Subject()) dict_delimiter_subject = Subject() ti = time.time() c.pipe(op.filter(lambda line: '}' in line or '{' in line), op.map(lambda line: True)).subscribe(dict_delimiter_subject) c.pipe( op.buffer(dict_delimiter_subject), op.skip(1), op.map(lambda lines: "".join(lines).replace('"', '\"')), op.filter(lambda line: '}' not in line), op.map(lambda line: "{}{}".format(line, "}")), op.map(lambda json_str: json.loads(json_str)), op.map(lambda dic: dict_to_dict_with_set(dic)), # op.take(20), op.group_by(lambda dic: get_dict_type(dic)), ).subscribe(local_subscriber) print("Start stream time: {}".format(str(time.time() - ti))) c.connect() print("Finish time: {}".format(str(time.time() - ti)))
def __init__(self, options: ContextOptions = None): if options is None: options = ContextOptions() self.client = deepkit.client.Client(options) deepkit.globals.last_context = self self.log_lock = Lock() self.defined_metrics = {} self.log_subject = Subject() self.metric_subject = Subject() self.speed_report_subject = Subject() self.shutting_down = False atexit.register(self.shutdown) self.wait_for_connect() self.last_iteration_time = 0 self.last_batch_time = 0 self.job_iteration = 0 self.job_iterations = 0 self.seconds_per_iteration = 0 self.seconds_per_iterations = [] self.debugger_controller = None if deepkit.utils.in_self_execution(): self.job_controller = JobController() self.debugger_controller = JobDebuggerController() def on_connect(connected): if connected: if deepkit.utils.in_self_execution(): asyncio.run_coroutine_threadsafe( self.client.register_controller( 'job/' + self.client.job_id, self.job_controller), self.client.loop) asyncio.run_coroutine_threadsafe( self.client.register_controller( 'job/' + self.client.job_id + '/debugger', self.debugger_controller), self.client.loop) self.client.connected.subscribe(on_connect) def on_metric(data: List): if len(data) == 0: return packed = {} for d in data: if d['id'] not in packed: packed[d['id']] = b'' packed[d['id']] += d['row'] for i, v in packed.items(): self.client.job_action( 'channelData', [i, base64.b64encode(v).decode('utf8')]) self.metric_subject.pipe(buffer(interval(1))).subscribe(on_metric) def on_speed_report(rows): # only save latest value, each second if len(rows) == 0: return self.client.job_action('streamFile', [ '.deepkit/speed.metric', base64.b64encode(rows[-1]).decode('utf8') ]) self.speed_report_subject.pipe(buffer( interval(1))).subscribe(on_speed_report) if deepkit.utils.in_self_execution: # the CLI handled output logging otherwise def on_log(data: List): if len(data) == 0: return packed = '' for d in data: packed += d self.client.job_action('log', ['main_0', packed]) self.log_subject.pipe(buffer(interval(1))).subscribe(on_log) if len(deepkit.globals.last_logs.getvalue()) > 0: self.log_subject.on_next(deepkit.globals.last_logs.getvalue()) if deepkit.utils.in_self_execution: # the CLI handled output logging otherwise p = psutil.Process() def on_hardware_metrics(dummy): net = psutil.net_io_counters() disk = psutil.disk_io_counters() data = struct.pack( '<BHdHHffff', 1, 0, time.time(), int(((p.cpu_percent(interval=None) / 100) / psutil.cpu_count()) * 65535), # stretch to max precision of uint16 int((p.memory_percent() / 100) * 65535), # stretch to max precision of uint16 float(net.bytes_recv), float(net.bytes_sent), float(disk.write_bytes), float(disk.read_bytes), ) self.client.job_action('streamFile', [ '.deepkit/hardware/main_0.hardware', base64.b64encode(data).decode('utf8') ]) interval(1).subscribe(on_hardware_metrics)
rx.range() rx.repeat_value() rx.start() rx.timer() """Mathematical""" op.average() op.concat() op.count() op.max() op.min() op.reduce() op.sum() """Transformation""" op.buffer() op.group_by() op.map() op.scan() # ... """Filtering""" op.debounce() op.distinct() op.filter() op.element_at() op.first() op.ignore_elements() op.last() op.skip() op.skip_last()
import rx import rx.operators as ops from rx.subject import Subject import time import threading numbers = Subject() windows = Subject() numbers.pipe(ops.buffer(windows)).subscribe( on_next=lambda i: print("on_next {}".format(i)), on_error=lambda e: print("on_error: {}".format(e)), on_completed=lambda: print("on_completed")) numbers.on_next(1) numbers.on_next(2) windows.on_next(True) numbers.on_next(3) numbers.on_next(4) numbers.on_next(5) windows.on_next(True)
def __init__(self, config_path: str): deepkit.globals.last_context = self self.log_lock = Lock() self.defined_metrics = {} self.log_subject = Subject() self.metric_subject = Subject() self.speed_report_subject = Subject() self.client = deepkit.client.Client(config_path) self.wait_for_connect() atexit.register(self.shutdown) self.last_iteration_time = 0 self.last_batch_time = 0 self.job_iteration = 0 self.job_iterations = 0 self.seconds_per_iteration = 0 self.seconds_per_iterations = [] self.debugger_controller = None if deepkit.utils.in_self_execution(): self.job_controller = JobController() self.debugger_controller = JobDebuggerController() def on_connect(connected): if connected: if deepkit.utils.in_self_execution(): asyncio.run_coroutine_threadsafe( self.client.register_controller( 'job/' + self.client.job_id, self.job_controller), self.client.loop) asyncio.run_coroutine_threadsafe( self.client.register_controller( 'job/' + self.client.job_id + '/debugger', self.debugger_controller), self.client.loop) self.client.connected.subscribe(on_connect) def on_log(data: List): if len(data) == 0: return packed = '' for d in data: packed += d self.client.job_action('log', ['main_0', packed]) self.log_subject.pipe(buffer(interval(1))).subscribe(on_log) if len(deepkit.globals.last_logs.getvalue()) > 0: self.log_subject.on_next(deepkit.globals.last_logs.getvalue()) def on_metric(data: List): if len(data) == 0: return packed = {} for d in data: if d['id'] not in packed: packed[d['id']] = [] packed[d['id']].append(d['row']) for i, v in packed.items(): self.client.job_action('channelData', [i, v]) self.metric_subject.pipe(buffer(interval(1))).subscribe(on_metric) def on_speed_report(rows): # only save latest value, each second if len(rows) == 0: return self.client.job_action('streamJsonFile', ['.deepkit/speed.csv', [rows[-1]]]) self.speed_report_subject.pipe(buffer( interval(1))).subscribe(on_speed_report) p = psutil.Process() self.client.job_action('streamJsonFile', [ '.deepkit/hardware/main_0.csv', [[ 'time', 'cpu', 'memory', 'network_rx', 'network_tx', 'block_write', 'block_read' ]] ]) def on_hardware_metrics(dummy): net = psutil.net_io_counters() disk = psutil.disk_io_counters() data = [ time.time(), (p.cpu_percent(interval=None) / 100) / psutil.cpu_count(), p.memory_percent() / 100, net.bytes_recv, net.bytes_sent, disk.write_bytes, disk.read_bytes, ] self.client.job_action('streamJsonFile', ['.deepkit/hardware/main_0.csv', [data]]) interval(1).subscribe(on_hardware_metrics)