def _window_to_group(self, value): return value.pipe( ops.to_iterable(), ops.map( lambda x: rx.from_iterable(x).pipe(ops.group_by( _group_by), ops.map(_group_to_batch), ops.merge_all())), ops.merge_all())
def __init__(self, concurrency_per_group, delay_seconds=0, description=None): self._queue = Subject() self._description = description self.request_scheduler = ThreadPoolScheduler(concurrency_per_group) producer_scheduler = ThreadPoolScheduler(concurrency_per_group) def on_next(result): output = result['output'] output.on_next({ 'value': result.get('value'), 'completed': result.get('completed') }) self._subscription = self._queue.pipe( observe_on(producer_scheduler), group_by(lambda r: r['group']), flat_map(lambda concurrency_group: concurrency_group.pipe( map(lambda r: r['work']), delay(delay_seconds), merge(max_concurrent=concurrency_per_group), merge_all(), )), take_until_disposed()).subscribe( on_next=on_next, on_error=lambda error: logging.exception( 'Error in {} request stream'.format(self)), scheduler=producer_scheduler)
def rmux_client(sources): response = sources.tcp_client.response.pipe(ops.share()) tcp_connect = rx.just(tcp_client.Connect( host='127.0.0.1', port='8080' )) create_observable = response.pipe( ops.flat_map(lambda connection: rx.just({'what': 'subscribe', 'id':42, 'name': '1234'}).pipe( ops.map(lambda i: json.dumps(i)), frame, ops.map(lambda j: tcp_client.Write(id=connection.id, data=j.encode())) )) ) console = response.pipe( ops.flat_map(lambda connection: connection.observable.pipe( ops.map(lambda i: i.data.decode('utf-8')), unframe, ops.map(lambda i: json.loads(i)), ops.group_by(lambda i: i['id']), ops.flat_map(lambda subscription: subscription.pipe( ops.map(notification), ops.dematerialize(), )) )), ops.map(lambda i: "item: {}\n".format(i)) ) tcp_sink = rx.merge(tcp_connect, create_observable) return Sink( tcp_client=tcp_client.Sink(request=tcp_sink), stdout=stdout.Sink(data=console), )
def word_counter(file_name): # count words using `group_by()` # tuple the word with the count return words_from_file(file_name).pipe( ops.group_by(lambda word: word), ops.flat_map(lambda grp: grp.pipe(ops.count(), ops.map(lambda ct: (grp.key, ct)))))
def action1(scheduler, state): c["outer"] = xs.pipe( ops.group_by( lambda x: x.lower().strip(), lambda x: x[::-1], )) return c["outer"]
def action1(scheduler, state): c["outer"] = xs.pipe( ops.group_by( lambda x: x.lower().strip(), lambda x: x[::-1], )) return c["outer"]
def factory(): def key_mapper(x): key_invoked[0] += 1 return x.lower().strip() return xs.pipe( ops.group_by(key_mapper, lambda x: x), ops.map(lambda g: g.key), )
def factory(): def key_mapper(x): key_invoked[0] += 1 return x.lower().strip() return xs.pipe( ops.group_by(key_mapper, lambda x: x), ops.map(lambda g: g.key), )
def subscription(scheduler, state): source = xs.pipe( ops.group_by( key_mapper=lambda x: x % 2, element_mapper=None, subject_mapper=lambda: rx.subject.ReplaySubject(2), ) ) return source.subscribe(observer_groups, scheduler=scheduler)
def _window_to_group(value): return value.pipe( ops.to_iterable(), ops.map(lambda x: rx.from_iterable(x).pipe( # Group window by 'organization', 'bucket' and 'precision' ops.group_by(_group_by), # Create batch (concatenation line protocols by \n) ops.map(_group_to_batch), ops.merge_all())), ops.merge_all())
def test_groupby_count(self): res = [] counts = rx.from_(range(10)).pipe( ops.group_by(lambda i: 'even' if i % 2 == 0 else 'odd'), ops.flat_map(lambda i: i.pipe( ops.count(), ops.map(lambda ii: (i.key, ii)), ))) counts.subscribe(on_next=res.append) assert res == [('even', 5), ('odd', 5)]
def requestQuotes(self, tickers): logger.debug(f'QuoteObserver.requestQuotes({tickers})') self.quoteSubscription = create( lambda o, s: beginStreamingQuotes(tickers, o, s)).pipe( op.subscribe_on(config.pool_scheduler), op.observe_on(config.pool_scheduler), op.do_action(lambda q: logger.debug(f'QO: {q}')), op.group_by(lambda q: q['symbol']), ).subscribe(on_next=self.handleQuote, on_error=lambda e: logger.debug(e), on_completed=lambda: logger.debug( 'QuoteObserver subscription completed'))
def factory(): def key_mapper(x): key_invoked[0] += 1 return x.lower().strip() def element_mapper(x): ele_invoked[0] += 1 return x[::-1] # Yes, this is reverse string in Python return xs.pipe( ops.group_by(key_mapper, element_mapper), ops.map(lambda g: g.key), )
def factory(): def key_mapper(x): key_invoked[0] += 1 return x.lower().strip() def element_mapper(x): ele_invoked[0] += 1 return x[::-1] return xs.pipe( ops.group_by(key_mapper, element_mapper), ops.map(lambda g: g.key), )
def factory(): def key_mapper(x): key_invoked[0] += 1 return x.lower().strip() def element_mapper(x): ele_invoked[0] += 1 return x[::-1] return xs.pipe( ops.group_by(key_mapper, element_mapper), ops.map(lambda g: g.key), )
def grouped_sample( key_mapper: ops.Mapper, sampler: Union[ops.timedelta, float, rx.Observable], ): """ Combination of "group_by", "flat_map" and "sample", groups an observable sequence by the "key_mapper" function, maps the resulting observable sequences with the "sample" operator and flatten it into a single observable sequence. """ return rx.pipe( ops.group_by(key_mapper), ops.flat_map(lambda x: x.pipe(ops.sample(sampler))), )
def factory(): def key_mapper(x): key_invoked[0] += 1 return x.lower().strip() def element_mapper(x): ele_invoked[0] += 1 return x[::-1] # Yes, this is reverse string in Python return xs.pipe( ops.group_by(key_mapper, element_mapper), ops.map(lambda g: g.key), )
def demo_group_by(): '''tuple unpacking''' a = rx.of([ {'id':1 , 'name': 'aaa'}, {'id':2 , 'name': 'bbb'}, {'id':1 , 'name': 'aaa'}, {'id':1 , 'name': 'aaa'}, {'id':2 , 'name': 'aaa'}, ]) a.pipe( ops.group_by(lambda x: x['id'], lambda x: x['name'], subject_mapper = rx.subject.ReplaySubject()), ops.to_iterable(), ).subscribe(print)
def _watch_heartbeat( self, key_mapper: Callable[[Any], str], ) -> Observable: """ :returns: Observable[Tuple[str, bool]] """ return rx.pipe( ops.group_by(key_mapper), ops.flat_map(lambda x: x.pipe( heartbeat(self.LIVELINESS), ops.map(lambda y: (x.key, y)), )), )
def __init__(self, concurrency_per_group, description=None): self.scheduler = ThreadPoolScheduler(concurrency_per_group) self._requests = Subject() self._output = ReplaySubject() self._description = description self._subscription = self._requests.pipe( group_by(lambda r: r['concurrency_group']), flat_map(lambda concurrency_group: concurrency_group.pipe( map(lambda r: r['request']), merge(max_concurrent=concurrency_per_group)))).subscribe( on_next=lambda request: self._output.on_next(request), on_error=lambda error: logging.exception( 'Error in {} request stream'.format(self)), on_completed=lambda: logging.error( '{} request stream unexpectedly completed'.format(self )), scheduler=self.scheduler)
def __init__( self, influxdb_client, write_options: WriteOptions = WriteOptions(), point_settings: PointSettings = PointSettings() ) -> None: self._influxdb_client = influxdb_client self._write_service = WriteService(influxdb_client.api_client) self._write_options = write_options self._point_settings = point_settings if influxdb_client.default_tags: for key, value in influxdb_client.default_tags.items(): self._point_settings.add_default_tag(key, value) if self._write_options.write_type is WriteType.batching: # Define Subject that listen incoming data and produces writes into InfluxDB self._subject = Subject() self._disposable = self._subject.pipe( # Split incoming data to windows by batch_size or flush_interval ops.window_with_time_or_count(count=write_options.batch_size, timespan=timedelta(milliseconds=write_options.flush_interval)), # Map window into groups defined by 'organization', 'bucket' and 'precision' ops.flat_map(lambda window: window.pipe( # Group window by 'organization', 'bucket' and 'precision' ops.group_by(lambda batch_item: batch_item.key), # Create batch (concatenation line protocols by \n) ops.map(lambda group: group.pipe( ops.to_iterable(), ops.map(lambda xs: _BatchItem(key=group.key, data=_body_reduce(xs), size=len(xs))))), ops.merge_all())), # Write data into InfluxDB (possibility to retry if its fail) ops.map(mapper=lambda batch: self._retryable(data=batch, delay=self._jitter_delay())), # ops.merge_all())\ .subscribe(self._on_next, self._on_error, self._on_complete) else: self._subject = None self._disposable = None
def __init__(self, concurrency_per_group, delay_seconds=0, description=None): self.scheduler = ThreadPoolScheduler(concurrency_per_group) self.request_scheduler = ThreadPoolScheduler(10) self._requests = Subject() self._output_subject = Subject() self._output = self._output_subject.pipe(share()) self._description = description self._subscription = self._requests.pipe( observe_on(self.scheduler), group_by(lambda r: r['concurrency_group']), flat_map(lambda concurrency_group: concurrency_group.pipe( map(lambda r: r['request']), merge(max_concurrent=concurrency_per_group), delay(delay_seconds))), take_until_disposed()).subscribe( on_next=lambda request: self._output_subject.on_next(request), on_error=lambda error: logging.exception( 'Error in {} request stream'.format(self)), on_completed=lambda: self.dispose(), scheduler=self.scheduler)
files = ['test.csv', 'test2.csv'] source = rx.from_([rx.from_(read_csv(filename)) for filename in files]).pipe(ops.merge_all(), ops.map(row_to_dict)) published = source.pipe(ops.publish()) def print_row(row): print( 'File: {filename} has {lines} lines and its size is {size} kb'.format( **row)) def print_group(group): return group.subscribe(print_row) # 두 곳에서 subscribe 한다. #1 maximum = published.pipe(ops.max(lambda a, b: a['lines'] - b['lines'])) maximum.subscribe(lambda row: print( '-- File with the most number of lines is "{filename}" with {lines} lines'. format(**row))) # 2 published.pipe(ops.group_by(lambda row: row['size'])).subscribe(print_group) # 이걸 해야 시작 published.connect()
conn.get_database("planning").get_collection("graphs2")) save_edges_in_db = partial( save_many_items, conn.get_database("planning").get_collection("edges2")) save_nodes_in_db = partial( save_many_items, conn.get_database("planning").get_collection("nodes2")) graph_type_map = {"source": "edge", "node_name": "node", "name": "graph"} get_dict_type = partial(get_obj_type_from_type_map, graph_type_map) edge_subject, node_subject, graph_subject = Subject(), Subject(), Subject() processed_edges = edge_subject.pipe( op.filter(lambda edge_dic: not exists(edge_dic)), op.group_by(lambda dic: "".join( [str(v) for k, v in dic.items() if k not in ['level', 'type']])), op.map(lambda o: general_edge_grouper(o)), op.merge_all(), op.buffer_with_count(1000), op.map(lambda dict_list: save_edges_in_db(dict_list)), op.buffer_with_count(5), op.map(lambda futures: perform_futures(futures)), op.map(lambda results: [r.inserted_ids for r in results])).subscribe(dumb) processed_nodes = node_subject.pipe( op.filter(lambda node_dic: not exists(node_dic)), op.group_by(lambda dic: "".join( [str(v) for k, v in dic.items() if k not in ['level']])), op.map(lambda o: general_node_grouper(o)), op.merge_all(), op.buffer_with_count(5000), op.map(lambda dict_list: save_nodes_in_db(dict_list)), op.buffer_with_count(5), op.map(lambda futures: perform_futures(futures)), op.map(lambda results: [r.inserted_ids for r in results])).subscribe(dumb)
def action1(scheduler, state): xs[0] = rx.from_iterable(["alpha", "apple", "beta", "bat", "gamma"]) \ .pipe(ops.group_by(lambda s: s[0]), ops.map(lambda xs: xs.pipe(ops.to_iterable(), ops.map(list))), ops.merge_all(), )
rx.repeat_value() rx.start() rx.timer() """Mathematical""" op.average() op.concat() op.count() op.max() op.min() op.reduce() op.sum() """Transformation""" op.buffer() op.group_by() op.map() op.scan() # ... """Filtering""" op.debounce() op.distinct() op.filter() op.element_at() op.first() op.ignore_elements() op.last() op.skip() op.skip_last() op.take()
import rx import rx.operators as ops from rx.subject import Subject def wrap_items(i): return i.pipe(ops.map(lambda j: 'obs {}: {}'.format(i, j))) numbers = rx.from_([1, 2, 3, 4, 5, 6]) numbers.pipe( ops.group_by(lambda i: i % 2 == 0), ops.flat_map(wrap_items), ).subscribe(on_next=lambda i: print("on_next {}".format(i)), on_error=lambda e: print("on_error: {}".format(e)), on_completed=lambda: print("on_completed"))
import rx from rx import operators as ops items = ["Alpha", "Beta", "Gamma", "Delta", "Epsilon"] rx.from_(items).pipe( ops.group_by(lambda s: len(s)), ops.flat_map(lambda grp: grp.pipe(ops.to_list())) ).subscribe( lambda i: print(i) )
def action1(scheduler, state): xs[0] = rx.from_iterable(["alpha", "apple", "beta", "bat", "gamma"]) \ .pipe(ops.group_by(lambda s: s[0]), ops.map(lambda xs: xs.pipe(ops.to_iterable(), ops.map(list))), ops.merge_all(), )
from rx import from_, of, from_iterable, operators as ops, create items = ["Alpha", "Beta", "Gamma", "Delta", "Epsilon"] from_(items).pipe(ops.group_by(key_mapper=lambda s: len(s)), ops.flat_map(lambda grp: grp.pipe(ops.to_list()))).subscribe( lambda i: print(i))
return 'even' return 'odd' # 그룹의 키 값을 참조할 수 있다. def subscribe_group_observable(group_observable): def print_count(count): print(f'Group key={group_observable.key} contains {count} items') group_observable.pipe( ops.count() ).subscribe(print_count) groups = rx.from_(range(3)).pipe( ops.group_by(key_selector) ) groups.subscribe(subscribe_group_observable) # Sample: A way to grab items and emit latest values at certain points print('-- Sample') # 10 ms 마다 인터벌 발생 (시간의 단위는 초이다.) rx.interval(0.01).pipe( # 1초 동안 동작 ops.take_until(rx.timer(1)), # 100ms 마다 이벤트 출력 (현재 받은 최신값) ops.sample(0.1) ).subscribe(print) time.sleep(2)