def model_publisher(scheduler, sources): file_source = sources.file.response.pipe(ops.share()) # kafka driver bootstrap. fixme kafka_source = sources.kafka.response.pipe( ops.do_action(print), ops.replay(), ops.ref_count(), ) kafka_source.subscribe() config, config_read_request, http_request = read_config_from_args( sources.argv.argv, file_source, sources.http.response, scheduler=scheduler) config = config.pipe(ops.first()) kafka_request = config.pipe(ops.map(lambda c: create_model_topics(c)), ) return ModelPublisherSink( file=file.Sink(request=rx.merge(config_read_request)), http=http.Sink(request=http_request), kafka=kafka.Sink(request=kafka_request), )
def _parse_config(config_data): ''' takes a stream with the content of the configuration file as input and returns a (hot) stream of arguments . ''' return config_data.pipe( ops.map(lambda i: yaml.load(i, Loader=yaml.FullLoader)), ops.replay(buffer_size=1), ops.ref_count(), )
def test_replay_time_multiple_connections(self): xs = rx.never() ys = xs.pipe(ops.replay(window=100)) connection1 = ys.connect() connection2 = ys.connect() assert connection1 == connection2 connection1.dispose() connection2.dispose() connection3 = ys.connect() assert connection1 != connection3
def test_replay_count_multiple_connections(self): xs = rx.never() ys = xs.pipe(ops.replay(None, 3)) connection1 = ys.connect() connection2 = ys.connect() assert connection1 == connection2 connection1.dispose() connection2.dispose() connection3 = ys.connect() assert connection1 != connection3
def makinage(aio_scheduler, sources): def on_error(e): raise e config, read_request, http_request = read_config_from_args( sources.argv.argv, sources.file.response, sources.http.response, scheduler=aio_scheduler ) first_config = rx.concat(config.pipe(ops.take(1),), rx.never()) kafka_source = sources.kafka.response.pipe( trace_observable("kafka source1"), ops.replay(), ops.ref_count(), trace_observable("kafka source2"), ) kafka_source.subscribe(on_error=on_error) kafka_request = first_config.pipe( ops.flat_map(lambda i: create_operators( i, config, kafka_source, sources.kafka.feedback.pipe(ops.share()), )), ops.subscribe_on(aio_scheduler), trace_observable("makinage"), ) ''' config.pipe(ops.subscribe_on(aio_scheduler)).subscribe( on_next=print, on_error=print, ) ''' return MakiNageSink( file=file.Sink(request=read_request), http=http.Sink(request=http_request), kafka=kafka.Sink(request=kafka_request), )
def action0(scheduler, state): ys[0] = xs.pipe(ops.replay(buffer_size=3, scheduler=scheduler))
def action0(scheduler, state): ys[0] = xs.pipe(ops.replay(window=100, scheduler=scheduler))
#emit modified date every second, if the date has changed since last time. #use a connectable observable so that multiple downstreams #all use the same file polling. file_changed_observable = file_check_interval_observable.pipe( #Uncomment this to see what happens if file takes 10 seconds to load. #ops.map(lambda i: intense_calculation(myinput_path, i)), ops.map(lambda i: get_modified_date(myinput_path)), ops.distinct_until_changed(), #Without this each subscriber would create a new stream #So we'd have one modification date check for the debug statement below #and one for the debug, which is unnecessary. #Replay ensures late subscribers #will read the file. ops.replay(buffer_size=1), ops.ref_count() ) #Debug subscription to show when modified date gets printed out. file_changed_observable.subscribe( on_next = lambda i: print("Got date modified - {0}".format(i)), on_error = lambda e: print("Error : {0}".format(e)), on_completed = lambda: print("Job Done!"), ) #sleep 15 seconds, to test if a late subscriber will get an emission before a file change #(it should). #time.sleep(15) #When a emission is produced upstream, read contents of the new file.
op.catch() op.retry() """Utility""" op.delay() op.materialize() op.time_interval() op.timeout() op.timestamp() """Conditional and Boolean""" op.all() op.contains() op.default_if_empty() op.sequence_equal() op.skip_until() op.skip_while() op.take_until() op.take_while() """Connectable""" op.publish() op.ref_count() op.replay() """Combining""" op.combine_latest() op.merge() op.start_with() op.zip()