def model_publisher(scheduler, sources):
file_source = sources.file.response.pipe(ops.share())
# kafka driver bootstrap. fixme
kafka_source = sources.kafka.response.pipe(
ops.do_action(print),
ops.replay(),
ops.ref_count(),
)
kafka_source.subscribe()
config, config_read_request, http_request = read_config_from_args(
sources.argv.argv,
file_source,
sources.http.response,
scheduler=scheduler)
config = config.pipe(ops.first())
kafka_request = config.pipe(ops.map(lambda c: create_model_topics(c)), )
return ModelPublisherSink(
file=file.Sink(request=rx.merge(config_read_request)),
http=http.Sink(request=http_request),
kafka=kafka.Sink(request=kafka_request),
)
def _parse_config(config_data):
''' takes a stream with the content of the configuration file as input
and returns a (hot) stream of arguments .
'''
return config_data.pipe(
ops.map(lambda i: yaml.load(i, Loader=yaml.FullLoader)),
ops.replay(buffer_size=1),
ops.ref_count(),
)
def test_replay_time_multiple_connections(self):
xs = rx.never()
ys = xs.pipe(ops.replay(window=100))
connection1 = ys.connect()
connection2 = ys.connect()
assert connection1 == connection2
connection1.dispose()
connection2.dispose()
connection3 = ys.connect()
assert connection1 != connection3
def test_replay_count_multiple_connections(self):
xs = rx.never()
ys = xs.pipe(ops.replay(None, 3))
connection1 = ys.connect()
connection2 = ys.connect()
assert connection1 == connection2
connection1.dispose()
connection2.dispose()
connection3 = ys.connect()
assert connection1 != connection3
def makinage(aio_scheduler, sources):
def on_error(e):
raise e
config, read_request, http_request = read_config_from_args(
sources.argv.argv,
sources.file.response,
sources.http.response,
scheduler=aio_scheduler
)
first_config = rx.concat(config.pipe(ops.take(1),), rx.never())
kafka_source = sources.kafka.response.pipe(
trace_observable("kafka source1"),
ops.replay(),
ops.ref_count(),
trace_observable("kafka source2"),
)
kafka_source.subscribe(on_error=on_error)
kafka_request = first_config.pipe(
ops.flat_map(lambda i: create_operators(
i, config,
kafka_source,
sources.kafka.feedback.pipe(ops.share()),
)),
ops.subscribe_on(aio_scheduler),
trace_observable("makinage"),
)
'''
config.pipe(ops.subscribe_on(aio_scheduler)).subscribe(
on_next=print,
on_error=print,
)
'''
return MakiNageSink(
file=file.Sink(request=read_request),
http=http.Sink(request=http_request),
kafka=kafka.Sink(request=kafka_request),
)
def action0(scheduler, state):
ys[0] = xs.pipe(ops.replay(buffer_size=3, scheduler=scheduler))
def action0(scheduler, state):
ys[0] = xs.pipe(ops.replay(window=100, scheduler=scheduler))
#emit modified date every second, if the date has changed since last time.
#use a connectable observable so that multiple downstreams
#all use the same file polling.
file_changed_observable = file_check_interval_observable.pipe(
#Uncomment this to see what happens if file takes 10 seconds to load.
#ops.map(lambda i: intense_calculation(myinput_path, i)),
ops.map(lambda i: get_modified_date(myinput_path)),
ops.distinct_until_changed(),
#Without this each subscriber would create a new stream
#So we'd have one modification date check for the debug statement below
#and one for the debug, which is unnecessary.
#Replay ensures late subscribers
#will read the file.
ops.replay(buffer_size=1),
ops.ref_count()
)
#Debug subscription to show when modified date gets printed out.
file_changed_observable.subscribe(
on_next = lambda i: print("Got date modified - {0}".format(i)),
on_error = lambda e: print("Error : {0}".format(e)),
on_completed = lambda: print("Job Done!"),
)
#sleep 15 seconds, to test if a late subscriber will get an emission before a file change
#(it should).
#time.sleep(15)
#When a emission is produced upstream, read contents of the new file.
op.catch() op.retry() """Utility""" op.delay() op.materialize() op.time_interval() op.timeout() op.timestamp() """Conditional and Boolean""" op.all() op.contains() op.default_if_empty() op.sequence_equal() op.skip_until() op.skip_while() op.take_until() op.take_while() """Connectable""" op.publish() op.ref_count() op.replay() """Combining""" op.combine_latest() op.merge() op.start_with() op.zip()
