def run_parallel(args):
worker_factory, simulation_cases = factory.resolve_configuration(args)
logger.debug("Shuffling the job queue")
random.shuffle(simulation_cases)
logger.debug("Creating the pool")
processes_no = int(args["-j"])
rxtools.configure_default_executor(processes_no)
wall_time = []
start_time = datetime.now()
results = []
logger.debug("Simulation cases: %s", simulation_cases)
logger.debug("Work will be divided into %d processes", processes_no)
sys = ActorSystem("multiprocTCPBase", logDefs=log_helper.EVOGIL_LOG_CONFIG)
with log_time(system_time, logger, "Pool evaluated in {time_res}s", out=wall_time):
def process_result(subres):
results.append(subres)
log_simulation_stats(start_time, subres[-1], len(simulation_cases))
rx.from_iterable(range(len(simulation_cases))).pipe(
ops.map(lambda i: worker_factory(simulation_cases[i], i)),
# ops.map(lambda w: rxtools.from_process(w.run)),
ops.map(lambda w : w.run()),
# ops.merge(max_concurrent=1)
ops.do_action(on_next=process_result)
).run()
log_summary(args, results, simulation_cases, wall_time)
rxtools.shutdown_default_executor()
sys.shutdown()
async def run(self):
msg = await self.receive(timeout=1)
if msg:
# print("Window Tick Stream")
db_id = msg.get_metadata("db_id")
window = await fx_db.get_fx_window(db_id)
# print(window)
rx.from_iterable(self.analysers).pipe(
ops.filter(lambda a: a.symbol == window.symbol)).subscribe_(
lambda tick_analyser: tick_analyser.on_next(window))
async def run(self):
msg = await self.receive(timeout=1)
if msg:
# print(msg)
tick_id = msg.get_metadata("fx_tick_id")
tick = await fx_db.get_fx_tick(tick_id)
# print(tick.symbol)
# rx.from_iterable(self.analysers).subscribe_(print)
rx.from_iterable(self.analysers).pipe(
ops.filter(lambda a: a.symbol == tick.symbol)).subscribe_(
lambda tick_analyser: tick_analyser.on_next(tick))
# print(fx_tick)
await asyncio.sleep(delay=sleep_delay)
def __init__(self,
endpoints: [typing.ServiceEndpoint] = None,
endpoint_clients: [typing.ServiceEndpointClient] = None,
request_correlator: typing.RequestCorrelator = DefaultRequestCorrelator()):
super().__init__(endpoint_clients, request_correlator)
if endpoints is None:
self.__endpoints = []
else:
self.__endpoints = endpoints
self.__requests = rx.from_iterable(map(lambda ep: ep.requests, self.__endpoints)).pipe(merge_all(), share())
self.__commands = rx.from_iterable(map(lambda ep: ep.commands, self.__endpoints)).pipe(merge_all(), share())
def _window_to_group(self, value):
return value.pipe(
ops.to_iterable(),
ops.map(
lambda x: rx.from_iterable(x).pipe(ops.group_by(
_group_by), ops.map(_group_to_batch), ops.merge_all())),
ops.merge_all())
def logs(self, number_of_lines: int) -> Observable[str]:
return rx.from_callable(lambda: self._client.post_json({'uri': self._uri, 'numberOfLines': number_of_lines},
url_postfix="/skill/logs"), self._scheduler) \
.pipe(
ops.map(lambda r: json.loads(r.content)),
ops.flat_map(lambda r: rx.from_iterable(r['logLines']))
)
def test_collect_works(self):
obs = open_orders.collect(
from_iterable([
IbApiMessage(type=IbApiMessageType.OPEN_ORDER,
payload=(0, test_utils.appl_contract(), _order1(),
_order_state1())),
IbApiMessage(type=IbApiMessageType.OPEN_ORDER,
payload=(15, test_utils.ibkr_contract(),
_order2(), _order_state2())),
]))
# TODO: for now, Contract, Order, OrderState, etc cannot be tested
# naively as they do not have __eq__ defined.
assert_that(
obs.run(),
contains_exactly(
all_of(
has_property('order_id', equal_to(0)),
has_property('contract', anything()),
has_property('order', anything()),
has_property('order_state', anything()),
),
all_of(
has_property('order_id', equal_to(15)),
has_property('contract', anything()),
has_property('order', anything()),
has_property('order_state', anything()),
)))
def _merge(*args: Union[Observable, Iterable[Observable]]) -> Observable:
sources = args[:]
if isinstance(sources[0], Iterable):
sources = sources[0]
return rx.from_iterable(sources).pipe(ops.merge_all())
def main():
loop = asyncio.get_event_loop()
io_scheduler = AsyncIOThreadSafeScheduler(loop=loop)
scheduler = ThreadPoolScheduler(multiprocessing.cpu_count())
video_stream_observable = rx.using(
lambda: VideoStreamDisposable(),
lambda d: rx.from_iterable(video_stream_iterable(d.cap)))
disposable = video_stream_observable.pipe(
ops.subscribe_on(scheduler),
ops.sample(1 / ARGS.fps), # sample frames based on fps
ops.filter(has_face), # filter frames without faces
ops.map(lambda frame: Image.fromarray(
cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))), # map frame to PIL image
ops.observe_on(io_scheduler),
ops.map(lambda img: ImageFacesPair(img, analyse_frame(img))
), # analyse faces on frame
ops.filter(
lambda img_faces_pair: any([
face.top_prediction.confidence >= ARGS.min_confidence and face.
top_prediction.confidence <= ARGS.max_confidence
for face in img_faces_pair.faces
])
), # proceed only if min_confidence <= person_confidence <= max_confidence
ops.do_action(on_next=save_frame)).subscribe(
on_error=lambda e: logger.exception(e))
try:
loop.run_forever()
except Exception as e:
logger.exception(e)
logger.info("Data collector shutdown")
disposable.dispose()
def test_collect_works(self):
obs = position.collect(
from_iterable([
IbApiMessage(type=IbApiMessageType.POSITION,
payload=('DU123', test_utils.appl_contract(),
100.0, 123.45)),
IbApiMessage(type=IbApiMessageType.POSITION,
payload=('DU123', test_utils.ibkr_contract(), 1.0,
45.6789)),
]))
# TODO: for now, Contract, etc cannot be tested naively as they do not have __eq__ defined.
assert_that(
obs.run(),
contains_exactly(
all_of(
has_property('account', equal_to('DU123')),
has_property('contract', anything()),
has_property('size', equal_to(100.0)),
has_property('average_cost', equal_to(123.45)),
),
all_of(
has_property('account', equal_to('DU123')),
has_property('contract', anything()),
has_property('size', equal_to(1.0)),
has_property('average_cost', equal_to(45.6789)),
)))
def get_files(base_dir: Path, filter_regex: str,
scheduler: rx.typing.Scheduler) -> rx.Observable:
"""Return an observable of files to process as FileTarget's."""
return rx.from_iterable(
fl.file_listing_iterator(base_dir, filter_regex),
scheduler=scheduler,
)
def to_rx(self, *inputs: rx.Observable) -> rx.Observable:
# NOTE If Module is not attached to graph, do not check len(inputs)
# TODO Does it make sense for module to be detached from graph?
if self._is_used_in_static_graph:
self._check_num_inputs(len(inputs), check_nodes=True)
self.consume(*inputs)
# Create dummy observable to satisfy type signature
return rx.from_iterable([])
def _solve(print=print):
total = rx.from_iterable(primes()) \
.pipe(
ops.take_while(lambda p: p < 2000000),
ops.sum(),
).run()
print(f'The sum of primes below 2m: {total}')
return True
def _solve(print=print):
v = rx.from_iterable(primes()) \
.pipe(
ops.skip(10000),
ops.take(1),
).run()
print(v)
return True
async def test_rx_support_request_channel_server_take_only_n(pipe: Tuple[RSocketServer, RSocketClient],
take_only_n):
server, client = pipe
received_messages = []
items_generated = 0
maximum_message_count = 3
wait_for_server_finish = asyncio.Event()
class Handler(BaseRequestHandler, DefaultSubscriber):
def on_next(self, value: Payload, is_complete=False):
received_messages.append(value)
if len(received_messages) < take_only_n:
self.subscription.request(1)
else:
self.subscription.cancel()
wait_for_server_finish.set()
def on_complete(self):
wait_for_server_finish.set()
async def on_error(self, error_code: ErrorCode, payload: Payload):
wait_for_server_finish.set()
def on_subscribe(self, subscription: Subscription):
super().on_subscribe(subscription)
subscription.request(1)
async def request_channel(self, payload: Payload) -> Tuple[Optional[Publisher], Optional[Subscriber]]:
return None, self
server.set_handler_using_factory(Handler)
rx_client = RxRSocket(client)
def generator():
nonlocal items_generated
for x in range(maximum_message_count):
items_generated += 1
yield Payload('Feed Item: {}'.format(x).encode('utf-8'))
await rx_client.request_channel(
Payload(b'request text'),
observable=rx.from_iterable(generator())
).pipe(
operators.to_list()
)
await wait_for_server_finish.wait()
maximum_message_received = min(maximum_message_count, take_only_n)
# assert items_generated == maximum_message_received # todo: Stop async generator on cancel from server requester
assert len(received_messages) == maximum_message_received
for i in range(maximum_message_received):
assert received_messages[i].data == ('Feed Item: %d' % i).encode()
def step(self):
last_result = (rx.from_iterable(self.islands).pipe(
ops.subscribe_on(NewThreadScheduler()),
ops.flat_map(lambda island: island.epoch(self.epoch_length).pipe(
ops.last())),
ops.buffer_with_count(len(self.islands)),
).run())
self.migration()
self.update_cost(last_result)
def test_input_module():
inputs = [Tensor((1, ), "int", x) for x in [1, 2, 3]]
expected = inputs
results = []
module = InputModule((1, ), "int")
obs = module.to_rx(rx.from_iterable(inputs))
obs.subscribe(lambda x: results.append(x))
assert results == expected
def _window_to_group(value):
return value.pipe(
ops.to_iterable(),
ops.map(lambda x: rx.from_iterable(x).pipe(
# Group window by 'organization', 'bucket' and 'precision'
ops.group_by(_group_by),
# Create batch (concatenation line protocols by \n)
ops.map(_group_to_batch),
ops.merge_all())),
ops.merge_all())
def assert_observed_data_ok(self, data, reduce_func, start_cond, end_cond,
exp_nexts, exp_errs, exp_compl):
obs = MockObserver()
stream = rx.from_iterable(data)
stream.pipe(observing.reduce_while(reduce_func, start_cond,
end_cond)).subscribe(obs)
self.assertEqual(exp_nexts, obs.nexts)
self.assertEqual(exp_errs, obs.errs)
self.assertEqual(exp_compl, obs.compl)
def _failed_transfers(store):
processing_files = ReplaySubject()
def transfer_files():
state = store.getState()
if (state.processing):
processing_files.on_next(state.processing)
store.subscribe(transfer_files)
return processing_files.pipe(
operators.map(lambda paths: rx.from_iterable(paths)),
operators.merge_all(), operators.flat_map(_transfer_file))
def convert_to_syncmaps():
input_observable = rx.from_iterable(get_files('srt', 'json'),
pool_scheduler)
input_observable.pipe(
ops.map(srt_to_dict),
ops.map(to_syncmap),
).subscribe(
on_next=lambda filename: print(f"Converted {filename}"),
on_completed=lambda: print("Converting completed"),
on_error=lambda err: print(f"ERROR: type: {type(err)}, message: {err}"
),
)
def test_collect_historical_trade_data(self):
bar_data_1 = IbBarData()
bar_data_1.date = '1577000000'
bar_data_1.open = 1
bar_data_1.high = 2
bar_data_1.low = 1.5
bar_data_1.close = 1.75
bar_data_1.average = 1.567
bar_data_1.volume = 9414151
bar_data_1.barCount = 3435
bar_data_2 = IbBarData()
bar_data_2.date = '1577003600'
bar_data_2.open = 2
bar_data_2.high = 3
bar_data_2.low = 1.25
bar_data_2.close = 2.25
bar_data_2.average = 2.123
bar_data_2.volume = 4543625
bar_data_2.barCount = 3821
messages = [
IbApiMessage(type=IbApiMessageType.HISTORICAL_DATA,
payload=(0, bar_data_1)),
IbApiMessage(type=IbApiMessageType.HISTORICAL_DATA,
payload=(0, bar_data_2)),
]
assert_that(
historical_data.collect(from_iterable(messages), 0,
types.HistoricalDataType.TRADES).run(),
contains_exactly(
equal_to(
types.BarData(
type=types.HistoricalDataType.TRADES,
time=datetime.datetime(2019, 12, 22, 7, 33, 20),
open=1,
high=2,
low=1.5,
close=1.75,
trade_data=types.BarData.TradeData(volume=9414151,
average_price=1.567,
bar_count=3435))),
equal_to(
types.BarData(
type=types.HistoricalDataType.TRADES,
time=datetime.datetime(2019, 12, 22, 8, 33, 20),
open=2,
high=3,
low=1.25,
close=2.25,
trade_data=types.BarData.TradeData(volume=4543625,
average_price=2.123,
bar_count=3821)))))
def run(onComplete=lambda: None):
limit = 10 ** 22
optimal_thread_count = multiprocessing.cpu_count()
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
count = rx.from_iterable(pandigital_step_numbers()) \
.pipe(
ops.take_while(lambda n: n < limit),
ops.count(),
) \
.run()
onComplete(count)
def start_with(source: Observable) -> Observable:
"""Partially applied start_with operator.
Prepends a sequence of values to an observable sequence.
Example:
>>> start_with(source)
Returns:
The source sequence prepended with the specified values.
"""
start = rx.from_iterable(args)
sequence = [start, source]
return rx.concat(*sequence)
def __init__(self, stub):
self._stub = stub
self._slots = [
'OnReceiveMsg',
'OnReceiveTrData',
'OnReceiveChejanData',
'OnEventConnect',
]
request = KiwoomOpenApiService_pb2.ListenRequest()
request.slots.extend(self._slots) # pylint: disable=no-member
self._response_iterator = self._stub.Listen(request)
self._subscription = rx.from_iterable(
self._response_iterator).subscribe(self)
def test_forward_module():
class MyForwardModule(ForwardModule):
def forward(self, x: Tensor) -> Tensor:
return Tensor((None, ), "str", str(x.data))
inputs = [Tensor((1, ), "int", x) for x in [1, 2, 3]]
expected = [Tensor((None, ), "str", x) for x in ["1", "2", "3"]]
results = []
module = MyForwardModule((None, ), "str")
obs = module.to_rx(rx.from_iterable(inputs))
obs.subscribe(lambda x: results.append(x))
assert results == expected
def start_with(source: Observable) -> Observable:
"""Partially applied start_with operator.
Prepends a sequence of values to an observable sequence.
Example:
>>> start_with(source)
Returns:
The source sequence prepended with the specified values.
"""
start = rx.from_iterable(args)
sequence = [start, source]
return rx.concat(*sequence)
def test_forward_async_module():
class MyForwardAsyncModule(ForwardAsyncModule):
def forward(self, xs: rx.Observable) -> rx.Observable:
return xs.pipe(ops.map(str))
inputs = [1, 2, 3]
expected = ["1", "2", "3"]
results = []
module = MyForwardAsyncModule((None, ), "str")
obs = module.to_rx(rx.from_iterable(inputs))
obs.subscribe(lambda x: results.append(x))
assert results == expected
def main():
loop = asyncio.get_event_loop()
io_scheduler = AsyncIOThreadSafeScheduler(loop=loop)
scheduler = ThreadPoolScheduler(multiprocessing.cpu_count())
semaphore = Subject()
semaphore_stream = semaphore.pipe(
ops.flat_map(lambda _: rx.of(True).pipe(
ops.delay(ARGS.block_time, scheduler=scheduler),
ops.start_with(False))), ops.start_with(True))
video_stream_observable = rx.using(
lambda: VideoStreamDisposable(),
lambda d: rx.from_iterable(video_stream_iterable(d.cap)))
gated_video_stream = video_stream_observable.pipe(
ops.subscribe_on(scheduler),
ops.sample(1 / ARGS.fps), # sample frames based on fps
ops.combine_latest(semaphore_stream),
ops.filter(lambda tup: tup[1]), # proceed only if semaphore allows
ops.map(lambda tup: tup[0]) # take only frame
)
disposable = gated_video_stream.pipe(
ops.filter(has_face), # filter frames without faces
ops.map(lambda frame: Image.fromarray(
cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))), # map frame to PIL image
ops.map(lambda img: img.resize(
(640, 360))), # resize image (inference will be faster)
ops.observe_on(io_scheduler),
ops.map(lambda img: ImageFacesPair(img, analyse_frame(img))
), # analyse frame for faces
ops.filter(lambda img_faces_pair: any([
face.top_prediction.confidence > ARGS.threshold
for face in img_faces_pair.faces
])), # proceed only if there is a known face in the frame
ops.throttle_first(1),
ops.flat_map(unlock_request), # unlock the door
ops.do_action(
on_next=lambda _: semaphore.on_next(True)
) # trigger semaphore which will block stream for "block-seconds" seconds (doors are unlocked for that long after unlock request)
).subscribe(on_error=lambda e: logger.exception(e))
try:
loop.run_forever()
except Exception as e:
logger.exception(e)
logger.info("Smart lock face recognition engine shutdown")
disposable.dispose()
def intensity_steps(steps: List[Step], flushes):
"""
Given an intensity value, return the list of steps
which should be played back. The logic indexes higher
based on the intensity value until it is reset by the final step
:param steps:
"""
return pipe(
ops.flat_map(lambda event: rx.from_iterable(steps).pipe(
ops.map(lambda step: (step, event)))),
ops.filter(lambda x: x[0] <= x[1]), ops.map(lambda x: x[0]),
ops.distinct(flushes=flushes))
def test_run_empty_graph():
def create_graph():
inputs = [Input((1, ), "int")]
outputs = inputs
return Model(inputs=inputs, outputs=outputs)
inputs = [Tensor((1, ), "int", x) for x in [1, 2, 3]]
expected = inputs
results = []
model = create_graph()
model.setup_blocking()
obs = model.to_rx(rx.from_iterable(inputs))
obs[0].subscribe(lambda x: results.append(x))
assert results == expected
def _merge(*sources: Observable) -> Observable:
return rx.from_iterable(sources).pipe(ops.merge_all())
def action1(scheduler, state):
xs[0] = rx.from_iterable(["alpha", "apple", "beta", "bat", "gamma"]) \
.pipe(ops.group_by(lambda s: s[0]),
ops.map(lambda xs: xs.pipe(ops.to_iterable(), ops.map(list))),
ops.merge_all(),
)
def test_observer_throws(self):
with self.assertRaises(RxException):
rx.from_iterable([1, 2, 3]).subscribe(lambda x: _raise('ex'))
def _sequence_equal(second: Observable, comparer: Callable[[Any, Any], bool] = None
) -> Callable[[Observable], Observable]:
comparer = comparer or default_comparer
if isinstance(second, collections.abc.Iterable):
second = rx.from_iterable(second)
def sequence_equal(source: Observable) -> Observable:
"""Determines whether two sequences are equal by comparing the
elements pairwise using a specified equality comparer.
Examples:
>>> res = sequence_equal([1,2,3])
>>> res = sequence_equal([{ "value": 42 }], lambda x, y: x.value == y.value)
>>> res = sequence_equal(rx.return_value(42))
>>> res = sequence_equal(rx.return_value({ "value": 42 }), lambda x, y: x.value == y.value)
Args:
source: Source obserable to compare.
Returns:
An observable sequence that contains a single element which
indicates whether both sequences are of equal length and their
corresponding elements are equal according to the specified
equality comparer.
"""
first = source
def subscribe(observer, scheduler=None):
donel = [False]
doner = [False]
ql = []
qr = []
def on_next1(x):
if len(qr) > 0:
v = qr.pop(0)
try:
equal = comparer(v, x)
except Exception as e:
observer.on_error(e)
return
if not equal:
observer.on_next(False)
observer.on_completed()
elif doner[0]:
observer.on_next(False)
observer.on_completed()
else:
ql.append(x)
def on_completed1():
donel[0] = True
if not ql:
if qr:
observer.on_next(False)
observer.on_completed()
elif doner[0]:
observer.on_next(True)
observer.on_completed()
def on_next2(x):
if len(ql) > 0:
v = ql.pop(0)
try:
equal = comparer(v, x)
except Exception as exception:
observer.on_error(exception)
return
if not equal:
observer.on_next(False)
observer.on_completed()
elif donel[0]:
observer.on_next(False)
observer.on_completed()
else:
qr.append(x)
def on_completed2():
doner[0] = True
if not qr:
if len(ql) > 0:
observer.on_next(False)
observer.on_completed()
elif donel[0]:
observer.on_next(True)
observer.on_completed()
subscription1 = first.subscribe_(on_next1, observer.on_error, on_completed1, scheduler)
subscription2 = second.subscribe_(on_next2, observer.on_error, on_completed2, scheduler)
return CompositeDisposable(subscription1, subscription2)
return Observable(subscribe)
return sequence_equal
