def test_import_identity_e2e(self):
self.gaiaRef = Gaia.login("http://localhost:8080", UsernamePasswordCredentials("admin", "admin"))
data = bytes("identity content", encoding="utf-8")
result = pipe(ops.first())(
self.gaiaRef.identity().import_identity('2fa4ff18-5c30-497b-9ad2-0d8eb51cd4da',
'identityName', data, True)).run()
self.assertEqual(len(result.uri), 72)
result = pipe(ops.first())(
self.gaiaRef.data("gaia://2fa4ff18-5c30-497b-9ad2-0d8eb51cd4da/").list()).run()
self.assertEqual(result[0].dictionary, {'filePath': 'identities/identityName',
'tenant': '2fa4ff18-5c30-497b-9ad2-0d8eb51cd4da'})
pass
def test_perceive(self):
gaia_ref = mock_gaia_ref(lambda x: MockResponse({
"data": {
"perceive": {
"perceiveData": {
"id": "asdf"
},
"perceiveAction": {
"id": "qwer"
}
}
}
}))
impulse1 = PerceiveActionImpulse(False, str(uuid.uuid4()), "", {})
impulse2 = PerceiveDataImpulse(str(uuid.uuid4()), "", {})
def perceive(p: Perception):
p.perceive_action(impulse1, lambda x: x.id())
p.perceive_data(impulse2, lambda x: x.id())
result = pipe(ops.first())(gaia_ref.perceive(perceive)).run()
perceiveData = result.dictionary.get("perceiveData")
perceiveAction = result.dictionary.get("perceiveAction")
assert perceiveData.get(
"id") is not None, "PerceiveData.id is in response"
assert perceiveAction.get(
"id") is not None, "perceiveAction.id is in response"
def test_export_identity_no_id(self):
def mock(request):
self.assertEqual(request.url_post_fix, "/identity/source")
return MockResponse(bytes("identity content", encoding="utf-8"))
self.gaiaRef = mock_gaia_ref(mock)
self.assertRaises(TypeError, lambda: pipe(ops.first())(self.gaiaRef.identity().export()).run())
def find_with_parent(parent, name):
return (list_folder(
credentials, parent, name_filter=name
) if parent else of([])).pipe(
first(
), # There might be more than one file with the same name - we just take the first
map(lambda files: files[0] if len(files) else None))
def _first(
predicate: Optional[Predicate] = None
) -> Callable[[Observable], Observable]:
"""Returns the first element of an observable sequence that
satisfies the condition in the predicate if present else the first
item in the sequence.
Examples:
>>> res = res = first()(source)
>>> res = res = first(lambda x: x > 3)(source)
Args:
predicate -- [Optional] A predicate function to evaluate for
elements in the source sequence.
Returns:
A function that takes an observable source and returns an
observable sequence containing the first element in the
observable sequence that satisfies the condition in the predicate if
provided, else the first item in the sequence.
"""
if predicate:
return pipe(ops.filter(predicate), ops.first())
return _first_or_default_async(False)
def model_publisher(scheduler, sources):
file_source = sources.file.response.pipe(ops.share())
# kafka driver bootstrap. fixme
kafka_source = sources.kafka.response.pipe(
ops.do_action(print),
ops.replay(),
ops.ref_count(),
)
kafka_source.subscribe()
config, config_read_request, http_request = read_config_from_args(
sources.argv.argv,
file_source,
sources.http.response,
scheduler=scheduler)
config = config.pipe(ops.first())
kafka_request = config.pipe(ops.map(lambda c: create_model_topics(c)), )
return ModelPublisherSink(
file=file.Sink(request=rx.merge(config_read_request)),
http=http.Sink(request=http_request),
kafka=kafka.Sink(request=kafka_request),
)
def timeout_check(process: subprocess.Popen, timeout: float,
ct: CancellationToken) -> rx.Observable:
"""Kills the given process after timeout has passed.
Args:
process: process to be killed
timeout: termination time in seconds
ct: CancellationToken
Return:
rx.Observable that fires only a single dictionary after the timeout
has passed.
"""
return rx.timer(timeout).pipe(
ops.do_action(
# Request cancellation so all simulation processes that
# share the same cancellation_token are also stopped.
# TODO not working as intended if simulation is short enough
# to stop before max_time has elapsed. Maybe let caller
# implement its own timeout check when multiple processes
# are being run.
on_next=lambda _: ct.request_cancellation()),
ops.map(
lambda _: {
# sutils.kill_process returns None, which can be casted to bool
MCERD.IS_RUNNING:
bool(sutils.kill_process(process)),
MCERD.MSG:
MCERD.SIM_TIMEOUT
}),
ops.first())
def test_list_files_in_nonexistent_dir(self):
def mock(request):
self.assertEqual(request.url_post_fix, "/data/list")
return MockResponse([])
self.gaiaRef = mock_gaia_ref(mock)
result = pipe(ops.first())(self.gaiaRef.data("gaia://tenant1/nonexistentDirectory").list()).run()
self.assertEqual(len(result), 0)
def test_write_new_file_with_path(self):
file = Path(tempfile.gettempdir()) / "file"
self.gaiaRef = mock_gaia_ref(self.mock_write)
with open(str(file), "wb+") as f:
f.write(b"f\x00\x00bar")
path = Path(file)
response = self.gaiaRef.data("gaia://tenant/somefolder").add("newFile", path)
assert pipe(ops.first())(response).run().uri == "gaia://tenant/somefolder/newFile"
def test_remove_nonexistent_file(self):
def mock(request):
self.assertEqual(request.url_post_fix, "/data/remove")
return MockResponse({"fileExisted": False})
self.gaiaRef = mock_gaia_ref(mock)
result = pipe(ops.first())(self.gaiaRef.data("gaia://tenant/somefolder/nonexistentFile").remove()).run()
self.assertEqual(result.file_existed, False)
def create():
def predicate(x):
if x < 4:
return False
else:
raise Exception(ex)
return xs.pipe(ops.first(predicate))
def test_first_async_one(self):
scheduler = TestScheduler()
xs = scheduler.create_hot_observable(on_next(150, 1), on_next(210, 2),
on_completed(250))
res = scheduler.start(lambda: xs.pipe(ops.first()))
assert res.messages == [on_next(210, 2), on_completed(210)]
assert xs.subscriptions == [subscribe(200, 210)]
def test_retrieve_data_as_bytes(self):
def mock(request):
self.assertEqual(request.url_post_fix, "/data/source")
return MockResponse(bytes("hello world", encoding="utf-8"))
self.gaiaRef = mock_gaia_ref(mock)
result: bytes = pipe(ops.first())(
self.gaiaRef.data("gaia://tenant/somefolder/somefolder/asdf1.pdf").as_bytes()).run()
self.assertEqual(result, bytes("hello world", "utf-8"))
def test_store(self):
store = create_store()
store_ = store.as_observable()
init_state_ = store_.pipe(
select(select_init_feature_module), filter(bool), first()
)
test_ = init_state_.pipe(
map(lambda state: self.assertEqual(state, "init")), first(),
)
store.add_feature_module(create_init_feature())
test_.subscribe()
store.on_completed()
def test_first_async_error(self):
ex = 'ex'
scheduler = TestScheduler()
xs = scheduler.create_hot_observable(on_next(150, 1),
on_error(210, ex))
res = scheduler.start(lambda: xs.pipe(ops.first()))
assert res.messages == [on_error(210, ex)]
assert xs.subscriptions == [subscribe(200, 210)]
def test_export_identity_mock(self):
def mock(request):
self.assertEqual(request.url_post_fix, "/identity/source")
return MockResponse(bytes("identity content", encoding="utf-8"))
self.gaiaRef = mock_gaia_ref(mock)
result = pipe(ops.first())(
self.gaiaRef.identity().export("00000000-0000-0000-0000-000000000000")).run()
self.assertEqual(result, bytes("identity content", "utf-8"))
def test_spot_instance_checker_through_404(requests_mock):
requests_mock.get(INSTANCE_ACTION_URL, text="test", status_code=404)
o = spot_instance_check_observable(0.1).pipe(ops.merge(rx.timer(0.5)),
ops.first())
def on_next(x):
assert x == 0
o.subscribe(on_next=on_next, scheduler=CurrentThreadScheduler())
def test_write_new_file_progress(self):
self.gaiaRef = mock_gaia_ref(self.mock_write)
data = bytes("234", encoding="utf-8")
def test_progress(value):
assert value == 100
config: DataRefRequestConfig = DataRefRequestConfig(test_progress)
response = self.gaiaRef.data("gaia://tenant/somefolder").add("newFile", data, False, config)
assert pipe(ops.first())(response).run().uri == "gaia://tenant/somefolder/newFile"
def test_list_files_in_existing_dir(self):
def mock(request):
self.assertEqual(request.url_post_fix, "/data/list")
return MockResponse([{"tenant": "tenant1", "filePath": "existingDirectory/file1",
"lastModified": "2020-11-18", "size": "1000"}])
self.gaiaRef = mock_gaia_ref(mock)
result = pipe(ops.first())(self.gaiaRef.data("gaia://tenant1/existingDirectory").list()).run()
self.assertEqual(len(result), 1)
self.assertEqual(result[0], FileListing({"tenant": "tenant1", "filePath": "existingDirectory/file1",
"lastModified": "2020-11-18", "size": "1000"}))
def _wait_for_order_fill(self,
market: Market,
client_id: int,
max_wait_in_seconds: int = 60):
self.logger.info(
f"Waiting up to {max_wait_in_seconds} seconds for {client_id}.")
return rx.interval(1.0).pipe(
ops.flat_map(lambda _: market.load_event_queue()),
ops.skip_while(lambda item: item.client_order_id != client_id),
ops.skip_while(lambda item: not item.event_flags.fill),
ops.first(), ops.map(lambda _: True),
ops.timeout(max_wait_in_seconds, rx.return_value(False))).run()
def test_retrieve_data_as_file(self):
file = Path(tempfile.gettempdir()) / "file"
content = b"binary data \x01\x00"
def mock(request):
self.assertEqual(request.url_post_fix, "/data/source")
return MockResponse(content)
self.gaiaRef = mock_gaia_ref(mock)
result: Path = pipe(ops.first())(
self.gaiaRef.data("gaia://tenant/somefolder/somefolder/file").as_file(str(file))).run()
self.assertEqual(result, file)
self.assertEqual(result.read_bytes(), content)
def test_does_not_block(self):
gaia_ref = mock_gaia_ref(self.return_after_seconds(5))
def config(x):
x.identity_id()
x.qualifier()
t0 = time.perf_counter()
pipe(ops.first())(gaia_ref.retrieve_intents(str(uuid4()), config))
t1 = time.perf_counter()
exec_time = t1 - t0
self.assertLess(exec_time, 5)
def test_spot_instance_checker_terminate(requests_mock):
body = {"action": "terminate", "time": "2017-09-18T08:22:00Z"}
requests_mock.get(INSTANCE_ACTION_URL, json=body)
o = spot_instance_check_observable(0.1).pipe(ops.merge(rx.timer(0.5)),
ops.first())
def on_next(x):
assert isinstance(x, CheckerMessage)
assert x.checker_type == "spot_instance"
assert x.body == f'"spot/instance-action": {body}'
o.subscribe(on_next=on_next, scheduler=CurrentThreadScheduler())
def test_perceive_action(self):
gaia_ref = mock_gaia_ref(lambda x: MockResponse(
{"data": {
"perceive": {
"perceiveAction": {
"id": "asdf"
}
}
}}))
impulse = PerceiveActionImpulse(False, str(uuid.uuid4()), "", {})
result = pipe(ops.first())(gaia_ref.perceive_action(impulse)).run()
assert result.dictionary.get(
"id") is not None, "PerceiveAction.id is in response"
def file_by_path(path: str) -> Observable:
root_stream = of({'id': 'root', 'path': '/'})
def find_with_parent(parent_stream, name):
return parent_stream.pipe(
flat_map(lambda parent: list_folder(parent, name_filter=name)),
map(lambda files: files[0] if len(files) else None),
flat_map(lambda file: of(file) if file else throw(
Exception('File {} does not exist.'.format(path)))))
return from_list(path.split('/')).pipe(
filter(lambda name: name and name.strip()
), # Allows double // and training /
reduce(find_with_parent, root_stream),
flat_map(lambda file_stream: file_stream.pipe(map(lambda file: file))),
first())
def test_type(self):
def reduce_to_list(dst: Iterable[int], src: int) -> Iterable:
return (*dst, src)
store = create_store()
store.add_feature_module(create_counter_feature())
result = BehaviorSubject(None)
store_: Observable[ReduxRootStore] = store.as_observable()
store_.pipe(operators.map(select_counter_feature),
operators.reduce(reduce_to_list, ()),
operators.first()).subscribe(result)
store.dispatch(INCREMENT_ACTION)
store.dispatch(DECREMENT_ACTION)
store.on_completed()
assert result.value == (0, 1, 0)
def cancellation_check(process: subprocess.Popen, interval: float,
ct: CancellationToken) -> rx.Observable:
"""Kills the given process if cancellation is requested from the
CancellationToken.
Args:
process: process that will killed if cancellation is requested
interval: cancellation check interval in seconds
ct: CancellationToken that is being checked
Return:
rx.Observable that fires a single dictionary after cancellation is
requested
"""
return rx.timer(0, interval).pipe(
ops.map(lambda _: MCERD._stop_if_cancelled(process, ct)),
ops.first(lambda x: not x),
ops.map(lambda _: {
MCERD.IS_RUNNING: False,
MCERD.MSG: MCERD.SIM_STOPPED
}),
)
def _first(predicate=None) -> Callable[[Observable], Observable]:
"""Returns the first element of an observable sequence that
satisfies the condition in the predicate if present else the first
item in the sequence.
Examples:
>>> res = res = first()(source)
>>> res = res = first(lambda x: x > 3)(source)
Args:
predicate -- [Optional] A predicate function to evaluate for
elements in the source sequence.
Returns:
A function that takes an observable source and returns an
observable sequence containing the first element in the
observable sequence that satisfies the condition in the predicate if
provided, else the first item in the sequence.
"""
if predicate:
return pipe(ops.filter(predicate), ops.first())
return _first_or_default_async(False)
def create():
return xs.pipe(ops.first(lambda x: x % 2 == 1))
def create():
return xs.pipe(ops.first(lambda x: x > 10))
def test_run_from_first(self):
result = rx.from_([1, 2, 3]).pipe(ops.first()).run()
assert result == 1
def create():
return xs.pipe(ops.first())
