def test_import_identity_e2e(self): self.gaiaRef = Gaia.login("http://localhost:8080", UsernamePasswordCredentials("admin", "admin")) data = bytes("identity content", encoding="utf-8") result = pipe(ops.first())( self.gaiaRef.identity().import_identity('2fa4ff18-5c30-497b-9ad2-0d8eb51cd4da', 'identityName', data, True)).run() self.assertEqual(len(result.uri), 72) result = pipe(ops.first())( self.gaiaRef.data("gaia://2fa4ff18-5c30-497b-9ad2-0d8eb51cd4da/").list()).run() self.assertEqual(result[0].dictionary, {'filePath': 'identities/identityName', 'tenant': '2fa4ff18-5c30-497b-9ad2-0d8eb51cd4da'}) pass
def test_perceive(self): gaia_ref = mock_gaia_ref(lambda x: MockResponse({ "data": { "perceive": { "perceiveData": { "id": "asdf" }, "perceiveAction": { "id": "qwer" } } } })) impulse1 = PerceiveActionImpulse(False, str(uuid.uuid4()), "", {}) impulse2 = PerceiveDataImpulse(str(uuid.uuid4()), "", {}) def perceive(p: Perception): p.perceive_action(impulse1, lambda x: x.id()) p.perceive_data(impulse2, lambda x: x.id()) result = pipe(ops.first())(gaia_ref.perceive(perceive)).run() perceiveData = result.dictionary.get("perceiveData") perceiveAction = result.dictionary.get("perceiveAction") assert perceiveData.get( "id") is not None, "PerceiveData.id is in response" assert perceiveAction.get( "id") is not None, "perceiveAction.id is in response"
def test_export_identity_no_id(self): def mock(request): self.assertEqual(request.url_post_fix, "/identity/source") return MockResponse(bytes("identity content", encoding="utf-8")) self.gaiaRef = mock_gaia_ref(mock) self.assertRaises(TypeError, lambda: pipe(ops.first())(self.gaiaRef.identity().export()).run())
def find_with_parent(parent, name): return (list_folder( credentials, parent, name_filter=name ) if parent else of([])).pipe( first( ), # There might be more than one file with the same name - we just take the first map(lambda files: files[0] if len(files) else None))
def _first( predicate: Optional[Predicate] = None ) -> Callable[[Observable], Observable]: """Returns the first element of an observable sequence that satisfies the condition in the predicate if present else the first item in the sequence. Examples: >>> res = res = first()(source) >>> res = res = first(lambda x: x > 3)(source) Args: predicate -- [Optional] A predicate function to evaluate for elements in the source sequence. Returns: A function that takes an observable source and returns an observable sequence containing the first element in the observable sequence that satisfies the condition in the predicate if provided, else the first item in the sequence. """ if predicate: return pipe(ops.filter(predicate), ops.first()) return _first_or_default_async(False)
def model_publisher(scheduler, sources): file_source = sources.file.response.pipe(ops.share()) # kafka driver bootstrap. fixme kafka_source = sources.kafka.response.pipe( ops.do_action(print), ops.replay(), ops.ref_count(), ) kafka_source.subscribe() config, config_read_request, http_request = read_config_from_args( sources.argv.argv, file_source, sources.http.response, scheduler=scheduler) config = config.pipe(ops.first()) kafka_request = config.pipe(ops.map(lambda c: create_model_topics(c)), ) return ModelPublisherSink( file=file.Sink(request=rx.merge(config_read_request)), http=http.Sink(request=http_request), kafka=kafka.Sink(request=kafka_request), )
def timeout_check(process: subprocess.Popen, timeout: float, ct: CancellationToken) -> rx.Observable: """Kills the given process after timeout has passed. Args: process: process to be killed timeout: termination time in seconds ct: CancellationToken Return: rx.Observable that fires only a single dictionary after the timeout has passed. """ return rx.timer(timeout).pipe( ops.do_action( # Request cancellation so all simulation processes that # share the same cancellation_token are also stopped. # TODO not working as intended if simulation is short enough # to stop before max_time has elapsed. Maybe let caller # implement its own timeout check when multiple processes # are being run. on_next=lambda _: ct.request_cancellation()), ops.map( lambda _: { # sutils.kill_process returns None, which can be casted to bool MCERD.IS_RUNNING: bool(sutils.kill_process(process)), MCERD.MSG: MCERD.SIM_TIMEOUT }), ops.first())
def test_list_files_in_nonexistent_dir(self): def mock(request): self.assertEqual(request.url_post_fix, "/data/list") return MockResponse([]) self.gaiaRef = mock_gaia_ref(mock) result = pipe(ops.first())(self.gaiaRef.data("gaia://tenant1/nonexistentDirectory").list()).run() self.assertEqual(len(result), 0)
def test_write_new_file_with_path(self): file = Path(tempfile.gettempdir()) / "file" self.gaiaRef = mock_gaia_ref(self.mock_write) with open(str(file), "wb+") as f: f.write(b"f\x00\x00bar") path = Path(file) response = self.gaiaRef.data("gaia://tenant/somefolder").add("newFile", path) assert pipe(ops.first())(response).run().uri == "gaia://tenant/somefolder/newFile"
def test_remove_nonexistent_file(self): def mock(request): self.assertEqual(request.url_post_fix, "/data/remove") return MockResponse({"fileExisted": False}) self.gaiaRef = mock_gaia_ref(mock) result = pipe(ops.first())(self.gaiaRef.data("gaia://tenant/somefolder/nonexistentFile").remove()).run() self.assertEqual(result.file_existed, False)
def create(): def predicate(x): if x < 4: return False else: raise Exception(ex) return xs.pipe(ops.first(predicate))
def test_first_async_one(self): scheduler = TestScheduler() xs = scheduler.create_hot_observable(on_next(150, 1), on_next(210, 2), on_completed(250)) res = scheduler.start(lambda: xs.pipe(ops.first())) assert res.messages == [on_next(210, 2), on_completed(210)] assert xs.subscriptions == [subscribe(200, 210)]
def test_retrieve_data_as_bytes(self): def mock(request): self.assertEqual(request.url_post_fix, "/data/source") return MockResponse(bytes("hello world", encoding="utf-8")) self.gaiaRef = mock_gaia_ref(mock) result: bytes = pipe(ops.first())( self.gaiaRef.data("gaia://tenant/somefolder/somefolder/asdf1.pdf").as_bytes()).run() self.assertEqual(result, bytes("hello world", "utf-8"))
def test_store(self): store = create_store() store_ = store.as_observable() init_state_ = store_.pipe( select(select_init_feature_module), filter(bool), first() ) test_ = init_state_.pipe( map(lambda state: self.assertEqual(state, "init")), first(), ) store.add_feature_module(create_init_feature()) test_.subscribe() store.on_completed()
def test_first_async_error(self): ex = 'ex' scheduler = TestScheduler() xs = scheduler.create_hot_observable(on_next(150, 1), on_error(210, ex)) res = scheduler.start(lambda: xs.pipe(ops.first())) assert res.messages == [on_error(210, ex)] assert xs.subscriptions == [subscribe(200, 210)]
def test_export_identity_mock(self): def mock(request): self.assertEqual(request.url_post_fix, "/identity/source") return MockResponse(bytes("identity content", encoding="utf-8")) self.gaiaRef = mock_gaia_ref(mock) result = pipe(ops.first())( self.gaiaRef.identity().export("00000000-0000-0000-0000-000000000000")).run() self.assertEqual(result, bytes("identity content", "utf-8"))
def test_spot_instance_checker_through_404(requests_mock): requests_mock.get(INSTANCE_ACTION_URL, text="test", status_code=404) o = spot_instance_check_observable(0.1).pipe(ops.merge(rx.timer(0.5)), ops.first()) def on_next(x): assert x == 0 o.subscribe(on_next=on_next, scheduler=CurrentThreadScheduler())
def test_write_new_file_progress(self): self.gaiaRef = mock_gaia_ref(self.mock_write) data = bytes("234", encoding="utf-8") def test_progress(value): assert value == 100 config: DataRefRequestConfig = DataRefRequestConfig(test_progress) response = self.gaiaRef.data("gaia://tenant/somefolder").add("newFile", data, False, config) assert pipe(ops.first())(response).run().uri == "gaia://tenant/somefolder/newFile"
def test_list_files_in_existing_dir(self): def mock(request): self.assertEqual(request.url_post_fix, "/data/list") return MockResponse([{"tenant": "tenant1", "filePath": "existingDirectory/file1", "lastModified": "2020-11-18", "size": "1000"}]) self.gaiaRef = mock_gaia_ref(mock) result = pipe(ops.first())(self.gaiaRef.data("gaia://tenant1/existingDirectory").list()).run() self.assertEqual(len(result), 1) self.assertEqual(result[0], FileListing({"tenant": "tenant1", "filePath": "existingDirectory/file1", "lastModified": "2020-11-18", "size": "1000"}))
def _wait_for_order_fill(self, market: Market, client_id: int, max_wait_in_seconds: int = 60): self.logger.info( f"Waiting up to {max_wait_in_seconds} seconds for {client_id}.") return rx.interval(1.0).pipe( ops.flat_map(lambda _: market.load_event_queue()), ops.skip_while(lambda item: item.client_order_id != client_id), ops.skip_while(lambda item: not item.event_flags.fill), ops.first(), ops.map(lambda _: True), ops.timeout(max_wait_in_seconds, rx.return_value(False))).run()
def test_retrieve_data_as_file(self): file = Path(tempfile.gettempdir()) / "file" content = b"binary data \x01\x00" def mock(request): self.assertEqual(request.url_post_fix, "/data/source") return MockResponse(content) self.gaiaRef = mock_gaia_ref(mock) result: Path = pipe(ops.first())( self.gaiaRef.data("gaia://tenant/somefolder/somefolder/file").as_file(str(file))).run() self.assertEqual(result, file) self.assertEqual(result.read_bytes(), content)
def test_does_not_block(self): gaia_ref = mock_gaia_ref(self.return_after_seconds(5)) def config(x): x.identity_id() x.qualifier() t0 = time.perf_counter() pipe(ops.first())(gaia_ref.retrieve_intents(str(uuid4()), config)) t1 = time.perf_counter() exec_time = t1 - t0 self.assertLess(exec_time, 5)
def test_spot_instance_checker_terminate(requests_mock): body = {"action": "terminate", "time": "2017-09-18T08:22:00Z"} requests_mock.get(INSTANCE_ACTION_URL, json=body) o = spot_instance_check_observable(0.1).pipe(ops.merge(rx.timer(0.5)), ops.first()) def on_next(x): assert isinstance(x, CheckerMessage) assert x.checker_type == "spot_instance" assert x.body == f'"spot/instance-action": {body}' o.subscribe(on_next=on_next, scheduler=CurrentThreadScheduler())
def test_perceive_action(self): gaia_ref = mock_gaia_ref(lambda x: MockResponse( {"data": { "perceive": { "perceiveAction": { "id": "asdf" } } }})) impulse = PerceiveActionImpulse(False, str(uuid.uuid4()), "", {}) result = pipe(ops.first())(gaia_ref.perceive_action(impulse)).run() assert result.dictionary.get( "id") is not None, "PerceiveAction.id is in response"
def file_by_path(path: str) -> Observable: root_stream = of({'id': 'root', 'path': '/'}) def find_with_parent(parent_stream, name): return parent_stream.pipe( flat_map(lambda parent: list_folder(parent, name_filter=name)), map(lambda files: files[0] if len(files) else None), flat_map(lambda file: of(file) if file else throw( Exception('File {} does not exist.'.format(path))))) return from_list(path.split('/')).pipe( filter(lambda name: name and name.strip() ), # Allows double // and training / reduce(find_with_parent, root_stream), flat_map(lambda file_stream: file_stream.pipe(map(lambda file: file))), first())
def test_type(self): def reduce_to_list(dst: Iterable[int], src: int) -> Iterable: return (*dst, src) store = create_store() store.add_feature_module(create_counter_feature()) result = BehaviorSubject(None) store_: Observable[ReduxRootStore] = store.as_observable() store_.pipe(operators.map(select_counter_feature), operators.reduce(reduce_to_list, ()), operators.first()).subscribe(result) store.dispatch(INCREMENT_ACTION) store.dispatch(DECREMENT_ACTION) store.on_completed() assert result.value == (0, 1, 0)
def cancellation_check(process: subprocess.Popen, interval: float, ct: CancellationToken) -> rx.Observable: """Kills the given process if cancellation is requested from the CancellationToken. Args: process: process that will killed if cancellation is requested interval: cancellation check interval in seconds ct: CancellationToken that is being checked Return: rx.Observable that fires a single dictionary after cancellation is requested """ return rx.timer(0, interval).pipe( ops.map(lambda _: MCERD._stop_if_cancelled(process, ct)), ops.first(lambda x: not x), ops.map(lambda _: { MCERD.IS_RUNNING: False, MCERD.MSG: MCERD.SIM_STOPPED }), )
def _first(predicate=None) -> Callable[[Observable], Observable]: """Returns the first element of an observable sequence that satisfies the condition in the predicate if present else the first item in the sequence. Examples: >>> res = res = first()(source) >>> res = res = first(lambda x: x > 3)(source) Args: predicate -- [Optional] A predicate function to evaluate for elements in the source sequence. Returns: A function that takes an observable source and returns an observable sequence containing the first element in the observable sequence that satisfies the condition in the predicate if provided, else the first item in the sequence. """ if predicate: return pipe(ops.filter(predicate), ops.first()) return _first_or_default_async(False)
def create(): return xs.pipe(ops.first(lambda x: x % 2 == 1))
def create(): return xs.pipe(ops.first(lambda x: x > 10))
def test_run_from_first(self): result = rx.from_([1, 2, 3]).pipe(ops.first()).run() assert result == 1
def create(): return xs.pipe(ops.first())