def _delete(self, prop, extended, muted, cached): if cached is None: cached = self._use_cache if cached: if self._cached is None: self._load() k = extended and PropertyType.EXTENDED or PropertyType.DEFAULT vals = self._cached[k]["values"] if prop not in vals: raise AttributeError(prop) del vals[prop] for _set in ("created", "updated"): if prop in self._cached[k][_set]: self._cached[k][_set].remove(prop) self._cached[k]["deleted"].add(prop) else: klass, k = extended and (SubjectExtProperty, PropertyType.EXTENDED) or ( SubjectProperty, PropertyType.DEFAULT) self._storage.delete(klass(prop)) if self._cached is not None: if prop in self._cached[k]["values"]: del self._cached[k]["values"][prop] for _set in ["created", "updated", "deleted"]: if prop in self._cached[k][_set]: self._cached[k][_set].remove(prop) if not muted: payload = {PayloadConst.PROPERTY_NAME: prop} from krules_core.providers import event_router_factory from krules_core import types event_router_factory().route(types.SUBJECT_PROPERTY_DELETED, self, payload)
def test_with_self(): class WithSelfSet(RuleFunctionBase): def execute(self, arg1, arg2, arg3): self.payload["arg1"] = arg1 self.payload["arg2"] = arg2 self.payload["arg3"] = arg3 RuleFactory.create( "test-with-self", subscribe_to="test-argprocessors-self", data={ processing: [ WithSelfSet(lambda self: self.payload["value_from"], arg2=lambda self: self.subject.get("value_from"), arg3=lambda p: "I'll never be called") ] }) payload = {"value_from": 1} subject = subject_factory("test-1") subject.set("value_from", 2) event_router_factory().route("test-argprocessors-self", subject, payload) proc_events_rx_factory.subscribe(lambda x: x[ rulename] == "test-with-self" and _assert(x[processing][0]["args"][ 0] == 1 and x[processing][0]["kwargs"]["arg2"] == 2 and isinstance( x[processing][0]["kwargs"]["arg3"], str))) assert payload["arg1"] == 1 assert payload["arg2"] == 2 assert inspect.isfunction(payload["arg3"])
def publish_proc_events_filtered(result, jp_expr, expt_value, debug=False): if jp_expr is not None: if not isinstance(jp_expr, list): jp_expr = [jp_expr] for expr in jp_expr: if callable(expt_value): _pass = expt_value(jp.match1(f"$[?({expr})]", [result])) else: _pass = (jp.match1(f"$[?({expr})]", [result]) == expt_value) if not _pass: return data = result event_info = data["event_info"] result_subject = subject_factory(data[RuleConst.RULENAME], event_info=event_info) if debug and result["type"] != RULE_PROC_EVENT: dispatch_policy = DispatchPolicyConst.NEVER else: dispatch_policy = DispatchPolicyConst.DIRECT event_router_factory().route(RULE_PROC_EVENT, result_subject, data, dispatch_policy=dispatch_policy)
def router(): router = event_router_factory() router.unregister_all() proc_events_rx_factory.override( providers.Singleton(rx.subjects.ReplaySubject)) return event_router_factory()
def test_k8s_subject(api, namespace): RuleFactory.create( 'test-k8s-subject', subscribe_to='test-k8s-subject', data={ RuleConst.FILTERS: [ Filter(lambda: (k8s_subject( resource_path= f"/api/v1/namespaces/{namespace}/pods/test-pod-1").ext_name == "test-pod-1")), Filter( K8sObjectsQuery(returns=lambda obj: (k8s_subject( obj).ext_name == "test-pod-1"))), Filter( K8sObjectsQuery(returns=lambda obj: (k8s_subject( obj.obj).ext_name == "test-pod-1"))) ], RuleConst.PROCESSING: [Process(True)] }) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-subject' and _assert( x[RuleConst.GOT_ERRORS] is False and x[RuleConst.PROCESSED], "test-k8s-subject proc failed")) event_router_factory().route( "test-k8s-subject", f"k8s:/api/v1/namespaces/{namespace}/pods/test-pod-1", {})
def test_extend_jp_match(): import jsonpath_rw_ext as jp from krules_core.arg_processors import processors class JPMatchSet(RuleFunctionBase): def execute(self, values, arg2): self.payload["values"] = values self.payload["elem-2"] = arg2 class JPPayloadMatchBase: def __init__(self, expr): self._expr = expr def match(self, instance): raise NotImplementedError() class jp_match(JPPayloadMatchBase): def match(self, instance): return jp.match(self._expr, instance.payload) class jp_match1(JPPayloadMatchBase): def match(self, instance): return jp.match1(self._expr, instance.payload) class JPProcessor(BaseArgProcessor): @staticmethod def interested_in(arg): return isinstance(arg, JPPayloadMatchBase) def process(self, instance): return self._arg.match(instance) processors.append(JPProcessor) RuleFactory.create("test-with-jp-expr", subscribe_to="test-argprocessors-jp-match", data={ processing: [ JPMatchSet(jp_match("$.elems[*].value"), jp_match1("$.elems[?id==2]")) ] }) payload = {"elems": [{"id": 1, "value": "a"}, {"id": 2, "value": "b"}]} event_router_factory().route("test-argprocessors-jp-match", "test-0", payload) proc_events_rx_factory.subscribe( lambda x: x[rulename] == "test-with-jp-expr" and _assert(x[processing][ 0]["args"][0] == ['a', 'b'] and x[processing][0]["args"][1] == { "id": 2, "value": "b" })) assert payload["values"] == ['a', 'b'] assert payload["elem-2"]["id"] == 2 and payload["elem-2"]["value"] == "b"
def _set(self, prop, value, extended, muted, cached): if cached is None: cached = self._use_cache if cached: if self._cached is None: self._load() kprops = extended and PropertyType.EXTENDED or PropertyType.DEFAULT vals = extended and self._cached[kprops]["values"] or self._cached[ kprops]["values"] if prop in vals: self._cached[kprops]["updated"].add(prop) else: self._cached[kprops]["created"].add(prop) try: old_value = vals[prop] except KeyError: old_value = None if inspect.isfunction(value): n_params = len(inspect.signature(value).parameters) if n_params == 0: value = value() elif n_params == 1: value = value(old_value) else: raise ValueError("to many arguments for {}".format(prop)) vals[prop] = value else: klass, k = extended and (SubjectExtProperty, PropertyType.EXTENDED) or ( SubjectProperty, PropertyType.DEFAULT) value, old_value = self._storage.set(klass(prop, value)) # update cached if self._cached: self._cached[k]["values"][prop] = value if prop in self._cached[k]["created"]: self._cached[k]["created"].remove(prop) if prop in self._cached[k]["updated"]: self._cached[k]["updated"].remove(prop) if prop in self._cached[k]["deleted"]: self._cached[k]["deleted"].remove(prop) if not muted and value != old_value: payload = { PayloadConst.PROPERTY_NAME: prop, PayloadConst.OLD_VALUE: old_value, PayloadConst.VALUE: value } from krules_core.providers import event_router_factory from krules_core import types event_router_factory().route(types.SUBJECT_PROPERTY_CHANGED, self, payload) return value, old_value
def publish_proc_events_all(result): data = result # data = json.loads(json.dumps(result, cls=_JSONEncoder).encode("utf-8")) event_info = data.get("event_info", {}) result_subject = subject_factory(data[RuleConst.RULENAME], event_info=event_info) event_router_factory().route( RULE_PROC_EVENT, result_subject, data, dispatch_policy=DispatchPolicyConst.DIRECT )
def set_multus_resources(obj, payload): app = obj.obj.get("metadata").get("labels", {}).get("app") if app is not None: event_router_factory().route( "k8s.resource.detector", "k8s:{}".format(obj.obj["metadata"]["selfLink"]), obj.obj, dispatch_policy=DispatchPolicyConst.DIRECT) payload[app] = payload.get(app, []) payload[app].append(obj)
def publish_proc_events_errors(result): if not result.get("got_errors", False): return data = result # data = json.loads(json.dumps(result, cls=_JSONEncoder).encode("utf-8")) event_info = data["event_info"] result_subject = subject_factory(data[RuleConst.RULENAME], event_info=event_info) event_router_factory().route( RULE_PROC_EVENT, result_subject, data, dispatch_policy=DispatchPolicyConst.DIRECT )
def test_dispatched_event(fake_receiver): from krules_core import event_types event_dispatcher_factory.override( providers.Singleton(lambda: CloudEventsDispatcher( fake_receiver.url, "pytest", test=True))) router = event_router_factory() subject = subject_factory("test-subject") subject.set_ext("ext1", "val1") subject.set_ext("ext2", "2") _id, code, sent_headers = router.route("test-type", subject, {"key1": "hello"}) assert (200 <= code < 300) assert (sent_headers.get("ce-id") == _id) assert (sent_headers.get("ce-source") == "pytest") assert (sent_headers.get("ce-subject") == "test-subject") assert (sent_headers.get("ce-type") == "test-type") assert (sent_headers.get("ce-Originid") == _id) assert (sent_headers.get("ce-ext1") == "val1") assert (sent_headers.get("ce-ext2") == "2") # with event info subject = subject_factory("test-subject", event_info={"originid": 1234}) _, _, sent_headers = router.route("test-type", subject, {"key1": "hello"}) assert (sent_headers.get("id") != sent_headers.get("ce-Originid")) assert (sent_headers.get("ce-Originid") == '1234') # property name _, _, sent_headers = router.route(event_types.SUBJECT_PROPERTY_CHANGED, subject, {PayloadConst.PROPERTY_NAME: "foo"}) assert (sent_headers.get("ce-propertyname") == 'foo')
def __init__( self, import_name, static_url_path=None, static_folder="static", static_host=None, host_matching=False, subdomain_matching=False, template_folder="templates", instance_path=None, instance_relative_config=False, root_path=None, ): super().__init__( import_name, static_url_path, static_folder, static_host, host_matching, subdomain_matching, template_folder, instance_path, instance_relative_config, root_path ) json_logging.ENABLE_JSON_LOGGING = True json_logging.init_flask(enable_json=True) json_logging.init_request_instrument(self) self.logger = logging.getLogger(self.name) self.logger.setLevel(int(os.environ.get("LOGGING_LEVEL", logging.INFO))) self.logger.addHandler(logging.StreamHandler(sys.stdout)) self.logger.propagate = False self.logger_core = logging.getLogger("__core__") self.logger_core.setLevel(int(os.environ.get("CORE_LOGGING_LEVEL", logging.ERROR))) self.logger_core.addHandler(logging.StreamHandler(sys.stdout)) self.logger_core.propagate = False self.logger_router = logging.getLogger("__router__") self.logger_router.setLevel(int(os.environ.get("ROUTER_LOGGING_LEVEL", logging.ERROR))) self.logger_router.addHandler(logging.StreamHandler(sys.stdout)) self.logger_router.propagate = False self.req_logger = logging.getLogger("flask-request-logger") self.req_logger.setLevel(logging.ERROR) self.req_logger.propagate = False init() try: import env env.init() except ImportError: self.logger.warning("No app env defined!") subject_factory.override(providers.Factory(lambda *args, **kw: g_wrap(subject_factory.cls, *args, **kw))) self.router = event_router_factory()
def publish_proc_events_filtered(result, jp_expr, expt_value): if callable(expt_value): _pass = expt_value(jp.match1(jp_expr, result)) else: _pass = (jp.match1(jp_expr, result) == expt_value) if not _pass: return data = result # data = json.loads(json.dumps(result, cls=_JSONEncoder).encode("utf-8")) event_info = data["event_info"] result_subject = subject_factory(data[RuleConst.RULENAME], event_info=event_info) event_router_factory().route( RULE_PROC_EVENT, result_subject, data, dispatch_policy=DispatchPolicyConst.DIRECT )
def tests_query_foreach(api, namespace): RuleFactory.create( 'test-k8s-query-foreach', subscribe_to="test-query-foreach", data={ RuleConst.PROCESSING: [ K8sObjectsQuery(foreach=lambda obj: (obj.obj["metadata"][ "labels"].update({"updated": "foreach"}), obj.update()), apiversion="v1", kind="Pod", namespace=namespace, selector={"app": "pytest-temp"}), ] }) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-query-foreach' and _assert(x[RuleConst.GOT_ERRORS] is False and x[RuleConst.PROCESSED], "test-k8s-query-foreach proc failed")) watch = pykube.Pod.objects(api).filter(namespace=namespace, selector={ "app": "pytest-temp" }).watch() event_router_factory().route("test-query-foreach", "none", {}) try: updated = 0 for ev in watch: if ev.type == 'ADDED' and ev.object.obj["metadata"]["labels"][ "updated"] == "foreach": updated += 1 if updated == 2: break except: assert False, "query update event failed" assert updated == 2 event_router_factory().unregister_all()
def test_callable_dispatch_url(fake_receiver): def _get_dispatch_url(subject, type): assert not isinstance(subject, str) return fake_receiver.url event_dispatcher_factory.override( providers.Singleton(lambda: CloudEventsDispatcher( _get_dispatch_url, "pytest", test=True))) router = event_router_factory() _id, code, sent_headers = router.route("test-type", "test_subject", {"key1": "hello"}) assert (200 <= code < 300)
def test_simple_callable(): class SimpleSet(RuleFunctionBase): def execute(self, arg1, arg2, arg3, arg4, arg5): self.payload["arg1"] = arg1 self.payload["arg2"] = arg2 self.payload["arg3"] = arg3 self.payload["arg4"] = arg4 self.payload["arg5"] = arg5 RuleFactory.create( "test-simple-callable", subscribe_to="test-argprocessors-callables", data={ processing: [ SimpleSet(lambda: 1, 2, arg3=lambda: 3, arg4=4, arg5=lambda p: "I'll never be called") ] } ) payload = {} event_router_factory().route("test-argprocessors-callables", "test-0", payload) proc_events_rx_factory().subscribe( lambda x: x[rulename] == "test-simple-callable" and _assert( x[processing][0]["args"][0] == 1 and x[processing][0]["kwargs"]["arg3"] == 3 and x[processing][0]["kwargs"]["arg4"] == 4 and isinstance(x[processing][0]["kwargs"]["arg5"], str) ) ) assert payload["arg1"] == 1 assert payload["arg2"] == 2 assert payload["arg3"] == 3 assert payload["arg4"] == 4 assert inspect.isfunction(payload["arg5"])
def test_delete(api, namespace): RuleFactory.create( 'test-k8s-delete-in-context', subscribe_to="test-delete-in-context", data={ RuleConst.PROCESSING: [ # update from subject K8sObjectDelete() ] }) objs = pykube.Pod.objects(api).filter(namespace=namespace, selector={"app": "pytest-temp"}) #watch = objs.watch().filter(field_selector={"metadata.name": "test-pod-1"}) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-delete-in-context' and _assert(x[RuleConst.GOT_ERRORS] is False and x[RuleConst.PROCESSED], "test-k8s-delete-in-context proc failed")) event_router_factory().route( "test-delete-in-context", f"k8s:/api/v1/namespaces/{namespace}/pods/test-pod-1", {}) # for ev in watch: # if ev.type == "DELETED": # break # assert len(objs) == 1 RuleFactory.create( 'test-k8s-delete-off-context', subscribe_to="test-delete-off-context", data={ RuleConst.PROCESSING: [ # update from subject K8sObjectDelete(name="test-pod-2", apiversion="v1", kind="Pod", namespace=namespace) ] }) objs = pykube.Pod.objects(api).filter(namespace=namespace, selector={"app": "pytest-temp"}) watch = objs.watch().filter(field_selector={"metadata.name": "test-pod-2"}) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-delete-off-context' and _assert(x[RuleConst.GOT_ERRORS] is False and x[RuleConst.PROCESSED], "test-k8s-delete-off-context proc failed")) event_router_factory().route("test-delete-off-context", f"k8s:/api/v1/namespaces/{namespace}", {}) # for ev in watch: # if ev.type == "DELETED": # break # assert len(objs) == 0 event_router_factory().unregister_all()
def execute(self, message=None, subject=None, payload=None, hash=None, when=lambda _: datetime.now(), replace=False): if message is None: message = self.message if subject is None: subject = self.subject if payload is None: payload = self.payload if str(self.subject) != str(subject): subject = subject_factory(str(subject), event_info=self.subject.event_info()) if callable(when): when = when(self) if type(when) is not str: when = when.isoformat() new_payload = { "message": message, "subject": str(subject), "payload": payload, "when": when, "replace": replace } event_router_factory().route( "schedule-message", subject, new_payload, dispatch_policy=DispatchPolicyConst.DIRECT)
def execute(self, type=None, subject=None, payload=None, dispatch_policy=DispatchPolicyConst.DEFAULT): """ Args: type: The event type. Default is current processing event type subject: New subject or the current subject as default payload: The payload of the event or the current payload dispatch_policy: Router -> dispatcher policy. Available choices are defined in krules_core.route.router.DispatchPolicyConst as: DEFAULT: Dispatched outside only when no handler is found in current ruleset ALWAYS: Always dispatched even if an handler is found and processed in the current ruleset NEVER: Never dispatched outside DIRECT: Skip to search for a local handler and send outside directly """ from krules_core.providers import event_router_factory if type is None: type = self.type if subject is None: subject = self.subject if payload is None: payload = self.payload event_router_factory().route(type, subject, payload, dispatch_policy=dispatch_policy)
def test_with_payload_and_subject(): class WithPayloadSet(RuleFunctionBase): def execute(self, arg1, arg2, arg3): self.payload["arg1"] = arg1 self.payload["arg2"] = arg2 self.payload["arg3"] = arg3 RuleFactory.create( "test-with-payload-and-subject", subscribe_to="test-argprocessors-payload-and-subject", data={ processing: [ WithPayloadSet(lambda payload: payload["value_from"], arg2=lambda subject: subject.get("value_from"), arg3=lambda p: "I'll never be called") ] }) _payload = {"value_from": 1} _subject = subject_factory("test-1") _subject.set("value_from", 2) event_router_factory().route("test-argprocessors-payload-and-subject", _subject, _payload) proc_events_rx_factory().subscribe( lambda x: x[rulename] == "test-with-payload-and-subject" and _assert(x[ processing][0]["args"][0] == 1 and x[processing][0][ "kwargs"]["arg2"] == 2 and hasattr( x[processing][0]["kwargs"]["arg3"], "__call__"))) assert _payload["arg1"] == 1 assert _payload["arg2"] == 2 assert inspect.isfunction(_payload["arg3"])
def test_router(): subject = "test-subject" proc_events_rx_factory.queue.clear() start_time = datetime.now() router = event_router_factory() router.unregister_all() RuleFactory.create('test-empty-rule', subscribe_to="some-type", data={}) proc_events_rx_factory.subscribe(lambda x: _assert( x[RuleConst.TYPE] == "some-type" and "key1" in x[RuleConst.PAYLOAD] and x[RuleConst.PAYLOAD]["key1"] == "val1", )) router.route('some-type', subject, {"key1": "val1"}) end_time = datetime.now() logging.getLogger().debug("######### {}".format(end_time - start_time)) assert router.unregister_all() == 1, "Expected 1 element"
def router(): router = event_router_factory() router.unregister_all() proc_events_rx_factory.queue.clear() return event_router_factory()
def test_update(api, namespace): RuleFactory.create( 'test-k8s-update-in-context', subscribe_to="test-update-in-context", data={ RuleConst.PROCESSING: [ # update from subject K8sObjectUpdate(lambda obj: (obj["metadata"]["labels"].update( {"updated": "pod-1"}))), ] }) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-update-in-context' and _assert(x[RuleConst.PROCESSED] and x[RuleConst.GOT_ERRORS] is False, "test-k8s-update-in-context proc failed")) objs = pykube.Pod.objects(api).filter(namespace=namespace, selector={"app": "pytest-temp"}) watch = objs.watch().filter(field_selector={"metadata.name": "test-pod-1"}) event_router_factory().route( "test-update-in-context", f"k8s:/api/v1/namespaces/{namespace}/pods/test-pod-1", {}) try: for ev in watch: if ev.type == 'ADDED' and ev.object.obj["metadata"]["labels"][ "updated"] == "pod-1": break except: assert False, "update pod-1 event failed" RuleFactory.create( 'test-k8s-update-off-context', subscribe_to="test-update-off-context", data={ RuleConst.PROCESSING: [ # update from subject K8sObjectUpdate( lambda obj: (obj["metadata"]["labels"].update({"updated": "pod-2"})), name="test-pod-2", apiversion="v1", kind="Pod", namespace=namespace), ] }) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-update-off-context' and _assert(x[RuleConst.GOT_ERRORS] is False and x[RuleConst.PROCESSED], "test-k8s-update-off-context proc failed")) objs = pykube.Pod.objects(api).filter(namespace=namespace, selector={"app": "pytest-temp"}) watch = objs.watch().filter(field_selector={"metadata.name": "test-pod-2"}) event_router_factory().route("test-update-off-context", "none", {}) try: for ev in watch: if ev.type == 'ADDED' and ev.object.obj["metadata"]["labels"][ "updated"] == "pod-2": break except: assert False, "update pod-2 event failed" event_router_factory().unregister_all()
def main(): start_time = datetime.now() try: dispatch_policy = os.environ.get("DISPATCH_POLICY", DispatchPolicyConst.NEVER) m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v1.Event(), request.headers, io.BytesIO(request.data), lambda x: json.load(x)) event_info = event.Properties() event_info.update(event_info.pop("extensions")) event_data = event_info.pop("data") app.logger.debug("RCVR: {}".format(event_data)) type = event_info.get("type") subject = event_info.get("subject", "sys-0") g.subjects = [] # TODO: important!! # need to find a way to avoid a return of messages from the same service # (for example when resending it again after intercepted in the first time) # this workaround only works when in a knative service or at least when SOURCE environment # variable is set if event_info["source"] == os.environ.get("K_SERVICE", os.environ.get("SOURCE")): return Response(status=201) event_info["originid"] = event_info.get("originid", event_info.get("id")) logger.debug("subject: {}".format(subject)) logger.debug("event_data: {}".format(event_data)) from dependency_injector import providers subject = subject_factory(name=subject, event_info=event_info, event_data=event_data) event_data["_event_info"] = event_info # TODO: KRUL-155 try: event_router_factory().route(type, subject, event_data, dispatch_policy=dispatch_policy) finally: for sub in g.subjects: sub.store() exec_time = (datetime.now() - start_time).total_seconds() logger.info( "Event", extra={ 'props': { 'event_info': event_info, 'type': type, 'subject': subject.name, 'exec_time': exec_time, #'headers': list(headers.keys()) } }) return Response(status=200) except Exception as ex: app.logger.error(ex, exc_info=True) return Response(status=201)
def test_create(api, namespace): RuleFactory.create('test-k8s-create', subscribe_to="test-create", data={ RuleConst.PROCESSING: [ K8sObjectCreate({ "apiVersion": "v1", "kind": "Pod", "metadata": { "name": "test-pod-1", "labels": { "app": "pytest-temp" } }, "spec": { "containers": [{ "name": "hello", "image": "karthequian/helloworld" }] } }), K8sObjectCreate({ "apiVersion": "v1", "kind": "Pod", "metadata": { "name": "test-pod-2", "labels": { "app": "pytest-temp" } }, "spec": { "containers": [{ "name": "hello", "image": "karthequian/helloworld" }] } }) ] }) proc_events_rx_factory().subscribe( lambda x: x[RuleConst.RULENAME] == 'test-k8s-create' and _assert( not x[RuleConst.GOT_ERRORS] and x[RuleConst.PROCESSED], "test-k8s-create proc failed")) event_router_factory().route("test-create", "some", {}) pykube.Pod.objects(api).filter(namespace=namespace) objs = pykube.Pod.objects(api).filter(namespace=namespace, selector={"app": "pytest-temp"}) # wait for pods ready # watch = objs.watch() # ready = 0 # for ev in watch: # print(ev.type, ev.object.name, ev.object.ready) # if ev.object.ready: # ready += 1 # if ready == 2: # break # assert len(objs) == 2 event_router_factory().unregister_all()