def test_from_request_cannot_read(binary_headers): with pytest.raises(exceptions.UnsupportedEventConverter): m = marshaller.HTTPMarshaller([binary.NewBinaryHTTPCloudEventConverter()]) m.FromRequest(v1.Event(), {}, "") with pytest.raises(exceptions.UnsupportedEventConverter): m = marshaller.HTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()]) m.FromRequest(v1.Event(), binary_headers, "")
def target(): # Deserialize the HTTP request into a CloudEvent try: logging.info('Deserializing CloudEvent in binary format') # Instantiate the Binary Format marshaller mb = marshaller.NewHTTPMarshaller( [binary.NewBinaryHTTPCloudEventConverter()]) ce = mb.FromRequest(v1.Event(), request.headers, request.stream.read(), lambda x: x) except: logging.warning( 'Deserializing application/cloudevents+json CloudEvent') # Instantiate the JSON Structured CloudEvent marshaller m = marshaller.NewHTTPMarshaller( [structured.NewJSONHTTPCloudEventConverter()]) ce = m.FromRequest() except: logging.warning('Could not deserialize CloudEvent') # Create a CloudEvent to send as response data = 'Data received was not understandable as a CloudEvent' event = ( v1.Event().SetContentType("text/plain").SetData(data).SetEventID( str(uuid.uuid4())).SetSource("from_your_own_target"). SetSubject("your_event_subject").SetEventTime( datetime.now( timezone.utc).astimezone().isoformat()).SetEventType( "io.triggermesh.target.byown")) return app.response_class(response=body, headers=headers, status=400) # Do your Transformation or Target work based on the eventype if ce.EventType() == "io.triggermesh.byown.create": logging.info("Create event type") elif ce.EventType() == "io.triggermesh.byown.delete": logging.info("Delete event type") else: logging.warning("Unknown event type %s" % ce.EventType()) # Create a CloudEvent to send as response data = 'this is some data' event = (v1.Event().SetContentType("text/plain").SetData(data).SetEventID( str(uuid.uuid4())).SetSource("from_your_own_target").SetSubject( "your_event_subject").SetEventTime( datetime.now( timezone.utc).astimezone().isoformat()).SetEventType( "io.triggermesh.target.byown")) # Prepare the Header and Body to send a request back as a CloudEvent m = marshaller.NewHTTPMarshaller( [structured.NewJSONHTTPCloudEventConverter()]) headers, body = m.ToRequest(event, converters.TypeStructured, lambda x: x) return app.response_class(response=body, headers=headers, status=200)
def do_POST(self): # Process an HTTP POST request and return a response with an HTTP 200 status. content_len = int(self.headers.get('Content-Length')) request_body = self.rfile.read(content_len) m = marshaller.NewHTTPMarshaller( [binary.NewBinaryHTTPCloudEventConverter()]) event = m.FromRequest(v1.Event(), self.headers, io.BytesIO(request_body), lambda x: json.load(x)) event_type = event.EventType() assert event_type in ALLOWED_EVENT_TYPES extensions = event.Extensions() extensions["djangoapp"] = FakeSourceModel._meta.app_label extensions["djangomodel"] = FakeSourceModel._meta.model_name event_data = event.Data() if event_type in ("django.orm.post.init", "django.orm.pre.save", "django.orm.post.save", "django.orm.pre.delete", "django.orm.post.delete", "django.orm.m2m.change"): assert "data" in event_data instance_data = event_data["data"] assert "id" in instance_data and "name" in instance_data and "enabled" in instance_data assert event_data["db_table"] == FakeSourceModel._meta.db_table check_expected_kwargs(event_type, event_data["signal_kwargs"]) self.send_response(requests.codes.ok) self.end_headers() return
def test_extensions_are_set_upstream(): extensions = {"extension-key": "extension-value"} event = v1.Event().SetExtensions(extensions) m = marshaller.NewDefaultHTTPMarshaller() new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert event.Extensions() == extensions assert "ce-extension-key" in new_headers
def main(): start_time = datetime.now() try: dispatch_policy = os.environ.get("DISPATCH_POLICY", DispatchPolicyConst.NEVER) m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v1.Event(), request.headers, io.BytesIO(request.data), lambda x: json.load(x)) event_info = event.Properties() event_info.update(event_info.pop("extensions")) event_data = event_info.pop("data") app.logger.debug("RCVR: {}".format(event_data)) event_type = event_info.get("type") subject = event_info.get("subject", "sys-0") if event_info["source"] == os.environ.get("K_SERVICE", os.environ.get("SOURCE")): return Response(status=201) event_info["originid"] = event_info.get("originid", event_info.get("id")) app.logger.debug("subject: {}".format(subject)) app.logger.debug("event_data: {}".format(event_data)) subject = subject_factory(name=subject, event_info=event_info, event_data=event_data) event_data["_event_info"] = event_info # TODO: KRUL-155 try: app.router.route(event_type, subject, event_data, dispatch_policy=dispatch_policy) finally: pass exec_time = (datetime.now() - start_time).total_seconds() app.logger.info( "Event", extra={ 'props': { 'event_info': event_info, 'type': event_type, 'subject': subject.name, 'exec_time': exec_time, # 'headers': list(headers.keys()) } }) return Response(status=200) except Exception as ex: app.logger.error(ex, exc_info=True) return Response(status=201)
def test_v1_time_property(): event = v1.Event() time1 = "1234" event.time = time1 assert event.EventTime() == time1 time2 = "4321" event.SetEventTime(time2) assert event.time == time2
def test_v1_subject_property(): event = v1.Event() subject1 = "<my-subject>" event.subject = subject1 assert event.Subject() == subject1 subject2 = "<my-subject2>" event.SetSubject(subject2) assert event.subject == subject2
def test_v1_schema_property(): event = v1.Event() schema1 = "<my-schema>" event.schema = schema1 assert event.Schema() == schema1 schema2 = "<my-schema2>" event.SetSchema(schema2) assert event.schema == schema2
def test_binary_event_v1(): event = (v1.Event().SetContentType("application/octet-stream").SetData( b"\x00\x01")) m = marshaller.NewHTTPMarshaller( [structured.NewJSONHTTPCloudEventConverter()]) _, body = m.ToRequest(event, converters.TypeStructured, lambda x: x) assert isinstance(body, bytes) content = json.loads(body) assert "data" not in content assert content["data_base64"] == "AAE=", f"Content is: {content}"
def callback(message): e = {"event": json.dumps(message.data.decode("utf-8"))} local_time = datetime.now(timezone.utc).astimezone() event = ( v1.Event().SetContentType("application/json").SetData(e).SetEventID( "my-id").SetSource("from-galaxy-far-far-away").SetEventTime( local_time.isoformat()).SetEventType( "com.google.cloudstorage").SetExtensions("")) print(message.data) res = run_structured(event, K_SINK) message.ack()
def test_web_app_echo(): _, r = app.test_client.post("/echo", headers=test_data.headers[v1.Event], data=test_data.body) assert r.status == 200 event = m.FromRequest(v1.Event(), dict(r.headers), r.body, lambda x: x) assert event is not None props = event.Properties() for key in test_data.headers[v1.Event].keys(): if key == "Content-Type": assert "datacontenttype" in props else: assert key.lstrip("ce-") in props
def get_cloudevent_from_signal(sender, **kwargs): from django.conf import settings event_type = _get_event_type_from_signal(kwargs.pop("signal")) obj_meta = sender._meta app = obj_meta.app_label model = obj_meta.model_name payload = {} if "instance" in kwargs: instance = kwargs.pop("instance") payload["data"] = {} if type(instance) != sender: # m2m signal obj_meta = instance._meta model = instance._meta.model_name kwargs["model"] = kwargs["model"]._meta.model_name kwargs["updated_pks"] = list(kwargs.pop("pk_set")) payload["data"] = _get_instance_dict(instance) for m_field in obj_meta.many_to_many: field_name = m_field.name m2m_data = [] for m2m_obj in getattr(instance, field_name).all(): m2m_data.append(_get_instance_dict(m2m_obj)) payload["data"][field_name] = m2m_data # TODO parse related_fields # for r_field in obj_meta.related_objects: subject = "DCE:%s.%s/%s" % (app, model, instance.pk) else: subject = "DCE:%s.%s" % (app, model) payload["signal_kwargs"] = { **kwargs } payload["db_table"] = obj_meta.db_table extensions = { "djangoapp": app, "djangomodel": model, } event_id = str(uuid.uuid4()) event = v1.Event() event.SetContentType('application/json') event.SetEventID(event_id) event.SetSource(os.environ.get(settings.CLOUDEVENTS_ENV["SOURCE_VAR"], "django-orm")) event.SetSubject(subject) event.SetEventTime(datetime.utcnow().replace(tzinfo=pytz.UTC).isoformat()) event.SetEventType(event_type) event.SetExtensions(extensions) event.Set('Originid', event_id) event.SetData(payload) return event
def dispatch(self, event_type, subject, payload): if isinstance(subject, str): subject = subject_factory(subject) _event_info = subject.event_info() _id = str(uuid.uuid4()) logging.debug("new event id: {}".format(_id)) event = v1.Event() event.SetContentType('application/json') event.SetEventID(_id) event.SetSource(self._source) event.SetSubject(str(subject)) event.SetEventTime( datetime.utcnow().replace(tzinfo=pytz.UTC).isoformat()) event.SetEventType(event_type) # set extended properties ext_props = subject.get_ext_props() property_name = payload.get(PayloadConst.PROPERTY_NAME, None) if property_name is not None: ext_props.update({"propertyname": property_name}) event.SetExtensions(ext_props) event.Set('Originid', str(_event_info.get("originid", _id))) event.SetData(payload) m = marshaller.NewHTTPMarshaller( [binary.NewBinaryHTTPCloudEventConverter()]) headers, body = m.ToRequest(event, converters.TypeBinary, lambda x: json.dumps(x, cls=_JSONEncoder)) # headers['Ce-Originid'] = str(_event_info.get("Originid", _id)) if callable(self._dispatch_url): dispatch_url = self._dispatch_url(subject, event_type) else: dispatch_url = self._dispatch_url response = requests.post(dispatch_url, headers=headers, data=body) response.raise_for_status() if self._test: return _id, response.status_code, headers return _id
def fake_receiver_app(environ, start_response): """Simplest possible WSGI application""" request = Request(environ) m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v1.Event(), request.headers, io.BytesIO(request.data), lambda x: json.load(x)) event_info = event.Properties() event_info.update(event_info.pop("extensions")) subject = subject_factory(event_info.get("subject", "sys-0")) assert "originid" in event_info assert "subject" in event_info assert "data" in event_info status = '200 OK' response_headers = [('Content-type', 'text/plain')] start_response(status, response_headers) return ['Ok']
def OnTopicEvent(self, request, context): """Subscribes events from Pubsub.""" pubsub_topic = request.pubsub_name + DELIMITER + request.topic if pubsub_topic not in self._topic_map: context.set_code(grpc.StatusCode.UNIMPLEMENTED) # type: ignore raise NotImplementedError( f'topic {request.topic} is not implemented!') event = v1.Event() event.SetEventType(request.type) event.SetEventID(request.id) event.SetSource(request.source) event.SetData(request.data) event.SetContentType(request.data_content_type) # TODO: add metadata from context to CE envelope self._topic_map[pubsub_topic](event) return empty_pb2.Empty()
def test_object_event_v1(): event = (v1.Event().SetContentType("application/json").SetData( {"name": "john"})) m = marshaller.NewDefaultHTTPMarshaller() _, structuredBody = m.ToRequest(event) assert isinstance(structuredBody, bytes) structuredObj = json.loads(structuredBody) errorMsg = f"Body was {structuredBody}, obj is {structuredObj}" assert isinstance(structuredObj, dict), errorMsg assert isinstance(structuredObj["data"], dict), errorMsg assert len(structuredObj["data"]) == 1, errorMsg assert structuredObj["data"]["name"] == "john", errorMsg headers, binaryBody = m.ToRequest(event, converters.TypeBinary) assert isinstance(headers, dict) assert isinstance(binaryBody, bytes) assert headers["content-type"] == "application/json" assert binaryBody == b'{"name": "john"}', f"Binary is {binaryBody!r}"
async def echo(request): event = m.FromRequest(v1.Event(), dict(request.headers), request.body, lambda x: x) hs, body = m.ToRequest(event, converters.TypeBinary, lambda x: x) return response.text(body, headers=hs)
def test_to_request_invalid_converter(): with pytest.raises(exceptions.NoSuchConverter): m = marshaller.HTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()]) m.ToRequest(v1.Event(), "")
def main(): start_time = datetime.now() try: dispatch_policy = os.environ.get("DISPATCH_POLICY", DispatchPolicyConst.NEVER) m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v1.Event(), request.headers, io.BytesIO(request.data), lambda x: json.load(x)) event_info = event.Properties() event_info.update(event_info.pop("extensions")) event_data = event_info.pop("data") app.logger.debug("RCVR: {}".format(event_data)) type = event_info.get("type") subject = event_info.get("subject", "sys-0") g.subjects = [] # TODO: important!! # need to find a way to avoid a return of messages from the same service # (for example when resending it again after intercepted in the first time) # this workaround only works when in a knative service or at least when SOURCE environment # variable is set if event_info["source"] == os.environ.get("K_SERVICE", os.environ.get("SOURCE")): return Response(status=201) event_info["originid"] = event_info.get("originid", event_info.get("id")) logger.debug("subject: {}".format(subject)) logger.debug("event_data: {}".format(event_data)) from dependency_injector import providers subject = subject_factory(name=subject, event_info=event_info, event_data=event_data) event_data["_event_info"] = event_info # TODO: KRUL-155 try: event_router_factory().route(type, subject, event_data, dispatch_policy=dispatch_policy) finally: for sub in g.subjects: sub.store() exec_time = (datetime.now() - start_time).total_seconds() logger.info( "Event", extra={ 'props': { 'event_info': event_info, 'type': type, 'subject': subject.name, 'exec_time': exec_time, #'headers': list(headers.keys()) } }) return Response(status=200) except Exception as ex: app.logger.error(ex, exc_info=True) return Response(status=201)
def test_to_request_wrong_marshaller(): with pytest.raises(exceptions.InvalidDataMarshaller): m = marshaller.NewDefaultHTTPMarshaller() _ = m.ToRequest(v1.Event(), data_marshaller="")
def test_from_request_wrong_unmarshaller(): with pytest.raises(exceptions.InvalidDataUnmarshaller): m = marshaller.NewDefaultHTTPMarshaller() _ = m.FromRequest(v1.Event(), {}, "", None)
def post(self): """ Handle post request. Extract data. Call event handler and optionally send a reply event. """ if not self.model.ready: self.model.load() try: body = json.loads(self.request.body) except json.decoder.JSONDecodeError as e: raise tornado.web.HTTPError( status_code=HTTPStatus.BAD_REQUEST, reason="Unrecognized request format: %s" % e, ) # Extract payload from request request_handler: RequestHandler = get_request_handler( self.protocol, body) request_handler.validate() request = request_handler.extract_request() # Create event from request body event = v1.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = http_marshaller.FromRequest(event, self.request.headers, self.request.body, json.loads) logging.debug(json.dumps(event.Properties())) # Extract any desired request headers headers = {} for (key, val) in self.request.headers.get_all(): headers[key] = val response: Optional[ModelResponse] = self.model.process_event( request, headers) if response is None: return runtime_metrics = response.metrics if runtime_metrics is not None: if validate_metrics(runtime_metrics): self.seldon_metrics.update(runtime_metrics, self.event_type) else: logging.error("Metrics returned are invalid: " + str(runtime_metrics)) if response.data is not None: responseStr = json.dumps(response.data) # Create event from response if reply_url is active if not self.reply_url == "": if event.EventID() is None or event.EventID() == "": resp_event_id = uuid.uuid1().hex else: resp_event_id = event.EventID() revent = ( v1.Event().SetContentType("application/json").SetData( responseStr).SetEventID(resp_event_id).SetSource( self.event_source).SetEventType( self.event_type).SetExtensions( event.Extensions())) logging.debug(json.dumps(revent.Properties())) sendCloudEvent(revent, self.reply_url) self.write(json.dumps(response.data))
async def is_ok(request): m.FromRequest(v1.Event(), dict(request.headers), request.body, lambda x: x) return response.text("OK")