def test_from_request_cannot_read(binary_headers):
with pytest.raises(exceptions.UnsupportedEventConverter):
m = marshaller.HTTPMarshaller([binary.NewBinaryHTTPCloudEventConverter()])
m.FromRequest(v1.Event(), {}, "")
with pytest.raises(exceptions.UnsupportedEventConverter):
m = marshaller.HTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()])
m.FromRequest(v1.Event(), binary_headers, "")
def target():
# Deserialize the HTTP request into a CloudEvent
try:
logging.info('Deserializing CloudEvent in binary format')
# Instantiate the Binary Format marshaller
mb = marshaller.NewHTTPMarshaller(
[binary.NewBinaryHTTPCloudEventConverter()])
ce = mb.FromRequest(v1.Event(), request.headers, request.stream.read(),
lambda x: x)
except:
logging.warning(
'Deserializing application/cloudevents+json CloudEvent')
# Instantiate the JSON Structured CloudEvent marshaller
m = marshaller.NewHTTPMarshaller(
[structured.NewJSONHTTPCloudEventConverter()])
ce = m.FromRequest()
except:
logging.warning('Could not deserialize CloudEvent')
# Create a CloudEvent to send as response
data = 'Data received was not understandable as a CloudEvent'
event = (
v1.Event().SetContentType("text/plain").SetData(data).SetEventID(
str(uuid.uuid4())).SetSource("from_your_own_target").
SetSubject("your_event_subject").SetEventTime(
datetime.now(
timezone.utc).astimezone().isoformat()).SetEventType(
"io.triggermesh.target.byown"))
return app.response_class(response=body, headers=headers, status=400)
# Do your Transformation or Target work based on the eventype
if ce.EventType() == "io.triggermesh.byown.create":
logging.info("Create event type")
elif ce.EventType() == "io.triggermesh.byown.delete":
logging.info("Delete event type")
else:
logging.warning("Unknown event type %s" % ce.EventType())
# Create a CloudEvent to send as response
data = 'this is some data'
event = (v1.Event().SetContentType("text/plain").SetData(data).SetEventID(
str(uuid.uuid4())).SetSource("from_your_own_target").SetSubject(
"your_event_subject").SetEventTime(
datetime.now(
timezone.utc).astimezone().isoformat()).SetEventType(
"io.triggermesh.target.byown"))
# Prepare the Header and Body to send a request back as a CloudEvent
m = marshaller.NewHTTPMarshaller(
[structured.NewJSONHTTPCloudEventConverter()])
headers, body = m.ToRequest(event, converters.TypeStructured, lambda x: x)
return app.response_class(response=body, headers=headers, status=200)
def do_POST(self):
# Process an HTTP POST request and return a response with an HTTP 200 status.
content_len = int(self.headers.get('Content-Length'))
request_body = self.rfile.read(content_len)
m = marshaller.NewHTTPMarshaller(
[binary.NewBinaryHTTPCloudEventConverter()])
event = m.FromRequest(v1.Event(), self.headers,
io.BytesIO(request_body), lambda x: json.load(x))
event_type = event.EventType()
assert event_type in ALLOWED_EVENT_TYPES
extensions = event.Extensions()
extensions["djangoapp"] = FakeSourceModel._meta.app_label
extensions["djangomodel"] = FakeSourceModel._meta.model_name
event_data = event.Data()
if event_type in ("django.orm.post.init", "django.orm.pre.save",
"django.orm.post.save", "django.orm.pre.delete",
"django.orm.post.delete", "django.orm.m2m.change"):
assert "data" in event_data
instance_data = event_data["data"]
assert "id" in instance_data and "name" in instance_data and "enabled" in instance_data
assert event_data["db_table"] == FakeSourceModel._meta.db_table
check_expected_kwargs(event_type, event_data["signal_kwargs"])
self.send_response(requests.codes.ok)
self.end_headers()
return
def test_extensions_are_set_upstream():
extensions = {"extension-key": "extension-value"}
event = v1.Event().SetExtensions(extensions)
m = marshaller.NewDefaultHTTPMarshaller()
new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x)
assert event.Extensions() == extensions
assert "ce-extension-key" in new_headers
def main():
start_time = datetime.now()
try:
dispatch_policy = os.environ.get("DISPATCH_POLICY",
DispatchPolicyConst.NEVER)
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v1.Event(), request.headers,
io.BytesIO(request.data), lambda x: json.load(x))
event_info = event.Properties()
event_info.update(event_info.pop("extensions"))
event_data = event_info.pop("data")
app.logger.debug("RCVR: {}".format(event_data))
event_type = event_info.get("type")
subject = event_info.get("subject", "sys-0")
if event_info["source"] == os.environ.get("K_SERVICE",
os.environ.get("SOURCE")):
return Response(status=201)
event_info["originid"] = event_info.get("originid",
event_info.get("id"))
app.logger.debug("subject: {}".format(subject))
app.logger.debug("event_data: {}".format(event_data))
subject = subject_factory(name=subject,
event_info=event_info,
event_data=event_data)
event_data["_event_info"] = event_info # TODO: KRUL-155
try:
app.router.route(event_type,
subject,
event_data,
dispatch_policy=dispatch_policy)
finally:
pass
exec_time = (datetime.now() - start_time).total_seconds()
app.logger.info(
"Event",
extra={
'props': {
'event_info': event_info,
'type': event_type,
'subject': subject.name,
'exec_time': exec_time,
# 'headers': list(headers.keys())
}
})
return Response(status=200)
except Exception as ex:
app.logger.error(ex, exc_info=True)
return Response(status=201)
def test_v1_time_property():
event = v1.Event()
time1 = "1234"
event.time = time1
assert event.EventTime() == time1
time2 = "4321"
event.SetEventTime(time2)
assert event.time == time2
def test_v1_subject_property():
event = v1.Event()
subject1 = "<my-subject>"
event.subject = subject1
assert event.Subject() == subject1
subject2 = "<my-subject2>"
event.SetSubject(subject2)
assert event.subject == subject2
def test_v1_schema_property():
event = v1.Event()
schema1 = "<my-schema>"
event.schema = schema1
assert event.Schema() == schema1
schema2 = "<my-schema2>"
event.SetSchema(schema2)
assert event.schema == schema2
def test_binary_event_v1():
event = (v1.Event().SetContentType("application/octet-stream").SetData(
b"\x00\x01"))
m = marshaller.NewHTTPMarshaller(
[structured.NewJSONHTTPCloudEventConverter()])
_, body = m.ToRequest(event, converters.TypeStructured, lambda x: x)
assert isinstance(body, bytes)
content = json.loads(body)
assert "data" not in content
assert content["data_base64"] == "AAE=", f"Content is: {content}"
def callback(message):
e = {"event": json.dumps(message.data.decode("utf-8"))}
local_time = datetime.now(timezone.utc).astimezone()
event = (
v1.Event().SetContentType("application/json").SetData(e).SetEventID(
"my-id").SetSource("from-galaxy-far-far-away").SetEventTime(
local_time.isoformat()).SetEventType(
"com.google.cloudstorage").SetExtensions(""))
print(message.data)
res = run_structured(event, K_SINK)
message.ack()
def test_web_app_echo():
_, r = app.test_client.post("/echo",
headers=test_data.headers[v1.Event],
data=test_data.body)
assert r.status == 200
event = m.FromRequest(v1.Event(), dict(r.headers), r.body, lambda x: x)
assert event is not None
props = event.Properties()
for key in test_data.headers[v1.Event].keys():
if key == "Content-Type":
assert "datacontenttype" in props
else:
assert key.lstrip("ce-") in props
def get_cloudevent_from_signal(sender, **kwargs):
from django.conf import settings
event_type = _get_event_type_from_signal(kwargs.pop("signal"))
obj_meta = sender._meta
app = obj_meta.app_label
model = obj_meta.model_name
payload = {}
if "instance" in kwargs:
instance = kwargs.pop("instance")
payload["data"] = {}
if type(instance) != sender: # m2m signal
obj_meta = instance._meta
model = instance._meta.model_name
kwargs["model"] = kwargs["model"]._meta.model_name
kwargs["updated_pks"] = list(kwargs.pop("pk_set"))
payload["data"] = _get_instance_dict(instance)
for m_field in obj_meta.many_to_many:
field_name = m_field.name
m2m_data = []
for m2m_obj in getattr(instance, field_name).all():
m2m_data.append(_get_instance_dict(m2m_obj))
payload["data"][field_name] = m2m_data
# TODO parse related_fields
# for r_field in obj_meta.related_objects:
subject = "DCE:%s.%s/%s" % (app, model, instance.pk)
else:
subject = "DCE:%s.%s" % (app, model)
payload["signal_kwargs"] = {
**kwargs
}
payload["db_table"] = obj_meta.db_table
extensions = {
"djangoapp": app,
"djangomodel": model,
}
event_id = str(uuid.uuid4())
event = v1.Event()
event.SetContentType('application/json')
event.SetEventID(event_id)
event.SetSource(os.environ.get(settings.CLOUDEVENTS_ENV["SOURCE_VAR"], "django-orm"))
event.SetSubject(subject)
event.SetEventTime(datetime.utcnow().replace(tzinfo=pytz.UTC).isoformat())
event.SetEventType(event_type)
event.SetExtensions(extensions)
event.Set('Originid', event_id)
event.SetData(payload)
return event
def dispatch(self, event_type, subject, payload):
if isinstance(subject, str):
subject = subject_factory(subject)
_event_info = subject.event_info()
_id = str(uuid.uuid4())
logging.debug("new event id: {}".format(_id))
event = v1.Event()
event.SetContentType('application/json')
event.SetEventID(_id)
event.SetSource(self._source)
event.SetSubject(str(subject))
event.SetEventTime(
datetime.utcnow().replace(tzinfo=pytz.UTC).isoformat())
event.SetEventType(event_type)
# set extended properties
ext_props = subject.get_ext_props()
property_name = payload.get(PayloadConst.PROPERTY_NAME, None)
if property_name is not None:
ext_props.update({"propertyname": property_name})
event.SetExtensions(ext_props)
event.Set('Originid', str(_event_info.get("originid", _id)))
event.SetData(payload)
m = marshaller.NewHTTPMarshaller(
[binary.NewBinaryHTTPCloudEventConverter()])
headers, body = m.ToRequest(event, converters.TypeBinary,
lambda x: json.dumps(x, cls=_JSONEncoder))
# headers['Ce-Originid'] = str(_event_info.get("Originid", _id))
if callable(self._dispatch_url):
dispatch_url = self._dispatch_url(subject, event_type)
else:
dispatch_url = self._dispatch_url
response = requests.post(dispatch_url, headers=headers, data=body)
response.raise_for_status()
if self._test:
return _id, response.status_code, headers
return _id
def fake_receiver_app(environ, start_response):
"""Simplest possible WSGI application"""
request = Request(environ)
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v1.Event(), request.headers,
io.BytesIO(request.data), lambda x: json.load(x))
event_info = event.Properties()
event_info.update(event_info.pop("extensions"))
subject = subject_factory(event_info.get("subject", "sys-0"))
assert "originid" in event_info
assert "subject" in event_info
assert "data" in event_info
status = '200 OK'
response_headers = [('Content-type', 'text/plain')]
start_response(status, response_headers)
return ['Ok']
def OnTopicEvent(self, request, context):
"""Subscribes events from Pubsub."""
pubsub_topic = request.pubsub_name + DELIMITER + request.topic
if pubsub_topic not in self._topic_map:
context.set_code(grpc.StatusCode.UNIMPLEMENTED) # type: ignore
raise NotImplementedError(
f'topic {request.topic} is not implemented!')
event = v1.Event()
event.SetEventType(request.type)
event.SetEventID(request.id)
event.SetSource(request.source)
event.SetData(request.data)
event.SetContentType(request.data_content_type)
# TODO: add metadata from context to CE envelope
self._topic_map[pubsub_topic](event)
return empty_pb2.Empty()
def test_object_event_v1():
event = (v1.Event().SetContentType("application/json").SetData(
{"name": "john"}))
m = marshaller.NewDefaultHTTPMarshaller()
_, structuredBody = m.ToRequest(event)
assert isinstance(structuredBody, bytes)
structuredObj = json.loads(structuredBody)
errorMsg = f"Body was {structuredBody}, obj is {structuredObj}"
assert isinstance(structuredObj, dict), errorMsg
assert isinstance(structuredObj["data"], dict), errorMsg
assert len(structuredObj["data"]) == 1, errorMsg
assert structuredObj["data"]["name"] == "john", errorMsg
headers, binaryBody = m.ToRequest(event, converters.TypeBinary)
assert isinstance(headers, dict)
assert isinstance(binaryBody, bytes)
assert headers["content-type"] == "application/json"
assert binaryBody == b'{"name": "john"}', f"Binary is {binaryBody!r}"
async def echo(request):
event = m.FromRequest(v1.Event(), dict(request.headers), request.body,
lambda x: x)
hs, body = m.ToRequest(event, converters.TypeBinary, lambda x: x)
return response.text(body, headers=hs)
def test_to_request_invalid_converter():
with pytest.raises(exceptions.NoSuchConverter):
m = marshaller.HTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()])
m.ToRequest(v1.Event(), "")
def main():
start_time = datetime.now()
try:
dispatch_policy = os.environ.get("DISPATCH_POLICY",
DispatchPolicyConst.NEVER)
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v1.Event(), request.headers,
io.BytesIO(request.data), lambda x: json.load(x))
event_info = event.Properties()
event_info.update(event_info.pop("extensions"))
event_data = event_info.pop("data")
app.logger.debug("RCVR: {}".format(event_data))
type = event_info.get("type")
subject = event_info.get("subject", "sys-0")
g.subjects = []
# TODO: important!!
# need to find a way to avoid a return of messages from the same service
# (for example when resending it again after intercepted in the first time)
# this workaround only works when in a knative service or at least when SOURCE environment
# variable is set
if event_info["source"] == os.environ.get("K_SERVICE",
os.environ.get("SOURCE")):
return Response(status=201)
event_info["originid"] = event_info.get("originid",
event_info.get("id"))
logger.debug("subject: {}".format(subject))
logger.debug("event_data: {}".format(event_data))
from dependency_injector import providers
subject = subject_factory(name=subject,
event_info=event_info,
event_data=event_data)
event_data["_event_info"] = event_info # TODO: KRUL-155
try:
event_router_factory().route(type,
subject,
event_data,
dispatch_policy=dispatch_policy)
finally:
for sub in g.subjects:
sub.store()
exec_time = (datetime.now() - start_time).total_seconds()
logger.info(
"Event",
extra={
'props': {
'event_info': event_info,
'type': type,
'subject': subject.name,
'exec_time': exec_time,
#'headers': list(headers.keys())
}
})
return Response(status=200)
except Exception as ex:
app.logger.error(ex, exc_info=True)
return Response(status=201)
def test_to_request_wrong_marshaller():
with pytest.raises(exceptions.InvalidDataMarshaller):
m = marshaller.NewDefaultHTTPMarshaller()
_ = m.ToRequest(v1.Event(), data_marshaller="")
def test_from_request_wrong_unmarshaller():
with pytest.raises(exceptions.InvalidDataUnmarshaller):
m = marshaller.NewDefaultHTTPMarshaller()
_ = m.FromRequest(v1.Event(), {}, "", None)
def post(self):
"""
Handle post request. Extract data. Call event handler and optionally send a reply event.
"""
if not self.model.ready:
self.model.load()
try:
body = json.loads(self.request.body)
except json.decoder.JSONDecodeError as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Unrecognized request format: %s" % e,
)
# Extract payload from request
request_handler: RequestHandler = get_request_handler(
self.protocol, body)
request_handler.validate()
request = request_handler.extract_request()
# Create event from request body
event = v1.Event()
http_marshaller = marshaller.NewDefaultHTTPMarshaller()
event = http_marshaller.FromRequest(event, self.request.headers,
self.request.body, json.loads)
logging.debug(json.dumps(event.Properties()))
# Extract any desired request headers
headers = {}
for (key, val) in self.request.headers.get_all():
headers[key] = val
response: Optional[ModelResponse] = self.model.process_event(
request, headers)
if response is None:
return
runtime_metrics = response.metrics
if runtime_metrics is not None:
if validate_metrics(runtime_metrics):
self.seldon_metrics.update(runtime_metrics, self.event_type)
else:
logging.error("Metrics returned are invalid: " +
str(runtime_metrics))
if response.data is not None:
responseStr = json.dumps(response.data)
# Create event from response if reply_url is active
if not self.reply_url == "":
if event.EventID() is None or event.EventID() == "":
resp_event_id = uuid.uuid1().hex
else:
resp_event_id = event.EventID()
revent = (
v1.Event().SetContentType("application/json").SetData(
responseStr).SetEventID(resp_event_id).SetSource(
self.event_source).SetEventType(
self.event_type).SetExtensions(
event.Extensions()))
logging.debug(json.dumps(revent.Properties()))
sendCloudEvent(revent, self.reply_url)
self.write(json.dumps(response.data))
async def is_ok(request):
m.FromRequest(v1.Event(), dict(request.headers), request.body, lambda x: x)
return response.text("OK")
