def test_empty_json_structured():
headers = {"Content-Type": "application/cloudevents+json"}
data = ""
with pytest.raises(cloud_exceptions.MissingRequiredFields) as e:
from_http(headers, data)
assert "Failed to read specversion from both headers and data" in str(
e.value)
def test_non_dict_data_no_headers_bug():
# Test for issue #116
headers = {"Content-Type": "application/cloudevents+json"}
data = "123"
with pytest.raises(cloud_exceptions.MissingRequiredFields) as e:
from_http(headers, data)
assert "Failed to read specversion from both headers and data" in str(e.value)
assert "The following deserialized data has no 'get' method" in str(e.value)
def test_wrong_specversion():
headers = {"Content-Type": "application/cloudevents+json"}
data = json.dumps({
"specversion": "0.2",
"type": "word.found.name",
"id": "96fb5f0b-001e-0108-6dfe-da6e2806f124",
"source": "<my-source>",
})
with pytest.raises(cloud_exceptions.InvalidRequiredFields) as e:
from_http(headers, data)
assert "Found invalid specversion 0.2" in str(e.value)
async def post(self, name: str):
if has_binary_headers(self.request.headers):
try:
#Use default unmarshaller if contenttype is set in header
if "ce-contenttype" in self.request.headers:
body = from_http(self.request.headers, self.request.body)
else:
body = from_http(self.request.headers, self.request.body,
lambda x: x)
except (ce.MissingRequiredFields, ce.InvalidRequiredFields,
ce.InvalidStructuredJSON, ce.InvalidHeadersFormat,
ce.DataMarshallerError, ce.DataUnmarshallerError) as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Cloud Event Exceptions: %s" % e)
else:
try:
body = json.loads(self.request.body)
except json.decoder.JSONDecodeError as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Unrecognized request format: %s" % e)
model = self.get_model(name)
request = model.preprocess(body)
request = self.validate(request)
response = (await
model.predict(request)) if inspect.iscoroutinefunction(
model.predict) else model.predict(request)
response = model.postprocess(response)
if has_binary_headers(self.request.headers):
event = CloudEvent(body._attributes, response)
if is_binary(self.request.headers):
eventheader, eventbody = to_binary(event)
elif is_structured(self.request.headers):
eventheader, eventbody = to_structured(event)
for k, v in eventheader.items():
if k != "ce-time":
self.set_header(k, v)
else: #utc now() timestamp
self.set_header(
'ce-time',
datetime.utcnow().replace(tzinfo=pytz.utc).strftime(
'%Y-%m-%dT%H:%M:%S.%f%z'))
if isinstance(eventbody, (bytes, bytearray)):
response = eventbody
else:
response = eventbody.data
self.write(response)
def test_http_data_unmarshaller_exceptions(binary_headers, structured_data):
# binary
with pytest.raises(cloud_exceptions.DataUnmarshallerError) as e:
from_http(binary_headers, None, data_unmarshaller=lambda x: 1 / 0)
assert ("Failed to unmarshall data with error: "
"ZeroDivisionError('division by zero')" in str(e.value))
# structured
headers = {"Content-Type": "application/cloudevents+json"}
with pytest.raises(cloud_exceptions.DataUnmarshallerError) as e:
from_http(headers, structured_data, data_unmarshaller=lambda x: 1 / 0)
assert ("Failed to unmarshall data with error: "
"ZeroDivisionError('division by zero')" in str(e.value))
def read_gcs():
# create a CloudEvent
event = from_http(request.headers, request.get_data())
print(
f"Found {event['source']} with type "
f"{event['type']} and subject {event['subject']}"
)
# a GCS object is created
if 'methodname' in event and event['methodname'] == 'storage.objects.create':
m = re.search(gcs_pattern, event['resourcename'])
if not m:
return ('No resourcename attribute', 500)
if m.group(1) == gcs_bucket:
print(f"resource: {m.group(0)}")
bucket = storage_client.get_bucket(m.group(1))
blob = bucket.get_blob(m.group(2))
temp_file_name = m.group(2)
blob.download_to_filename(temp_file_name)
with open (temp_file_name, 'r') as myfile:
html_doc = myfile.read()
parse(html_doc)
return ('', 204)
def view_func(path):
ce_exception = None
event = None
try:
event = from_http(request.headers, request.get_data())
except (
cloud_exceptions.MissingRequiredFields,
cloud_exceptions.InvalidRequiredFields,
) as e:
ce_exception = e
if not ce_exception:
function(event)
return "OK"
# Not a CloudEvent. Try converting to a CloudEvent.
try:
function(event_conversion.background_event_to_cloudevent(request))
except EventConversionException as e:
flask.abort(
400,
description=(
"Function was defined with FUNCTION_SIGNATURE_TYPE=cloudevent but"
" parsing CloudEvent failed and converting from background event to"
f" CloudEvent also failed.\nGot HTTP headers: {request.headers}\nGot"
f" data: {request.get_data()}\nGot CloudEvent exception: {repr(ce_exception)}"
f"\nGot background event conversion exception: {repr(e)}"
),
)
return "OK"
def get_binary_cloudevent(self) -> CloudEvent:
try:
# Use default unmarshaller if contenttype is set in header
if "ce-contenttype" in self.request.headers:
event = from_http(self.request.headers, self.request.body)
else:
event = from_http(self.request.headers, self.request.body,
lambda x: x)
return event
except (ce.MissingRequiredFields, ce.InvalidRequiredFields,
ce.InvalidStructuredJSON, ce.InvalidHeadersFormat,
ce.DataMarshallerError, ce.DataUnmarshallerError) as e:
raise tornado.web.HTTPError(status_code=HTTPStatus.BAD_REQUEST,
reason="Cloud Event Exceptions: %s" %
e)
def test_valid_structured_events(specversion):
# Test creating multiple cloud events
events_queue = []
num_cloudevents = 30
for i in range(num_cloudevents):
event = {
"id": f"id{i}",
"source": f"source{i}.com.test",
"type": f"cloudevent.test.type",
"specversion": specversion,
"data": {
"payload": f"payload-{i}"
},
}
events_queue.append(
from_http(
{"content-type": "application/cloudevents+json"},
json.dumps(event),
))
for i, event in enumerate(events_queue):
assert event["id"] == f"id{i}"
assert event["source"] == f"source{i}.com.test"
assert event["specversion"] == specversion
assert event.data["payload"] == f"payload-{i}"
def test_missing_required_fields_binary(headers):
with pytest.raises((ValueError)):
# CloudEvent constructor throws TypeError if missing required field
# and NotImplementedError because structured calls aren't
# implemented. In this instance one of the required keys should have
# prefix e-id instead of ce-id therefore it should throw
_ = from_http(json.dumps(test_data), headers=headers)
def echo():
try:
event = from_http(request.headers, request.get_data(),None)
data = event.data
# hack to handle non JSON payload, e.g. xml
if not isinstance(data,dict):
data = str(event.data)
e = {
"attributes": event._attributes,
"data": data
}
app.logger.info(f'"***cloud event*** {json.dumps(e)}')
return {}, 204
except Exception as e:
sc = 400
msg = f'could not decode cloud event: {e}'
app.logger.error(msg)
message = {
'status': sc,
'error': msg,
}
resp = jsonify(message)
resp.status_code = sc
return resp
def test_create_binary_image():
# Create image and turn image into bytes
attributes = {
"type": "com.example.string",
"source": "https://example.com/event-producer",
}
# Create CloudEvent
event = CloudEvent(attributes, image_bytes)
# Create http headers/body content
headers, body = to_binary(event)
# Unmarshall CloudEvent and re-create image
reconstruct_event = from_http(headers,
body,
data_unmarshaller=lambda x: io.BytesIO(x))
# reconstruct_event.data is an io.BytesIO object due to data_unmarshaller
restore_image = Image.open(reconstruct_event.data)
assert restore_image.size == image_expected_shape
# # Test cloudevent extension from http fields and data
assert isinstance(body, bytes)
assert body == image_bytes
def home():
event = from_http(request.headers, request.get_data())
if event['type'] == 'dev.pulsifer.radio.request':
action = event.data['action']
if action in actions:
global activeThread
if action == 'blink':
stop_running_thread()
activeThread = threading.Thread(name="blinky", target=blink, args=(lock,stop,))
activeThread.start()
elif action == 'rainbow':
stop_running_thread()
activeThread = threading.Thread(name="rainbow", target=rainbow, args=(lock,stop,))
activeThread.start()
elif action == 'wow':
stop_running_thread()
activeThread = threading.Thread(name="wow", target=wow, args=(lock,stop,))
activeThread.start()
else: eval(action)()
return json.dumps({
'action': activeThread.getName(),
'alive': activeThread.is_alive(),
'brightness': brightness,
})
return "", 501
return "", 400
def test_create_structured_image():
# Create image and turn image into bytes
attributes = {
"type": "com.example.string",
"source": "https://example.com/event-producer",
}
# Create CloudEvent
event = CloudEvent(attributes, image_bytes)
# Create http headers/body content
headers, body = to_structured(event)
# Structured has cloudevent attributes marshalled inside the body. For this
# reason we must load the byte object to create the python dict containing
# the cloudevent attributes
data = json.loads(body)
# Test cloudevent extension from http fields and data
assert isinstance(data, dict)
assert base64.b64decode(data["data_base64"]) == image_bytes
# Unmarshall CloudEvent and re-create image
reconstruct_event = from_http(headers,
body,
data_unmarshaller=lambda x: io.BytesIO(x))
# reconstruct_event.data is an io.BytesIO object due to data_unmarshaller
restore_image = Image.open(reconstruct_event.data)
assert restore_image.size == image_expected_shape
def test_missing_required_fields_structured(body):
with pytest.raises(cloud_exceptions.MissingRequiredFields) as e:
_ = from_http(
{"Content-Type": "application/cloudevents+json"},
json.dumps(body),
)
async def post(self, name: str):
if has_binary_headers(self.request.headers):
try:
# Use default unmarshaller if contenttype is set in header
if "ce-contenttype" in self.request.headers:
body = from_http(self.request.headers, self.request.body)
else:
body = from_http(self.request.headers, self.request.body,
lambda x: x)
except (ce.MissingRequiredFields, ce.InvalidRequiredFields,
ce.InvalidStructuredJSON, ce.InvalidHeadersFormat,
ce.DataMarshallerError, ce.DataUnmarshallerError) as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Cloud Event Exceptions: %s" % e)
else:
try:
body = json.loads(self.request.body)
except json.decoder.JSONDecodeError as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Unrecognized request format: %s" % e)
# call model locally or remote model workers
model = self.get_model(name)
if not isinstance(model, RayServeHandle):
response = await model(body)
else:
model_handle = model
response = await model_handle.remote(body)
# process response from the model
if has_binary_headers(self.request.headers):
event = CloudEvent(body._attributes, response)
if is_binary(self.request.headers):
eventheader, eventbody = to_binary(event)
elif is_structured(self.request.headers):
eventheader, eventbody = to_structured(event)
for k, v in eventheader.items():
if k != "ce-time":
self.set_header(k, v)
else: # utc now() timestamp
self.set_header(
'ce-time',
datetime.utcnow().replace(tzinfo=pytz.utc).strftime(
'%Y-%m-%dT%H:%M:%S.%f%z'))
response = eventbody
self.write(response)
def test_empty_data_binary_event(specversion):
# Testing if cloudevent breaks when no structured data field present
headers = {
"Content-Type": "application/octet-stream",
"ce-specversion": specversion,
"ce-type": "word.found.name",
"ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124",
"ce-time": "2018-10-23T12:28:22.4579346Z",
"ce-source": "<source-url>",
}
event = from_http(headers, None)
assert event.data == None
data = ""
# Data of empty string will be marshalled into None
event = from_http(headers, data)
assert event.data == None
def home():
# Retrieve the CloudEvent
event = from_http(request.headers, request.get_data())
# Process the event
process_event(event.data)
return "", 204
def handle_post():
context = {'request': request}
try:
context['cloud_event'] = from_http(request.headers,
request.get_data())
except Exception:
app.logger.warning('No CloudEvent available')
return func.main(context)
def test_generic_exception():
headers = {"Content-Type": "application/cloudevents+json"}
data = json.dumps({
"specversion": "1.0",
"source": "s",
"type": "t",
"id": "1234-1234-1234",
"data": "",
})
with pytest.raises(cloud_exceptions.GenericException) as e:
from_http({}, None)
e.errisinstance(cloud_exceptions.MissingRequiredFields)
with pytest.raises(cloud_exceptions.GenericException) as e:
from_http({}, 123)
e.errisinstance(cloud_exceptions.InvalidStructuredJSON)
with pytest.raises(cloud_exceptions.GenericException) as e:
from_http(headers, data, data_unmarshaller=lambda x: 1 / 0)
e.errisinstance(cloud_exceptions.DataUnmarshallerError)
with pytest.raises(cloud_exceptions.GenericException) as e:
event = from_http(headers, data)
to_binary(event, data_marshaller=lambda x: 1 / 0)
e.errisinstance(cloud_exceptions.DataMarshallerError)
def home():
# create a CloudEvent
event = from_http(request.get_data(), request.headers)
# you can access cloudevent fields as seen below
print(f"Found {event['id']} from {event['source']} with type "
f"{event['type']} and specversion {event['specversion']}")
return "", 204
def test_missing_required_fields_structured(body):
with pytest.raises((TypeError, NotImplementedError)):
# CloudEvent constructor throws TypeError if missing required field
# and NotImplementedError because structured calls aren't
# implemented. In this instance one of the required keys should have
# prefix e-id instead of ce-id therefore it should throw
_ = from_http(
json.dumps(body), attributes={"Content-Type": "application/json"}
)
def test_known_empty_edge_cases(binary_headers, test_data):
expect_data = test_data
if test_data in ["", b""]:
expect_data = None
elif test_data == ():
# json.dumps(()) outputs '[]' hence list not tuple check
expect_data = []
# Remove ce- prefix
headers = {key[3:]: value for key, value in binary_headers.items()}
# binary
event = from_http(*to_binary(CloudEvent(headers, test_data)))
assert event.data == expect_data
# structured
event = from_http(*to_structured(CloudEvent(headers, test_data)))
assert event.data == expect_data
def test_empty_data_structured_event(specversion):
# Testing if cloudevent breaks when no structured data field present
attributes = {
"specversion": specversion,
"datacontenttype": "application/cloudevents+json",
"type": "word.found.name",
"id": "96fb5f0b-001e-0108-6dfe-da6e2806f124",
"time": "2018-10-23T12:28:22.4579346Z",
"source": "<source-url>",
}
event = from_http({"content-type": "application/cloudevents+json"},
json.dumps(attributes))
assert event.data == None
attributes["data"] = ""
# Data of empty string will be marshalled into None
event = from_http({"content-type": "application/cloudevents+json"},
json.dumps(attributes))
assert event.data == None
def test_empty_data_binary_event(specversion):
# Testing if cloudevent breaks when no structured data field present
headers = {
"Content-Type": "application/octet-stream",
"ce-specversion": specversion,
"ce-type": "word.found.name",
"ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124",
"ce-time": "2018-10-23T12:28:22.4579346Z",
"ce-source": "<source-url>",
}
_ = from_http("", headers)
async def echo(request):
decoder = None
if "binary-payload" in request.headers:
decoder = lambda x: x
event = from_http(dict(request.headers),
request.body,
data_unmarshaller=decoder)
data = (event.data if isinstance(event.data,
(bytes, bytearray, memoryview)) else
json.dumps(event.data).encode())
return response.raw(data, headers={k: event[k] for k in event})
def cloud_event_to_background_event(request) -> Tuple[Any, Context]:
"""Converts a background event represented by the given HTTP request into a CloudEvent."""
try:
event = from_http(request.headers, request.get_data())
data = event.data
service, name = _split_ce_source(event["source"])
if event["type"] not in _CE_TO_BACKGROUND_TYPE:
raise EventConversionException(
f'Unable to find background event equivalent type for "{event["type"]}"'
)
if service == _PUBSUB_CE_SERVICE:
resource = {
"service": service,
"name": name,
"type": _PUBSUB_MESSAGE_TYPE
}
if "message" in data:
data = data["message"]
if "messageId" in data:
del data["messageId"]
if "publishTime" in data:
del data["publishTime"]
elif service == _FIREBASE_AUTH_CE_SERVICE:
resource = name
if "metadata" in data:
for old, new in _FIREBASE_AUTH_METADATA_FIELDS_CE_TO_BACKGROUND.items(
):
if old in data["metadata"]:
data["metadata"][new] = data["metadata"][old]
del data["metadata"][old]
elif service == _STORAGE_CE_SERVICE:
resource = {
"name": f"{name}/{event['subject']}",
"service": service,
"type": data["kind"],
}
elif service == _FIREBASE_DB_CE_SERVICE:
name = re.sub("/locations/[^/]+", "", name)
resource = f"{name}/{event['subject']}"
else:
resource = f"{name}/{event['subject']}"
context = Context(
eventId=event["id"],
timestamp=event["time"],
eventType=_CE_TO_BACKGROUND_TYPE[event["type"]],
resource=resource,
)
return (data, context)
except (AttributeError, KeyError, TypeError, MissingRequiredFields):
raise EventConversionException(
"Failed to convert CloudEvent to BackgroundEvent.")
def handle_post():
context = Context(request)
try:
context.cloud_event = from_http(request.headers,
request.get_data())
except Exception:
app.logger.warning('No CloudEvent available')
try:
return func.main(context)
except Exception as err:
return f"Function threw {err}", 500
def main():
event = from_http(request.headers, request.get_data())
app.logger.warning(event)
# you can access cloudevent fields as seen below
print(
f"Found {event['id']} from {event['source']} with type "
f"{event['type']} and specversion {event['specversion']}"
)
return "", 204
def handle_post():
app.logger.info(pretty_print_POST(request))
# Read CloudEvent from the request
cloud_event = from_http(request.headers, request.get_data())
# Parse the event body
country, tableId = read_event_data(cloud_event)
query_covid_dataset(country, tableId)
return 'OK', 200
