def send_structured_cloud_event(url: str):
# Create cloudevent
attributes = {
"type": "com.example.base64",
"source": "https://example.com/event-producer",
}
event = CloudEvent(attributes, image_bytes)
# Create cloudevent HTTP headers and content
# Note that to_structured will create a data_base64 data field in
# specversion 1.0 (default specversion) if given
# an event whose data field is of type bytes.
headers, body = to_structured(event)
# Send cloudevent
requests.post(url, headers=headers, data=body)
print(f"Sent {event['id']} of type {event['type']}")
def test_known_empty_edge_cases(binary_headers, test_data):
expect_data = test_data
if test_data in ["", b""]:
expect_data = None
elif test_data == ():
# json.dumps(()) outputs '[]' hence list not tuple check
expect_data = []
# Remove ce- prefix
headers = {key[3:]: value for key, value in binary_headers.items()}
# binary
event = from_http(*to_binary(CloudEvent(headers, test_data)))
assert event.data == expect_data
# structured
event = from_http(*to_structured(CloudEvent(headers, test_data)))
assert event.data == expect_data
def test_structured_to_request(specversion):
attributes = {
"specversion": specversion,
"type": "word.found.name",
"id": "96fb5f0b-001e-0108-6dfe-da6e2806f124",
"source": "pytest",
}
data = {"message": "Hello World!"}
event = CloudEvent(attributes, data)
headers, body_bytes = to_structured(event)
assert isinstance(body_bytes, bytes)
body = json.loads(body_bytes)
assert headers["content-type"] == "application/cloudevents+json"
for key in attributes:
assert body[key] == attributes[key]
assert body["data"] == data, f"|{body_bytes}|| {body}"
async def test_predict_ce_structured(self, http_server_client):
event = dummy_cloud_event({"instances": [[1, 2]]})
headers, body = to_structured(event)
resp = await http_server_client.fetch('/v1/models/TestModel:predict',
method="POST",
headers=headers,
body=body)
body = json.loads(resp.body)
assert resp.code == 200
assert resp.headers['content-type'] == "application/cloudevents+json"
assert body["id"] != "36077800-0c23-4f38-a0b4-01f4369f670a"
assert body["data"] == {"predictions": [[1, 2]]}
assert body['specversion'] == "1.0"
assert body['source'] == "io.kserve.kfserver.TestModel"
assert body['type'] == "io.kserve.inference.response"
assert body['time'] > "2021-01-28T21:04:43.144141+00:00"
def test_tekton_source_event_processed(client):
attributes = {
"type": "tekton.foo",
"source": "https://example.com/event-producer",
"time": 0,
"id": "bar"
}
data = {"pipelineRun": {"metadata": {"uid": "foo"}}}
event = CloudEvent(attributes, data)
# Creates the HTTP request representation of the CloudEvent in structured content mode
headers, body = to_structured(event)
pubsub_msg = {
"message": {
"data": base64.b64encode(body).decode("utf-8"),
"attributes": {
"headers": json.dumps(headers)
},
"message_id": "foobar",
},
}
event = {
"event_type": "tekton.foo",
"id": "foo",
"metadata": body.decode(),
"time_created": 0,
"signature": "bar",
"msg_id": "foobar",
"source": "tekton",
}
shared.insert_row_into_bigquery = mock.MagicMock()
r = client.post(
"/",
data=json.dumps(pubsub_msg),
headers={"Content-Type": "application/json"},
)
shared.insert_row_into_bigquery.assert_called_with(event)
assert r.status_code == 204
def formatCloudEvent(message: str, subject: str) -> str:
"""
Returns a correctly formatted CloudEvents compliant event
"""
attributes = {
"specversion": "1.0",
"type": "org.tmforum.for.type.event.an.invented.burton.brian",
"source": "http://seccon.canvas.svc.cluster.local",
"subject": subject,
"id": str(uuid.uuid4()),
"time": datetime.datetime.now().isoformat(),
"datacontenttype": "application/json",
}
data = {"message": message}
event = CloudEvent(attributes, data)
headers, body = to_structured(event)
return body
def format_cloud_event(message: str, subject: str) -> str:
"""
Returns a correctly formatted CloudEvents compliant event
"""
attributes = {
'specversion': '1.0',
'type': 'org.tmforum.for.type.event.an.invented.burton.brian',
'source': 'http://seccon.canvas.svc.cluster.local',
'subject': subject,
'id': str(uuid.uuid4()),
'time': datetime.datetime.now().isoformat(),
'datacontenttype': 'application/json',
}
data = {'message': message}
event = CloudEvent(attributes, data)
_, body = to_structured(event)
return body
def test_roundtrip_non_json_event(converter, specversion):
input_data = io.BytesIO()
for _ in range(100):
for j in range(20):
assert 1 == input_data.write(j.to_bytes(1, byteorder="big"))
compressed_data = bz2.compress(input_data.getvalue())
attrs = {"source": "test", "type": "t"}
event = CloudEvent(attrs, compressed_data)
if converter == converters.TypeStructured:
headers, data = to_structured(event, data_marshaller=lambda x: x)
elif converter == converters.TypeBinary:
headers, data = to_binary(event, data_marshaller=lambda x: x)
headers["binary-payload"] = "true" # Decoding hint for server
_, r = app.test_client.post("/event", headers=headers, data=data)
assert r.status_code == 200
for key in attrs:
assert r.headers[key] == attrs[key]
assert compressed_data == r.body, r.body
def index():
# Gets the GCS bucket name from the CloudEvent header
# Example: "storage.googleapis.com/projects/_/buckets/my-bucket"
event = from_http(request.headers, request.get_data())
storage_data = StorageObjectData.from_dict(event.data)
print(
f"Detected change in GCS bucket: {storage_data.bucket}, object: {storage_data.name}"
)
attributes = {
"id": str(uuid.uuid4()),
"source": "https://localhost",
"specversion": "1.0",
"type": "com.example.kuberun.events.received",
}
data = {"message": "Event received"}
event = CloudEvent(attributes, data)
headers, body = to_structured(event)
response = make_response(body, 200)
response.headers.update(headers)
return response
async def test_predict_custom_ce_attributes(self, http_server_client):
with mock.patch.dict(
os.environ, {
"CE_SOURCE": "io.kserve.kfserver.CustomSource",
"CE_TYPE": "io.kserve.custom_type"
}):
event = dummy_cloud_event({"instances": [[1, 2]]})
headers, body = to_structured(event)
resp = await http_server_client.fetch(
'/v1/models/TestModel:predict',
method="POST",
headers=headers,
body=body)
body = json.loads(resp.body)
assert resp.code == 200
assert resp.headers[
'content-type'] == "application/cloudevents+json"
assert body["id"] != "36077800-0c23-4f38-a0b4-01f4369f670a"
assert body["data"] == {"predictions": [[1, 2]]}
assert body['source'] == "io.kserve.kfserver.CustomSource"
assert body['type'] == "io.kserve.custom_type"
def test_to_structured_http_deprecated(event):
with pytest.deprecated_call():
assert to_structured(event) == to_structured_http(event)
#!/usr/bin/python3
import logging
logging.basicConfig(filename='cloud_events.log', format='%(message)s', level=logging.INFO)
import pgpubsub, mysecrets
pubsub = pgpubsub.connect(host='35.233.160.178', user='******', database='main', password=mysecrets.DB_PASSWORD)
from cloudevents.http import CloudEvent, to_binary, to_structured
import requests, json
attributes = {
"type": "com.trifacta.lightweight",
"source": "35.233.160.178:5432",
}
import boto3
sns = boto3.client('sns', region_name='us-west-2', aws_access_key_id=mysecrets.KEY, aws_secret_access_key=mysecrets.SECRET)
pubsub.listen('cloud_events')
for e in pubsub.events():
event = CloudEvent(attributes, data=json.loads(e.payload))
headers, body = to_structured(event)
logging.info(body.decode())
sns.publish(TopicArn='arn:aws:sns:us-west-2:163305015547:cloud_events', Message=str(body))
def test_event(client, cloudevent_1_0):
headers, data = to_structured(cloudevent_1_0)
resp = client.post("/", headers=headers, data=data)
assert resp.status_code == 200
assert resp.data == b"OK"
def process_image():
# app.logger.debug(request.headers)
# create a CloudEvent
event = from_http(request.headers, request.get_data())
# you can access cloudevent fields as seen below
app.logger.info(
f"Found {event['id']} from {event['source']} with type "
f"{event['type']} and specversion {event['specversion']}"
)
# app.logger.info(event)
data = event.data
if 'image' in data and 'time' in data:
frame = convert_b64jpeg_to_image(data['image'].split(',')[1])
app.logger.info(data['time'] + " " + str(frame.shape))
# check / set cam ID
cam_id = 0
if 'id' in data:
cam_id = data['id']
# Call TF Yolo for object (damage) detection
start = time.time()
detected_classes, image_pred = my_tf.predict(frame)
end = time.time()
app.logger.info('Predict: Total object detection took {:.5f} seconds'.format(end - start))
if detected_classes:
app.logger.info(detected_classes)
status = 1
# Create a CloudEvent
# - The CloudEvent "id" is generated if omitted. "specversion" defaults to "1.0".
try:
attributes = {
'type': ce_action_type,
'source': ce_action_source,
}
event_data = {
'uuid': str(uuid.uuid4()), # TO-DO: Please with event uuid
'failure': detected_classes,
'status': status,
'time': data['time']
}
event = CloudEvent(attributes, event_data)
# Creates the HTTP request representation of the CloudEvent in structured content mode
headers, body = to_structured(event)
# POST
requests.post(kn_broker_url, data=body, headers=headers)
except:
app.logger.error(f'Failed to send CloudEvent to: {kn_broker_url}')
else:
status = 0
text = data['time']
# Respond with another event (optional)
response = make_response({
'text': text,
'id': cam_id,
'status': status,
'image': convert_image_to_jpeg(image_pred),
})
response.headers["Ce-Id"] = str(uuid.uuid4())
response.headers["Ce-specversion"] = "1.0"
response.headers["Ce-Source"] = "manuela/eventing/image-processor"
response.headers["Ce-Type"] = "manuela.image-processor.response"
else:
app.logger.warning("Payload not valid.")
response = make_response({
'msg': 'Payload not valid'
})
return response
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from cloudevents.http import CloudEvent, to_structured
import requests
# Create a cloudevent using https://github.com/cloudevents/sdk-python
# Note we only need source and type because the cloudevents constructor by
# default will set "specversion" to the most recent cloudevent version (e.g. 1.0)
# and "id" to a generated uuid.uuid4 string.
attributes = {
"Content-Type": "application/json",
"source": "from-galaxy-far-far-away",
"type": "cloudevent.greet.you",
}
data = {"name": "john"}
event = CloudEvent(attributes, data)
# Send the event to our local docker container listening on port 8080
headers, data = to_structured(event)
requests.post("http://localhost:8080/", headers=headers, data=data)
