def start_receiver():
"""Start listening to HTTP requests
:param func: the callback to call upon a cloudevents request
:type func: cloudevent -> none
"""
m = marshaller.NewDefaultHTTPMarshaller()
class BaseHttp(http.server.BaseHTTPRequestHandler):
def do_POST(self):
content_type = self.headers.get('Content-Type')
content_len = int(self.headers.get('Content-Length'))
headers = dict(self.headers)
data = self.rfile.read(content_len)
data = data.decode('utf-8')
event = v02.Event()
event = m.FromRequest(event, headers, data, str)
run_event(event)
self.send_response(204)
self.end_headers()
return
class ThreadedCEServer(ThreadingMixIn, http.server.HTTPServer):
"Deal with concurrent requests in a different thread"
server = ThreadedCEServer(('', 8000), BaseHttp)
print('Starting server on port 8000, use <Ctrl-C> to stop')
try:
server.serve_forever()
except:
server.server_close()
def _to_http(
event: CloudEvent,
format: str = converters.TypeStructured,
data_marshaller: types.MarshallerType = None,
) -> typing.Tuple[dict, typing.Union[bytes, str]]:
"""
Returns a tuple of HTTP headers/body dicts representing this cloudevent
:param format: constant specifying an encoding format
:type format: str
:param data_marshaller: Callable function to cast event.data into
either a string or bytes
:type data_marshaller: types.MarshallerType
:returns: (http_headers: dict, http_body: bytes or str)
"""
if data_marshaller is None:
data_marshaller = _marshaller_by_format[format]
if event._attributes["specversion"] not in _obj_by_version:
raise cloud_exceptions.InvalidRequiredFields(
f"Unsupported specversion: {event._attributes['specversion']}")
event_handler = _obj_by_version[event._attributes["specversion"]]()
for k, v in event._attributes.items():
event_handler.Set(k, v)
event_handler.data = event.data
return marshaller.NewDefaultHTTPMarshaller().ToRequest(
event_handler, format, data_marshaller=data_marshaller)
def main():
start_time = datetime.now()
try:
dispatch_policy = os.environ.get("DISPATCH_POLICY",
DispatchPolicyConst.NEVER)
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v1.Event(), request.headers,
io.BytesIO(request.data), lambda x: json.load(x))
event_info = event.Properties()
event_info.update(event_info.pop("extensions"))
event_data = event_info.pop("data")
app.logger.debug("RCVR: {}".format(event_data))
event_type = event_info.get("type")
subject = event_info.get("subject", "sys-0")
if event_info["source"] == os.environ.get("K_SERVICE",
os.environ.get("SOURCE")):
return Response(status=201)
event_info["originid"] = event_info.get("originid",
event_info.get("id"))
app.logger.debug("subject: {}".format(subject))
app.logger.debug("event_data: {}".format(event_data))
subject = subject_factory(name=subject,
event_info=event_info,
event_data=event_data)
event_data["_event_info"] = event_info # TODO: KRUL-155
try:
app.router.route(event_type,
subject,
event_data,
dispatch_policy=dispatch_policy)
finally:
pass
exec_time = (datetime.now() - start_time).total_seconds()
app.logger.info(
"Event",
extra={
'props': {
'event_info': event_info,
'type': event_type,
'subject': subject.name,
'exec_time': exec_time,
# 'headers': list(headers.keys())
}
})
return Response(status=200)
except Exception as ex:
app.logger.error(ex, exc_info=True)
return Response(status=201)
def test_default_http_marshaller_with_binary():
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v02.Event(), data.headers,
io.StringIO(json.dumps(data.body)), json.load)
assert event is not None
assert event.Get("type") == (data.ce_type, True)
assert event.Get("data") == (data.body, True)
assert event.Get("id") == (data.ce_id, True)
def test_extensions_are_set_upstream():
extensions = {"extension-key": "extension-value"}
event = v1.Event().SetExtensions(extensions)
m = marshaller.NewDefaultHTTPMarshaller()
new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x)
assert event.Extensions() == extensions
assert "ce-extension-key" in new_headers
def post(self):
"""
Handle post request. Extract data. Call event handler and optionally send a reply event.
"""
if not self.model.ready:
self.model.load()
try:
body = json.loads(self.request.body)
except json.decoder.JSONDecodeError as e:
raise tornado.web.HTTPError(
status_code=HTTPStatus.BAD_REQUEST,
reason="Unrecognized request format: %s" % e,
)
# Extract payload from request
request_handler: RequestHandler = get_request_handler(
self.protocol, body)
request_handler.validate()
request = request_handler.extract_request()
# Create event from request body
event = v02.Event()
http_marshaller = marshaller.NewDefaultHTTPMarshaller()
event = http_marshaller.FromRequest(event, self.request.headers,
self.request.body, json.loads)
logging.debug(json.dumps(event.Properties()))
# Extract any desired request headers
headers = {}
for (key, val) in self.request.headers.get_all():
headers[key] = val
response = self.model.process_event(request, headers)
if response is not None:
responseStr = json.dumps(response)
# Create event from response if reply_url is active
if not self.reply_url == "":
if event.EventID() is None or event.EventID() == "":
resp_event_id = uuid.uuid1().hex
else:
resp_event_id = event.EventID()
revent = (
v02.Event().SetContentType("application/json").SetData(
responseStr).SetEventID(resp_event_id).SetSource(
self.event_source).SetEventType(
self.event_type).SetExtensions(
event.Extensions()))
logging.debug(json.dumps(revent.Properties()))
sendCloudEvent(revent, self.reply_url)
self.write(json.dumps(response))
def test_default_http_marshaller_with_binary(event_class):
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(event_class(), data.headers[event_class],
io.StringIO(json.dumps(data.body)), json.load)
assert event is not None
assert event.EventType() == data.ce_type
assert event.EventID() == data.ce_id
assert event.ContentType() == data.contentType
assert event.Data() == data.body
def test_default_http_marshaller_with_structured():
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(
v02.Event(),
{"Content-Type": "application/cloudevents+json"},
io.StringIO(json.dumps(data.ce)),
lambda x: x.read(),
)
assert event is not None
assert event.Get("type") == (data.ce_type, True)
assert event.Get("id") == (data.ce_id, True)
def test_default_http_marshaller_with_structured(event_class):
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(
event_class(),
{"Content-Type": "application/cloudevents+json"},
io.StringIO(json.dumps(data.json_ce[event_class])),
lambda x: x.read(),
)
assert event is not None
assert event.EventType() == data.ce_type
assert event.EventID() == data.ce_id
assert event.ContentType() == data.contentType
def callback(message):
print(message)
print(sink_url)
event = (v02.Event().SetContentType("application/json").SetData(
message.data.decode()).SetEventID("xxx-yyy-zzz-www").SetSource(
"google cloud storage").SetEventTime(
str(message.attributes['eventTime'])).SetEventType(
str(message.attributes['eventType'])))
m = marshaller.NewDefaultHTTPMarshaller()
headers, body = m.ToRequest(event, converters.TypeStructured, json.dumps)
requests.post(sink_url, data=body.getvalue(), headers=headers)
message.ack()
def test_structured_event_to_request_upstream(event_class):
m = marshaller.NewDefaultHTTPMarshaller()
http_headers = {"content-type": "application/cloudevents+json"}
event = m.FromRequest(event_class(), http_headers,
json.dumps(data.json_ce[event_class]))
assert event is not None
assert event.EventType() == data.ce_type
assert event.EventID() == data.ce_id
assert event.ContentType() == data.contentType
new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x)
for key in new_headers:
if key == "content-type":
assert new_headers[key] == http_headers[key]
continue
def test_event_pipeline_upstream(event_class):
event = (event_class().SetContentType(data.contentType).SetData(
data.body).SetEventID(data.ce_id).SetSource(data.source).SetEventTime(
data.eventTime).SetEventType(data.ce_type))
m = marshaller.NewDefaultHTTPMarshaller()
new_headers, body = m.ToRequest(event, converters.TypeBinary, lambda x: x)
assert new_headers is not None
assert "ce-specversion" in new_headers
assert "ce-type" in new_headers
assert "ce-source" in new_headers
assert "ce-id" in new_headers
assert "ce-time" in new_headers
assert "content-type" in new_headers
assert isinstance(body, bytes)
assert data.body == body.decode("utf-8")
def test_binary_event_to_request_upstream():
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(
v02.Event(),
{"Content-Type": "application/cloudevents+json"},
io.StringIO(json.dumps(data.ce)),
lambda x: x.read(),
)
assert event is not None
assert event.Get("type") == (data.ce_type, True)
assert event.Get("id") == (data.ce_id, True)
new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x)
assert new_headers is not None
assert "ce-specversion" in new_headers
def test_binary_event_to_request_upstream(event_class):
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(
event_class(),
{"Content-Type": "application/cloudevents+json"},
json.dumps(data.json_ce[event_class]),
)
assert event is not None
assert event.EventType() == data.ce_type
assert event.EventID() == data.ce_id
assert event.ContentType() == data.contentType
new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x)
assert new_headers is not None
assert "ce-specversion" in new_headers
def test_structured_event_to_request_upstream():
copy_of_ce = copy.deepcopy(data.ce)
m = marshaller.NewDefaultHTTPMarshaller()
http_headers = {"content-type": "application/cloudevents+json"}
event = m.FromRequest(v02.Event(), http_headers,
io.StringIO(json.dumps(data.ce)), lambda x: x.read())
assert event is not None
assert event.Get("type") == (data.ce_type, True)
assert event.Get("id") == (data.ce_id, True)
new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x)
for key in new_headers:
if key == "content-type":
assert new_headers[key] == http_headers[key]
continue
assert key in copy_of_ce
def test_general_structured_properties(event_class):
copy_of_ce = copy.deepcopy(data.json_ce[event_class])
m = marshaller.NewDefaultHTTPMarshaller()
http_headers = {"content-type": "application/cloudevents+json"}
event = m.FromRequest(
event_class(),
http_headers,
json.dumps(data.json_ce[event_class]),
lambda x: x,
)
# Test python properties
assert event is not None
assert event.type == data.ce_type
assert event.id == data.ce_id
assert event.content_type == data.contentType
assert event.source == data.source
new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x)
for key in new_headers:
if key == "content-type":
assert new_headers[key] == http_headers[key]
continue
assert key in copy_of_ce
# Test setters
new_type = str(uuid4())
new_id = str(uuid4())
new_content_type = str(uuid4())
new_source = str(uuid4())
event.extensions = {"test": str(uuid4)}
event.type = new_type
event.id = new_id
event.content_type = new_content_type
event.source = new_source
assert event is not None
assert (event.type == new_type) and (event.type == event.EventType())
assert (event.id == new_id) and (event.id == event.EventID())
assert (event.content_type
== new_content_type) and (event.content_type
== event.ContentType())
assert (event.source == new_source) and (event.source == event.Source())
assert event.extensions["test"] == event.Extensions()["test"]
assert event.specversion == event.CloudEventVersion()
def from_http(
data: typing.Union[str, bytes],
headers: typing.Dict[str, str],
data_unmarshaller: types.UnmarshallerType = None,
):
"""
Unwrap a CloudEvent (binary or structured) from an HTTP request.
:param data: the HTTP request body
:type data: typing.IO
:param headers: the HTTP headers
:type headers: typing.Dict[str, str]
:param data_unmarshaller: Callable function to map data to a python object
e.g. lambda x: x or lambda x: json.loads(x)
:type data_unmarshaller: types.UnmarshallerType
"""
if data_unmarshaller is None:
data_unmarshaller = _json_or_string
marshall = marshaller.NewDefaultHTTPMarshaller()
if converters.is_binary(headers):
specversion = headers.get("ce-specversion", None)
else:
raw_ce = json.loads(data)
specversion = raw_ce.get("specversion", None)
if specversion is None:
raise ValueError("could not find specversion in HTTP request")
event_handler = _obj_by_version.get(specversion, None)
if event_handler is None:
raise ValueError(f"found invalid specversion {specversion}")
event = marshall.FromRequest(event_handler(),
headers,
data,
data_unmarshaller=data_unmarshaller)
attrs = event.Properties()
attrs.pop("data", None)
attrs.pop("extensions", None)
attrs.update(**event.extensions)
return CloudEvent(attrs, event.data)
def fake_receiver_app(environ, start_response):
"""Simplest possible WSGI application"""
request = Request(environ)
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v1.Event(), request.headers,
io.BytesIO(request.data), lambda x: json.load(x))
event_info = event.Properties()
event_info.update(event_info.pop("extensions"))
subject = subject_factory(event_info.get("subject", "sys-0"))
assert "originid" in event_info
assert "subject" in event_info
assert "data" in event_info
status = '200 OK'
response_headers = [('Content-type', 'text/plain')]
start_response(status, response_headers)
return ['Ok']
def POST(self):
"""Accept the post data and return it."""
connection = pika.BlockingConnection(
pika.ConnectionParameters(
os.getenv('HOST', 'rabbitmq'),
int(os.getenv('PORT', '5672')),
os.getenv('VHOST', '/'),
pika.PlainCredentials(
os.getenv('USER', 'guest'),
os.getenv('PASS', 'guest')
)
)
)
main_channel = connection.channel()
main_channel.exchange_declare(exchange='gov.pnnl.datahub.events', durable=True, exchange_type='direct')
headers = cherrypy.request.headers
headers['Content-Type'] = 'application/cloudevents+json'
raw_data = cherrypy.request.body.read().decode('utf8')
print(raw_data)
data = io.StringIO(raw_data)
event = v01.Event()
http_marshaller = marshaller.NewDefaultHTTPMarshaller()
event = http_marshaller.FromRequest(event, headers, data, lambda x: x)
print(event.EventType())
main_channel.basic_publish(
exchange='gov.pnnl.datahub.events',
routing_key=event.EventType(),
body=json.dumps(event.Data()),
properties=pika.BasicProperties(
content_type='application/json',
headers={
'cloudEvents:specversion': event.CloudEventVersion(),
'cloudEvents:type': event.EventType(),
'cloudEvents:id': event.EventID(),
'cloudEvents:source': event.Source()
}
)
)
connection.close()
return {'message': 'success'}
def test_object_event_v1():
event = (v1.Event().SetContentType("application/json").SetData(
{"name": "john"}))
m = marshaller.NewDefaultHTTPMarshaller()
_, structuredBody = m.ToRequest(event)
assert isinstance(structuredBody, bytes)
structuredObj = json.loads(structuredBody)
errorMsg = f"Body was {structuredBody}, obj is {structuredObj}"
assert isinstance(structuredObj, dict), errorMsg
assert isinstance(structuredObj["data"], dict), errorMsg
assert len(structuredObj["data"]) == 1, errorMsg
assert structuredObj["data"]["name"] == "john", errorMsg
headers, binaryBody = m.ToRequest(event, converters.TypeBinary)
assert isinstance(headers, dict)
assert isinstance(binaryBody, bytes)
assert headers["content-type"] == "application/json"
assert binaryBody == b'{"name": "john"}', f"Binary is {binaryBody!r}"
def test_general_binary_properties(event_class):
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(
event_class(),
{"Content-Type": "application/cloudevents+json"},
json.dumps(data.json_ce[event_class]),
lambda x: x.read(),
)
new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x)
assert new_headers is not None
assert "ce-specversion" in new_headers
# Test properties
assert event is not None
assert event.type == data.ce_type
assert event.id == data.ce_id
assert event.content_type == data.contentType
assert event.source == data.source
# Test setters
new_type = str(uuid4())
new_id = str(uuid4())
new_content_type = str(uuid4())
new_source = str(uuid4())
event.extensions = {"test": str(uuid4)}
event.type = new_type
event.id = new_id
event.content_type = new_content_type
event.source = new_source
assert event is not None
assert (event.type == new_type) and (event.type == event.EventType())
assert (event.id == new_id) and (event.id == event.EventID())
assert (event.content_type == new_content_type) and (
event.content_type == event.ContentType()
)
assert (event.source == new_source) and (event.source == event.Source())
assert event.extensions["test"] == event.Extensions()["test"]
assert event.specversion == event.CloudEventVersion()
def sendCloudEvent(event: v02.Event, url: str):
"""
Send CloudEvent
Parameters
----------
event
CloudEvent to send
url
Url to send event
"""
http_marshaller = marshaller.NewDefaultHTTPMarshaller()
binary_headers, binary_data = http_marshaller.ToRequest(
event, converters.TypeBinary, json.dumps)
print("binary CloudEvent")
for k, v in binary_headers.items():
print("{0}: {1}\r\n".format(k, v))
print(binary_data)
response = requests.post(url, headers=binary_headers, data=binary_data)
response.raise_for_status()
def test_invalid_data_marshaller():
m = marshaller.NewDefaultHTTPMarshaller()
pytest.raises(exceptions.InvalidDataMarshaller, m.ToRequest, v01.Event(),
"blah", None)
def test_invalid_data_unmarshaller():
m = marshaller.NewDefaultHTTPMarshaller()
pytest.raises(exceptions.InvalidDataUnmarshaller, m.FromRequest,
v01.Event(), {}, None, None)
def from_http(
headers: typing.Dict[str, str],
data: typing.Union[str, bytes, None],
data_unmarshaller: types.UnmarshallerType = None,
):
"""
Unwrap a CloudEvent (binary or structured) from an HTTP request.
:param headers: the HTTP headers
:type headers: typing.Dict[str, str]
:param data: the HTTP request body. If set to None, "" or b'', the returned
event's data field will be set to None
:type data: typing.IO
:param data_unmarshaller: Callable function to map data to a python object
e.g. lambda x: x or lambda x: json.loads(x)
:type data_unmarshaller: types.UnmarshallerType
"""
if data is None or data == b"":
# Empty string will cause data to be marshalled into None
data = ""
if not isinstance(data, (str, bytes, bytearray)):
raise cloud_exceptions.InvalidStructuredJSON(
"Expected json of type (str, bytes, bytearray), "
f"but instead found type {type(data)}")
headers = {key.lower(): value for key, value in headers.items()}
if data_unmarshaller is None:
data_unmarshaller = _json_or_string
marshall = marshaller.NewDefaultHTTPMarshaller()
if is_binary(headers):
specversion = headers.get("ce-specversion", None)
else:
try:
raw_ce = json.loads(data)
except json.decoder.JSONDecodeError:
raise cloud_exceptions.MissingRequiredFields(
"Failed to read specversion from both headers and data. "
f"The following can not be parsed as json: {data}")
if hasattr(raw_ce, "get"):
specversion = raw_ce.get("specversion", None)
else:
raise cloud_exceptions.MissingRequiredFields(
"Failed to read specversion from both headers and data. "
f"The following deserialized data has no 'get' method: {raw_ce}"
)
if specversion is None:
raise cloud_exceptions.MissingRequiredFields(
"Failed to find specversion in HTTP request")
event_handler = _obj_by_version.get(specversion, None)
if event_handler is None:
raise cloud_exceptions.InvalidRequiredFields(
f"Found invalid specversion {specversion}")
event = marshall.FromRequest(event_handler(),
headers,
data,
data_unmarshaller=data_unmarshaller)
attrs = event.Properties()
attrs.pop("data", None)
attrs.pop("extensions", None)
attrs.update(**event.extensions)
if event.data == "" or event.data == b"":
# TODO: Check binary unmarshallers to debug why setting data to ""
# returns an event with data set to None, but structured will return ""
data = None
else:
data = event.data
return CloudEvent(attrs, data)
def handle():
m = marshaller.NewDefaultHTTPMarshaller()
event = m.FromRequest(v02.Event(), request.headers, request.data,
unpack)
return func(event.Data(), event)
db_password = os.environ['database-password']
db_host = os.environ['database-host']
db_db = os.environ['database-db']
model_version = os.environ['model_version']
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
s3client = boto3.client('s3',
'us-east-1',
endpoint_url=service_point,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
use_ssl=True if 'https' in service_point else False)
m = marshaller.NewDefaultHTTPMarshaller()
class ForkedHTTPServer(socketserver.ForkingMixIn, http.server.HTTPServer):
"""Handle requests with fork."""
class CloudeventsServer(object):
"""Listen for incoming HTTP cloudevents requests.
cloudevents request is simply a HTTP Post request following a well-defined
of how to pass the event data.
"""
def __init__(self, port=8080):
self.port = port
def start_receiver(self, func):
def test_to_request_wrong_marshaller():
with pytest.raises(exceptions.InvalidDataMarshaller):
m = marshaller.NewDefaultHTTPMarshaller()
_ = m.ToRequest(v1.Event(), data_marshaller="")
def test_from_request_wrong_unmarshaller():
with pytest.raises(exceptions.InvalidDataUnmarshaller):
m = marshaller.NewDefaultHTTPMarshaller()
_ = m.FromRequest(v1.Event(), {}, "", None)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import io
import requests
import sys
from cloudevents.sdk import marshaller
from cloudevents.sdk.event import v02
if __name__ == "__main__":
if len(sys.argv) < 2:
sys.exit("Usage: python with_requests.py " "<CloudEvent source URL>")
url = sys.argv[1]
response = requests.get(url)
response.raise_for_status()
headers = response.headers
data = io.BytesIO(response.content)
event = v02.Event()
http_marshaller = marshaller.NewDefaultHTTPMarshaller()
event = http_marshaller.FromRequest(event, headers, data, json.load)
print(json.dumps(event.Properties()))
