def start_receiver(): """Start listening to HTTP requests :param func: the callback to call upon a cloudevents request :type func: cloudevent -> none """ m = marshaller.NewDefaultHTTPMarshaller() class BaseHttp(http.server.BaseHTTPRequestHandler): def do_POST(self): content_type = self.headers.get('Content-Type') content_len = int(self.headers.get('Content-Length')) headers = dict(self.headers) data = self.rfile.read(content_len) data = data.decode('utf-8') event = v02.Event() event = m.FromRequest(event, headers, data, str) run_event(event) self.send_response(204) self.end_headers() return class ThreadedCEServer(ThreadingMixIn, http.server.HTTPServer): "Deal with concurrent requests in a different thread" server = ThreadedCEServer(('', 8000), BaseHttp) print('Starting server on port 8000, use <Ctrl-C> to stop') try: server.serve_forever() except: server.server_close()
def _to_http( event: CloudEvent, format: str = converters.TypeStructured, data_marshaller: types.MarshallerType = None, ) -> typing.Tuple[dict, typing.Union[bytes, str]]: """ Returns a tuple of HTTP headers/body dicts representing this cloudevent :param format: constant specifying an encoding format :type format: str :param data_marshaller: Callable function to cast event.data into either a string or bytes :type data_marshaller: types.MarshallerType :returns: (http_headers: dict, http_body: bytes or str) """ if data_marshaller is None: data_marshaller = _marshaller_by_format[format] if event._attributes["specversion"] not in _obj_by_version: raise cloud_exceptions.InvalidRequiredFields( f"Unsupported specversion: {event._attributes['specversion']}") event_handler = _obj_by_version[event._attributes["specversion"]]() for k, v in event._attributes.items(): event_handler.Set(k, v) event_handler.data = event.data return marshaller.NewDefaultHTTPMarshaller().ToRequest( event_handler, format, data_marshaller=data_marshaller)
def main(): start_time = datetime.now() try: dispatch_policy = os.environ.get("DISPATCH_POLICY", DispatchPolicyConst.NEVER) m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v1.Event(), request.headers, io.BytesIO(request.data), lambda x: json.load(x)) event_info = event.Properties() event_info.update(event_info.pop("extensions")) event_data = event_info.pop("data") app.logger.debug("RCVR: {}".format(event_data)) event_type = event_info.get("type") subject = event_info.get("subject", "sys-0") if event_info["source"] == os.environ.get("K_SERVICE", os.environ.get("SOURCE")): return Response(status=201) event_info["originid"] = event_info.get("originid", event_info.get("id")) app.logger.debug("subject: {}".format(subject)) app.logger.debug("event_data: {}".format(event_data)) subject = subject_factory(name=subject, event_info=event_info, event_data=event_data) event_data["_event_info"] = event_info # TODO: KRUL-155 try: app.router.route(event_type, subject, event_data, dispatch_policy=dispatch_policy) finally: pass exec_time = (datetime.now() - start_time).total_seconds() app.logger.info( "Event", extra={ 'props': { 'event_info': event_info, 'type': event_type, 'subject': subject.name, 'exec_time': exec_time, # 'headers': list(headers.keys()) } }) return Response(status=200) except Exception as ex: app.logger.error(ex, exc_info=True) return Response(status=201)
def test_default_http_marshaller_with_binary(): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v02.Event(), data.headers, io.StringIO(json.dumps(data.body)), json.load) assert event is not None assert event.Get("type") == (data.ce_type, True) assert event.Get("data") == (data.body, True) assert event.Get("id") == (data.ce_id, True)
def test_extensions_are_set_upstream(): extensions = {"extension-key": "extension-value"} event = v1.Event().SetExtensions(extensions) m = marshaller.NewDefaultHTTPMarshaller() new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert event.Extensions() == extensions assert "ce-extension-key" in new_headers
def post(self): """ Handle post request. Extract data. Call event handler and optionally send a reply event. """ if not self.model.ready: self.model.load() try: body = json.loads(self.request.body) except json.decoder.JSONDecodeError as e: raise tornado.web.HTTPError( status_code=HTTPStatus.BAD_REQUEST, reason="Unrecognized request format: %s" % e, ) # Extract payload from request request_handler: RequestHandler = get_request_handler( self.protocol, body) request_handler.validate() request = request_handler.extract_request() # Create event from request body event = v02.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = http_marshaller.FromRequest(event, self.request.headers, self.request.body, json.loads) logging.debug(json.dumps(event.Properties())) # Extract any desired request headers headers = {} for (key, val) in self.request.headers.get_all(): headers[key] = val response = self.model.process_event(request, headers) if response is not None: responseStr = json.dumps(response) # Create event from response if reply_url is active if not self.reply_url == "": if event.EventID() is None or event.EventID() == "": resp_event_id = uuid.uuid1().hex else: resp_event_id = event.EventID() revent = ( v02.Event().SetContentType("application/json").SetData( responseStr).SetEventID(resp_event_id).SetSource( self.event_source).SetEventType( self.event_type).SetExtensions( event.Extensions())) logging.debug(json.dumps(revent.Properties())) sendCloudEvent(revent, self.reply_url) self.write(json.dumps(response))
def test_default_http_marshaller_with_binary(event_class): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(event_class(), data.headers[event_class], io.StringIO(json.dumps(data.body)), json.load) assert event is not None assert event.EventType() == data.ce_type assert event.EventID() == data.ce_id assert event.ContentType() == data.contentType assert event.Data() == data.body
def test_default_http_marshaller_with_structured(): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( v02.Event(), {"Content-Type": "application/cloudevents+json"}, io.StringIO(json.dumps(data.ce)), lambda x: x.read(), ) assert event is not None assert event.Get("type") == (data.ce_type, True) assert event.Get("id") == (data.ce_id, True)
def test_default_http_marshaller_with_structured(event_class): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, io.StringIO(json.dumps(data.json_ce[event_class])), lambda x: x.read(), ) assert event is not None assert event.EventType() == data.ce_type assert event.EventID() == data.ce_id assert event.ContentType() == data.contentType
def callback(message): print(message) print(sink_url) event = (v02.Event().SetContentType("application/json").SetData( message.data.decode()).SetEventID("xxx-yyy-zzz-www").SetSource( "google cloud storage").SetEventTime( str(message.attributes['eventTime'])).SetEventType( str(message.attributes['eventType']))) m = marshaller.NewDefaultHTTPMarshaller() headers, body = m.ToRequest(event, converters.TypeStructured, json.dumps) requests.post(sink_url, data=body.getvalue(), headers=headers) message.ack()
def test_structured_event_to_request_upstream(event_class): m = marshaller.NewDefaultHTTPMarshaller() http_headers = {"content-type": "application/cloudevents+json"} event = m.FromRequest(event_class(), http_headers, json.dumps(data.json_ce[event_class])) assert event is not None assert event.EventType() == data.ce_type assert event.EventID() == data.ce_id assert event.ContentType() == data.contentType new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x) for key in new_headers: if key == "content-type": assert new_headers[key] == http_headers[key] continue
def test_event_pipeline_upstream(event_class): event = (event_class().SetContentType(data.contentType).SetData( data.body).SetEventID(data.ce_id).SetSource(data.source).SetEventTime( data.eventTime).SetEventType(data.ce_type)) m = marshaller.NewDefaultHTTPMarshaller() new_headers, body = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert new_headers is not None assert "ce-specversion" in new_headers assert "ce-type" in new_headers assert "ce-source" in new_headers assert "ce-id" in new_headers assert "ce-time" in new_headers assert "content-type" in new_headers assert isinstance(body, bytes) assert data.body == body.decode("utf-8")
def test_binary_event_to_request_upstream(): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( v02.Event(), {"Content-Type": "application/cloudevents+json"}, io.StringIO(json.dumps(data.ce)), lambda x: x.read(), ) assert event is not None assert event.Get("type") == (data.ce_type, True) assert event.Get("id") == (data.ce_id, True) new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert new_headers is not None assert "ce-specversion" in new_headers
def test_binary_event_to_request_upstream(event_class): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, json.dumps(data.json_ce[event_class]), ) assert event is not None assert event.EventType() == data.ce_type assert event.EventID() == data.ce_id assert event.ContentType() == data.contentType new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert new_headers is not None assert "ce-specversion" in new_headers
def test_structured_event_to_request_upstream(): copy_of_ce = copy.deepcopy(data.ce) m = marshaller.NewDefaultHTTPMarshaller() http_headers = {"content-type": "application/cloudevents+json"} event = m.FromRequest(v02.Event(), http_headers, io.StringIO(json.dumps(data.ce)), lambda x: x.read()) assert event is not None assert event.Get("type") == (data.ce_type, True) assert event.Get("id") == (data.ce_id, True) new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x) for key in new_headers: if key == "content-type": assert new_headers[key] == http_headers[key] continue assert key in copy_of_ce
def test_general_structured_properties(event_class): copy_of_ce = copy.deepcopy(data.json_ce[event_class]) m = marshaller.NewDefaultHTTPMarshaller() http_headers = {"content-type": "application/cloudevents+json"} event = m.FromRequest( event_class(), http_headers, json.dumps(data.json_ce[event_class]), lambda x: x, ) # Test python properties assert event is not None assert event.type == data.ce_type assert event.id == data.ce_id assert event.content_type == data.contentType assert event.source == data.source new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x) for key in new_headers: if key == "content-type": assert new_headers[key] == http_headers[key] continue assert key in copy_of_ce # Test setters new_type = str(uuid4()) new_id = str(uuid4()) new_content_type = str(uuid4()) new_source = str(uuid4()) event.extensions = {"test": str(uuid4)} event.type = new_type event.id = new_id event.content_type = new_content_type event.source = new_source assert event is not None assert (event.type == new_type) and (event.type == event.EventType()) assert (event.id == new_id) and (event.id == event.EventID()) assert (event.content_type == new_content_type) and (event.content_type == event.ContentType()) assert (event.source == new_source) and (event.source == event.Source()) assert event.extensions["test"] == event.Extensions()["test"] assert event.specversion == event.CloudEventVersion()
def from_http( data: typing.Union[str, bytes], headers: typing.Dict[str, str], data_unmarshaller: types.UnmarshallerType = None, ): """ Unwrap a CloudEvent (binary or structured) from an HTTP request. :param data: the HTTP request body :type data: typing.IO :param headers: the HTTP headers :type headers: typing.Dict[str, str] :param data_unmarshaller: Callable function to map data to a python object e.g. lambda x: x or lambda x: json.loads(x) :type data_unmarshaller: types.UnmarshallerType """ if data_unmarshaller is None: data_unmarshaller = _json_or_string marshall = marshaller.NewDefaultHTTPMarshaller() if converters.is_binary(headers): specversion = headers.get("ce-specversion", None) else: raw_ce = json.loads(data) specversion = raw_ce.get("specversion", None) if specversion is None: raise ValueError("could not find specversion in HTTP request") event_handler = _obj_by_version.get(specversion, None) if event_handler is None: raise ValueError(f"found invalid specversion {specversion}") event = marshall.FromRequest(event_handler(), headers, data, data_unmarshaller=data_unmarshaller) attrs = event.Properties() attrs.pop("data", None) attrs.pop("extensions", None) attrs.update(**event.extensions) return CloudEvent(attrs, event.data)
def fake_receiver_app(environ, start_response): """Simplest possible WSGI application""" request = Request(environ) m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v1.Event(), request.headers, io.BytesIO(request.data), lambda x: json.load(x)) event_info = event.Properties() event_info.update(event_info.pop("extensions")) subject = subject_factory(event_info.get("subject", "sys-0")) assert "originid" in event_info assert "subject" in event_info assert "data" in event_info status = '200 OK' response_headers = [('Content-type', 'text/plain')] start_response(status, response_headers) return ['Ok']
def POST(self): """Accept the post data and return it.""" connection = pika.BlockingConnection( pika.ConnectionParameters( os.getenv('HOST', 'rabbitmq'), int(os.getenv('PORT', '5672')), os.getenv('VHOST', '/'), pika.PlainCredentials( os.getenv('USER', 'guest'), os.getenv('PASS', 'guest') ) ) ) main_channel = connection.channel() main_channel.exchange_declare(exchange='gov.pnnl.datahub.events', durable=True, exchange_type='direct') headers = cherrypy.request.headers headers['Content-Type'] = 'application/cloudevents+json' raw_data = cherrypy.request.body.read().decode('utf8') print(raw_data) data = io.StringIO(raw_data) event = v01.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = http_marshaller.FromRequest(event, headers, data, lambda x: x) print(event.EventType()) main_channel.basic_publish( exchange='gov.pnnl.datahub.events', routing_key=event.EventType(), body=json.dumps(event.Data()), properties=pika.BasicProperties( content_type='application/json', headers={ 'cloudEvents:specversion': event.CloudEventVersion(), 'cloudEvents:type': event.EventType(), 'cloudEvents:id': event.EventID(), 'cloudEvents:source': event.Source() } ) ) connection.close() return {'message': 'success'}
def test_object_event_v1(): event = (v1.Event().SetContentType("application/json").SetData( {"name": "john"})) m = marshaller.NewDefaultHTTPMarshaller() _, structuredBody = m.ToRequest(event) assert isinstance(structuredBody, bytes) structuredObj = json.loads(structuredBody) errorMsg = f"Body was {structuredBody}, obj is {structuredObj}" assert isinstance(structuredObj, dict), errorMsg assert isinstance(structuredObj["data"], dict), errorMsg assert len(structuredObj["data"]) == 1, errorMsg assert structuredObj["data"]["name"] == "john", errorMsg headers, binaryBody = m.ToRequest(event, converters.TypeBinary) assert isinstance(headers, dict) assert isinstance(binaryBody, bytes) assert headers["content-type"] == "application/json" assert binaryBody == b'{"name": "john"}', f"Binary is {binaryBody!r}"
def test_general_binary_properties(event_class): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, json.dumps(data.json_ce[event_class]), lambda x: x.read(), ) new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert new_headers is not None assert "ce-specversion" in new_headers # Test properties assert event is not None assert event.type == data.ce_type assert event.id == data.ce_id assert event.content_type == data.contentType assert event.source == data.source # Test setters new_type = str(uuid4()) new_id = str(uuid4()) new_content_type = str(uuid4()) new_source = str(uuid4()) event.extensions = {"test": str(uuid4)} event.type = new_type event.id = new_id event.content_type = new_content_type event.source = new_source assert event is not None assert (event.type == new_type) and (event.type == event.EventType()) assert (event.id == new_id) and (event.id == event.EventID()) assert (event.content_type == new_content_type) and ( event.content_type == event.ContentType() ) assert (event.source == new_source) and (event.source == event.Source()) assert event.extensions["test"] == event.Extensions()["test"] assert event.specversion == event.CloudEventVersion()
def sendCloudEvent(event: v02.Event, url: str): """ Send CloudEvent Parameters ---------- event CloudEvent to send url Url to send event """ http_marshaller = marshaller.NewDefaultHTTPMarshaller() binary_headers, binary_data = http_marshaller.ToRequest( event, converters.TypeBinary, json.dumps) print("binary CloudEvent") for k, v in binary_headers.items(): print("{0}: {1}\r\n".format(k, v)) print(binary_data) response = requests.post(url, headers=binary_headers, data=binary_data) response.raise_for_status()
def test_invalid_data_marshaller(): m = marshaller.NewDefaultHTTPMarshaller() pytest.raises(exceptions.InvalidDataMarshaller, m.ToRequest, v01.Event(), "blah", None)
def test_invalid_data_unmarshaller(): m = marshaller.NewDefaultHTTPMarshaller() pytest.raises(exceptions.InvalidDataUnmarshaller, m.FromRequest, v01.Event(), {}, None, None)
def from_http( headers: typing.Dict[str, str], data: typing.Union[str, bytes, None], data_unmarshaller: types.UnmarshallerType = None, ): """ Unwrap a CloudEvent (binary or structured) from an HTTP request. :param headers: the HTTP headers :type headers: typing.Dict[str, str] :param data: the HTTP request body. If set to None, "" or b'', the returned event's data field will be set to None :type data: typing.IO :param data_unmarshaller: Callable function to map data to a python object e.g. lambda x: x or lambda x: json.loads(x) :type data_unmarshaller: types.UnmarshallerType """ if data is None or data == b"": # Empty string will cause data to be marshalled into None data = "" if not isinstance(data, (str, bytes, bytearray)): raise cloud_exceptions.InvalidStructuredJSON( "Expected json of type (str, bytes, bytearray), " f"but instead found type {type(data)}") headers = {key.lower(): value for key, value in headers.items()} if data_unmarshaller is None: data_unmarshaller = _json_or_string marshall = marshaller.NewDefaultHTTPMarshaller() if is_binary(headers): specversion = headers.get("ce-specversion", None) else: try: raw_ce = json.loads(data) except json.decoder.JSONDecodeError: raise cloud_exceptions.MissingRequiredFields( "Failed to read specversion from both headers and data. " f"The following can not be parsed as json: {data}") if hasattr(raw_ce, "get"): specversion = raw_ce.get("specversion", None) else: raise cloud_exceptions.MissingRequiredFields( "Failed to read specversion from both headers and data. " f"The following deserialized data has no 'get' method: {raw_ce}" ) if specversion is None: raise cloud_exceptions.MissingRequiredFields( "Failed to find specversion in HTTP request") event_handler = _obj_by_version.get(specversion, None) if event_handler is None: raise cloud_exceptions.InvalidRequiredFields( f"Found invalid specversion {specversion}") event = marshall.FromRequest(event_handler(), headers, data, data_unmarshaller=data_unmarshaller) attrs = event.Properties() attrs.pop("data", None) attrs.pop("extensions", None) attrs.update(**event.extensions) if event.data == "" or event.data == b"": # TODO: Check binary unmarshallers to debug why setting data to "" # returns an event with data set to None, but structured will return "" data = None else: data = event.data return CloudEvent(attrs, data)
def handle(): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest(v02.Event(), request.headers, request.data, unpack) return func(event.Data(), event)
db_password = os.environ['database-password'] db_host = os.environ['database-host'] db_db = os.environ['database-db'] model_version = os.environ['model_version'] logging.basicConfig(stream=sys.stdout, level=logging.INFO) s3client = boto3.client('s3', 'us-east-1', endpoint_url=service_point, aws_access_key_id=access_key, aws_secret_access_key=secret_key, use_ssl=True if 'https' in service_point else False) m = marshaller.NewDefaultHTTPMarshaller() class ForkedHTTPServer(socketserver.ForkingMixIn, http.server.HTTPServer): """Handle requests with fork.""" class CloudeventsServer(object): """Listen for incoming HTTP cloudevents requests. cloudevents request is simply a HTTP Post request following a well-defined of how to pass the event data. """ def __init__(self, port=8080): self.port = port def start_receiver(self, func):
def test_to_request_wrong_marshaller(): with pytest.raises(exceptions.InvalidDataMarshaller): m = marshaller.NewDefaultHTTPMarshaller() _ = m.ToRequest(v1.Event(), data_marshaller="")
def test_from_request_wrong_unmarshaller(): with pytest.raises(exceptions.InvalidDataUnmarshaller): m = marshaller.NewDefaultHTTPMarshaller() _ = m.FromRequest(v1.Event(), {}, "", None)
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import json import io import requests import sys from cloudevents.sdk import marshaller from cloudevents.sdk.event import v02 if __name__ == "__main__": if len(sys.argv) < 2: sys.exit("Usage: python with_requests.py " "<CloudEvent source URL>") url = sys.argv[1] response = requests.get(url) response.raise_for_status() headers = response.headers data = io.BytesIO(response.content) event = v02.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = http_marshaller.FromRequest(event, headers, data, json.load) print(json.dumps(event.Properties()))