def raise_for_length(req, limit=_MAXLEN):
# two possible error cases: no length specified, or length exceeds limit
# raise appropriate exception if either applies
length = req.content_length
if length is None:
raise HTTPLengthRequired
if length > limit:
raise HTTPRequestEntityTooLarge("Max size: {0} kB".format(limit /
2**10))
def validate_body_length(event):
content_length = 0
max_clength = 160000
if event.request.method == 'POST':
content_length = event.request.headers.get('Content-Length')
if not content_length:
raise HTTPBadRequest(
'No Content-Length was provided in request header'
) # pragam: no cover
elif event.request.method == 'GET':
content_length = len(event.request.params.get('geom', ''))
if int(content_length) > max_clength:
raise HTTPRequestEntityTooLarge(
'LineString too large, maximum Content-Length ' +
'of %s exceeded' % max_clength)
def put_key(request):
"""Uploads new key-retrieval information."""
# Validate that the request is text/<something>
if request.content_type:
if not request.content_type.startswith("text/"):
raise HTTPUnsupportedMediaType()
# Validate that the request is not too big
if request.content_length is None:
raise HTTPLengthRequired()
if request.content_length > 8 * 1024:
raise HTTPRequestEntityTooLarge()
# Store the uploaded data.
username = request.matchdict["username"]
store = request.registry.getUtility(IKeyRetrievalStorage)
store.set(username, request.body)
return Response(status=204)
def linestring(self, value):
if value is None or value == '':
raise HTTPBadRequest("Missing parameter geom")
try:
geom = geojson.loads(value,
object_hook=geojson.GeoJSON.to_instance)
except ValueError:
raise HTTPBadRequest("Error loading geometry in JSON string")
try:
geomToShape = shape(geom)
except Exception:
raise HTTPBadRequest("Error converting JSON to Shape")
try:
geomToShape.is_valid
except Exception:
raise HTTPBadRequest("Invalid Linestring syntax")
if len(geomToShape.coords) > self.nb_points_max:
raise HTTPRequestEntityTooLarge(
"Request Geometry contains too many points. Maximum number of points allowed: {}, found {}"
.format(self.nb_points_max, len(geomToShape.coords)))
self._linestring = geomToShape
def set_item(self, request):
"""Upload a new item by ID."""
# Validate the incoming data.
if request.content_type not in ("application/json", None):
msg = "Unsupported Media Type: %s" % (request.content_type, )
raise HTTPUnsupportedMediaType(msg)
if len(request.body) > MAX_ITEM_SIZE:
raise HTTPRequestEntityTooLarge()
try:
data = json.loads(request.body)
except ValueError:
raise HTTPJsonBadRequest(ERROR_MALFORMED_JSON)
item = self._parse_item(request, data)
# Check that we're putting it with the right id.
# Unfortunately we have to *return* the error response here
# rather than raise it, because raising HTTPForbidden will
# trigger pyramid's prompt-for-credentials handlers.
if request.matchdict["item"] != item.get_id():
return HTTPForbidden("Item ID does not match origin")
# Pass through to storage.
# Requires another round-trip through JSON; yuck.
request.body = json.dumps({"payload": json.dumps(item)})
return self.controller.set_item(request)
def process_request(self, request):
"""Consume an event batch request and return an appropriate response.
The API spec:
* the payload is a JSON list of objects, each object being an event
* batches are at most 40 KiB in size
* messages are signed with HMAC SHA-256
If the payload is valid, the events it contains will be put onto the
event queue. If there are issues with the request, error events will be
put into the error queue instead.
"""
request.environ["events.start_time"] = datetime.datetime.utcnow()
try:
signature_header = request.headers["X-Signature"]
except KeyError:
keyname = request.GET.get("key", "")
mac = request.GET.get("mac", "")
else:
keyname, mac = parse_signature(signature_header)
try:
key = self.keystore[keyname]
except KeyError:
keyname = "UNKNOWN"
key = "INVALID"
if request.content_length > MAXIMUM_BATCH_SIZE:
self._publish_error(request, keyname, "TOO_BIG")
return HTTPRequestEntityTooLarge()
body = request.body
if not request.headers.get("User-Agent"):
self._publish_error(request, keyname, "NO_USERAGENT")
return HTTPBadRequest("no user-agent provided")
# Handle Gzipped Requests
if request.headers.get('Content-Encoding') == 'gzip':
f = StringIO(body)
try:
body = gzip.GzipFile(fileobj=f).read()
except IOError:
return HTTPBadRequest("invalid gzip content")
expected_mac = hmac.new(key, body, hashlib.sha256).hexdigest()
_LOG.debug('Received request with key: %r, mac: %r, expected_mac: %r',
key, mac, expected_mac)
if not constant_time_compare(expected_mac, mac or ""):
self._publish_error(request, keyname, "INVALID_MAC")
return HTTPForbidden()
try:
batch = json.loads(body)
except ValueError:
self._publish_error(request, keyname, "INVALID_PAYLOAD")
return HTTPBadRequest("invalid json")
if not isinstance(batch, list):
self._publish_error(request, keyname, "INVALID_PAYLOAD")
return HTTPBadRequest("json root object must be a list")
reserialized_items = []
for item in batch:
reserialized = wrap_and_serialize_event(request, item)
if len(reserialized) > MAXIMUM_EVENT_SIZE:
self._publish_error(request, keyname, "EVENT_TOO_BIG")
return HTTPRequestEntityTooLarge()
reserialized_items.append(reserialized)
for item in reserialized_items:
self.event_queue.put(item)
self.metrics_client.counter("collected.http." + keyname).increment(
len(reserialized_items))
headers = {}
origin = request.headers.get("Origin")
if origin and is_allowed_origin(origin, self.allowed_origins):
headers.update(_CORS_HEADERS)
return Response(headers=headers)
def process_request(self, request):
"""Consume an event batch request and return an appropriate response.
The API spec:
* the payload is a JSON list of objects, each object being an event
* batches are at most 40 KiB in size
* messages are signed with HMAC SHA-256
If the payload is valid, the events it contains will be put onto the
event queue. If there are issues with the request, error events will be
put into the error queue instead.
"""
request.environ["events.start_time"] = datetime.datetime.utcnow()
if request.content_length > _MAXIMUM_CONTENT_LENGTH:
self.stats_client.count("client-error.too-big")
error = make_error_event(request, "TOO_BIG")
self.error_queue.put(error)
return HTTPRequestEntityTooLarge()
if not request.headers.get("User-Agent"):
self.stats_client.count("client-error.no-useragent")
error = make_error_event(request, "NO_USERAGENT")
self.error_queue.put(error)
return HTTPBadRequest("no user-agent provided")
try:
signature_header = request.headers["X-Signature"]
except KeyError:
keyname = request.GET.get("key", "")
mac = request.GET.get("mac", "")
else:
keyname, mac = parse_signature(signature_header)
key = self.keystore.get(keyname, "INVALID")
body = request.body
expected_mac = hmac.new(key, body, hashlib.sha256).hexdigest()
if not constant_time_compare(expected_mac, mac or ""):
self.stats_client.count("client-error.invalid-mac")
error = make_error_event(request, "INVALID_MAC")
self.error_queue.put(error)
return HTTPForbidden()
try:
batch = json.loads(body)
except ValueError:
self.stats_client.count("client-error.invalid-payload")
error = make_error_event(request, "INVALID_PAYLOAD")
self.error_queue.put(error)
return HTTPBadRequest("invalid json")
if not isinstance(batch, list):
self.stats_client.count("client-error.invalid-payload")
error = make_error_event(request, "INVALID_PAYLOAD")
self.error_queue.put(error)
return HTTPBadRequest("json root object must be a list")
reserialized_items = []
for item in batch:
reserialized = wrap_and_serialize_event(request, item)
if len(reserialized) > MAXIMUM_EVENT_SIZE:
self.stats_client.count("client-error.too-big")
error = make_error_event(request, "EVENT_TOO_BIG")
self.error_queue.put(error)
return HTTPRequestEntityTooLarge()
reserialized_items.append(reserialized)
for item in reserialized_items:
self.event_queue.put(item)
self.stats_client.count("collected.http", count=len(reserialized_items))
headers = {}
origin = request.headers.get("Origin")
if origin and is_allowed_origin(origin, self.allowed_origins):
headers.update(_CORS_HEADERS)
return Response(headers=headers)