def parse_single_bso(request):
"""Validator to parse a single BSO from the request body.
This validator accepts a single BSO in application/json format, parses
and validates it, and places it under the key "bso".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_data = json_loads(request.body)
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bso", "Invalid JSON in request body")
return
try:
bso = BSO(bso_data)
except ValueError:
request.errors.add("body", "bso", "Invalid BSO data")
return
consistent, msg = bso.validate()
if not consistent:
request.errors.add("body", "bso", "Invalid BSO: " + msg)
return
request.validated["bso"] = bso
def parse_single_bso(request):
"""Validator to parse a single BSO from the request body.
This validator accepts a single BSO in application/json format, parses
and validates it, and places it under the key "bso".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_data = json_loads(request.body)
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bso", "Invalid JSON in request body")
return
try:
bso = BSO(bso_data)
except ValueError:
request.errors.add("body", "bso", "Invalid BSO data")
return
consistent, msg = bso.validate()
if not consistent:
request.errors.add("body", "bso", "Invalid BSO: " + msg)
return
request.validated["bso"] = bso
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
# Log status on how many invalid BSOs we get, and why.
logmsg = "Invalid BSO %s/%s/%s (%s): %s"
userid = request.matchdict["userid"]
collection = request.matchdict.get("collection")
logger.info(logmsg, userid, collection, id, msg, bso)
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", None):
bso_datas = json.loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json.loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def test_validation(self):
bso = BSO()
result, failure = bso.validate()
self.assertTrue(result)
data = {'id': 'bigid' * 30}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'id': u'I AM A \N{SNOWMAN}'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': 9999999999}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '9999'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertTrue(bso['sortindex'], 9999)
data = {'sortindex': 'ok'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '12'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertEquals(bso['sortindex'], 12)
for bad_ttl in ('bouh', -1, 31537000):
data = {'ttl': bad_ttl}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'ttl': 3600}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 30000}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 300000}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {"boooo": ""}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {42: 17}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
def test_validation(self):
bso = BSO()
result, failure = bso.validate()
self.assertTrue(result)
data = {'id': 'bigid' * 30}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'id': u'I AM A \N{SNOWMAN}'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': 9999999999}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '9999'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertTrue(bso['sortindex'], 9999)
data = {'sortindex': 'ok'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '12'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertEquals(bso['sortindex'], 12)
for bad_ttl in ('bouh', -1, 31537000):
data = {'ttl': bad_ttl}
bso = BSO(data)
result, failure = bso.validate()
# XXX TODO: ttls that are too large are currently ignored.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=977397
# self.assertFalse(result)
if bad_ttl != 31537000:
self.assertFalse(result)
else:
self.assertTrue(result)
self.assertTrue('ttl' not in bso)
data = {'ttl': 3600}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 30000}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 300000}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {"boooo": ""}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {42: 17}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = []
if request.body:
for ln in request.body.split("\n"):
bso_datas.append(json_loads(ln))
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
count = 0
for bso_data in bso_datas:
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
# Log status on how many invalid BSOs we get, and why.
logmsg = "Invalid BSO %s/%s/%s (%s): %s"
userid = request.matchdict["userid"]
collection = request.matchdict.get("collection")
logger.info(logmsg, userid, collection, id, msg, bso)
continue
count += 1
if count > BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes > BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos