def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
# Log status on how many invalid BSOs we get, and why.
logmsg = "Invalid BSO %s/%s/%s (%s): %s"
userid = request.matchdict["userid"]
collection = request.matchdict.get("collection")
logger.info(logmsg, userid, collection, id, msg, bso)
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", None):
bso_datas = json.loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json.loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = []
if request.body:
for ln in request.body.split("\n"):
bso_datas.append(json_loads(ln))
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
count = 0
for bso_data in bso_datas:
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
# Log status on how many invalid BSOs we get, and why.
logmsg = "Invalid BSO %s/%s/%s (%s): %s"
userid = request.matchdict["userid"]
collection = request.matchdict.get("collection")
logger.info(logmsg, userid, collection, id, msg, bso)
continue
count += 1
if count > BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes > BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos