def parse_multiple_bsos(request): """Validator to parse a list of BSOs from the request body. This validator accepts a list of BSOs in either application/json or application/newlines format, parses and validates them. Valid BSOs are placed under the key "bsos". Invalid BSOs are placed under the key "invalid_bsos". """ content_type = request.content_type try: if content_type in ("application/json", "text/plain", None): bso_datas = json_loads(request.body) elif content_type == "application/newlines": bso_datas = [json_loads(ln) for ln in request.body.split("\n")] else: msg = "Unsupported Media Type: %s" % (content_type, ) request.errors.add("header", "Content-Type", msg) request.errors.status = 415 return except ValueError: request.errors.add("body", "bsos", "Invalid JSON in request body") return if not isinstance(bso_datas, (tuple, list)): request.errors.add("body", "bsos", "Input data was not a list") return BATCH_MAX_COUNT = get_limit_config(request, "max_post_records") BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes") valid_bsos = {} invalid_bsos = {} total_bytes = 0 for count, bso_data in enumerate(bso_datas): try: bso = BSO(bso_data) except ValueError: msg = "Input data was not a list of BSOs" request.errors.add("body", "bsos", msg) return try: id = bso["id"] except KeyError: request.errors.add("body", "bsos", "Input BSO has no ID") return if id in valid_bsos: request.errors.add("body", "bsos", "Input BSO has duplicate ID") return consistent, msg = bso.validate() if not consistent: invalid_bsos[id] = msg # Log status on how many invalid BSOs we get, and why. logmsg = "Invalid BSO %s/%s/%s (%s): %s" userid = request.matchdict["userid"] collection = request.matchdict.get("collection") logger.info(logmsg, userid, collection, id, msg, bso) continue if count >= BATCH_MAX_COUNT: invalid_bsos[id] = "retry bso" continue total_bytes += len(bso.get("payload", "")) if total_bytes >= BATCH_MAX_BYTES: invalid_bsos[id] = "retry bytes" continue valid_bsos[id] = bso request.validated["bsos"] = valid_bsos.values() request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request): """Validator to parse a list of BSOs from the request body. This validator accepts a list of BSOs in either application/json or application/newlines format, parses and validates them. Valid BSOs are placed under the key "bsos". Invalid BSOs are placed under the key "invalid_bsos". """ content_type = request.content_type try: if content_type in ("application/json", "text/plain", None): bso_datas = json_loads(request.body) elif content_type == "application/newlines": bso_datas = [json_loads(ln) for ln in request.body.split("\n")] else: msg = "Unsupported Media Type: %s" % (content_type,) request.errors.add("header", "Content-Type", msg) request.errors.status = 415 return except ValueError: request.errors.add("body", "bsos", "Invalid JSON in request body") return if not isinstance(bso_datas, (tuple, list)): request.errors.add("body", "bsos", "Input data was not a list") return valid_bsos = {} invalid_bsos = {} total_bytes = 0 for count, bso_data in enumerate(bso_datas): try: bso = BSO(bso_data) except ValueError: msg = "Input data was not a list of BSOs" request.errors.add("body", "bsos", msg) return try: id = bso["id"] except KeyError: request.errors.add("body", "bsos", "Input BSO has no ID") return if id in valid_bsos: request.errors.add("body", "bsos", "Input BSO has duplicate ID") return consistent, msg = bso.validate() if not consistent: invalid_bsos[id] = msg continue if count >= BATCH_MAX_COUNT: invalid_bsos[id] = "retry bso" continue total_bytes += len(bso.get("payload", "")) if total_bytes >= BATCH_MAX_BYTES: invalid_bsos[id] = "retry bytes" continue valid_bsos[id] = bso request.validated["bsos"] = valid_bsos.values() request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request): """Validator to parse a list of BSOs from the request body. This validator accepts a list of BSOs in either application/json or application/newlines format, parses and validates them. Valid BSOs are placed under the key "bsos". Invalid BSOs are placed under the key "invalid_bsos". """ content_type = request.content_type try: if content_type in ("application/json", None): bso_datas = json.loads(request.body) elif content_type == "application/newlines": bso_datas = [json.loads(ln) for ln in request.body.split("\n")] else: msg = "Unsupported Media Type: %s" % (content_type, ) request.errors.add("header", "Content-Type", msg) request.errors.status = 415 return except ValueError: request.errors.add("body", "bsos", "Invalid JSON in request body") return if not isinstance(bso_datas, (tuple, list)): request.errors.add("body", "bsos", "Input data was not a list") return valid_bsos = {} invalid_bsos = {} total_bytes = 0 for count, bso_data in enumerate(bso_datas): try: bso = BSO(bso_data) except ValueError: msg = "Input data was not a list of BSOs" request.errors.add("body", "bsos", msg) return try: id = bso["id"] except KeyError: request.errors.add("body", "bsos", "Input BSO has no ID") return if id in valid_bsos: request.errors.add("body", "bsos", "Input BSO has duplicate ID") return consistent, msg = bso.validate() if not consistent: invalid_bsos[id] = msg continue if count >= BATCH_MAX_COUNT: invalid_bsos[id] = "retry bso" continue total_bytes += len(bso.get("payload", "")) if total_bytes >= BATCH_MAX_BYTES: invalid_bsos[id] = "retry bytes" continue valid_bsos[id] = bso request.validated["bsos"] = valid_bsos.values() request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request): """Validator to parse a list of BSOs from the request body. This validator accepts a list of BSOs in either application/json or application/newlines format, parses and validates them. Valid BSOs are placed under the key "bsos". Invalid BSOs are placed under the key "invalid_bsos". """ content_type = request.content_type try: if content_type in ("application/json", "text/plain", None): bso_datas = json_loads(request.body) elif content_type == "application/newlines": bso_datas = [] if request.body: for ln in request.body.split("\n"): bso_datas.append(json_loads(ln)) else: msg = "Unsupported Media Type: %s" % (content_type,) request.errors.add("header", "Content-Type", msg) request.errors.status = 415 return except ValueError: request.errors.add("body", "bsos", "Invalid JSON in request body") return if not isinstance(bso_datas, (tuple, list)): request.errors.add("body", "bsos", "Input data was not a list") return BATCH_MAX_COUNT = get_limit_config(request, "max_post_records") BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes") valid_bsos = {} invalid_bsos = {} total_bytes = 0 count = 0 for bso_data in bso_datas: try: bso = BSO(bso_data) except ValueError: msg = "Input data was not a list of BSOs" request.errors.add("body", "bsos", msg) return try: id = bso["id"] except KeyError: request.errors.add("body", "bsos", "Input BSO has no ID") return if id in valid_bsos: request.errors.add("body", "bsos", "Input BSO has duplicate ID") return consistent, msg = bso.validate() if not consistent: invalid_bsos[id] = msg # Log status on how many invalid BSOs we get, and why. logmsg = "Invalid BSO %s/%s/%s (%s): %s" userid = request.matchdict["userid"] collection = request.matchdict.get("collection") logger.info(logmsg, userid, collection, id, msg, bso) continue count += 1 if count > BATCH_MAX_COUNT: invalid_bsos[id] = "retry bso" continue total_bytes += len(bso.get("payload", "")) if total_bytes > BATCH_MAX_BYTES: invalid_bsos[id] = "retry bytes" continue valid_bsos[id] = bso request.validated["bsos"] = valid_bsos.values() request.validated["invalid_bsos"] = invalid_bsos