Example #1
0
def parse_single_bso(request):
    """Validator to parse a single BSO from the request body.

    This validator accepts a single BSO in application/json format, parses
    and validates it, and places it under the key "bso".
    """
    content_type = request.content_type
    try:
        if content_type in ("application/json", "text/plain", None):
            bso_data = json_loads(request.body)
        else:
            msg = "Unsupported Media Type: %s" % (content_type, )
            request.errors.add("header", "Content-Type", msg)
            request.errors.status = 415
            return
    except ValueError:
        request.errors.add("body", "bso", "Invalid JSON in request body")
        return

    try:
        bso = BSO(bso_data)
    except ValueError:
        request.errors.add("body", "bso", "Invalid BSO data")
        return

    consistent, msg = bso.validate()
    if not consistent:
        request.errors.add("body", "bso", "Invalid BSO: " + msg)
        return

    request.validated["bso"] = bso
Example #2
0
def parse_single_bso(request):
    """Validator to parse a single BSO from the request body.

    This validator accepts a single BSO in application/json format, parses
    and validates it, and places it under the key "bso".
    """
    content_type = request.content_type
    try:
        if content_type in ("application/json", "text/plain", None):
            bso_data = json_loads(request.body)
        else:
            msg = "Unsupported Media Type: %s" % (content_type,)
            request.errors.add("header", "Content-Type", msg)
            request.errors.status = 415
            return
    except ValueError:
        request.errors.add("body", "bso", "Invalid JSON in request body")
        return

    try:
        bso = BSO(bso_data)
    except ValueError:
        request.errors.add("body", "bso", "Invalid BSO data")
        return

    consistent, msg = bso.validate()
    if not consistent:
        request.errors.add("body", "bso", "Invalid BSO: " + msg)
        return

    request.validated["bso"] = bso
Example #3
0
def parse_multiple_bsos(request):
    """Validator to parse a list of BSOs from the request body.

    This validator accepts a list of BSOs in either application/json or
    application/newlines format, parses and validates them.

    Valid BSOs are placed under the key "bsos".  Invalid BSOs are placed
    under the key "invalid_bsos".
    """
    content_type = request.content_type
    try:
        if content_type in ("application/json", "text/plain", None):
            bso_datas = json_loads(request.body)
        elif content_type == "application/newlines":
            bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
        else:
            msg = "Unsupported Media Type: %s" % (content_type, )
            request.errors.add("header", "Content-Type", msg)
            request.errors.status = 415
            return
    except ValueError:
        request.errors.add("body", "bsos", "Invalid JSON in request body")
        return

    if not isinstance(bso_datas, (tuple, list)):
        request.errors.add("body", "bsos", "Input data was not a list")
        return

    BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
    BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")

    valid_bsos = {}
    invalid_bsos = {}

    total_bytes = 0
    for count, bso_data in enumerate(bso_datas):
        try:
            bso = BSO(bso_data)
        except ValueError:
            msg = "Input data was not a list of BSOs"
            request.errors.add("body", "bsos", msg)
            return

        try:
            id = bso["id"]
        except KeyError:
            request.errors.add("body", "bsos", "Input BSO has no ID")
            return

        if id in valid_bsos:
            request.errors.add("body", "bsos", "Input BSO has duplicate ID")
            return

        consistent, msg = bso.validate()
        if not consistent:
            invalid_bsos[id] = msg
            # Log status on how many invalid BSOs we get, and why.
            logmsg = "Invalid BSO %s/%s/%s (%s): %s"
            userid = request.matchdict["userid"]
            collection = request.matchdict.get("collection")
            logger.info(logmsg, userid, collection, id, msg, bso)
            continue

        if count >= BATCH_MAX_COUNT:
            invalid_bsos[id] = "retry bso"
            continue

        total_bytes += len(bso.get("payload", ""))
        if total_bytes >= BATCH_MAX_BYTES:
            invalid_bsos[id] = "retry bytes"
            continue

        valid_bsos[id] = bso

    request.validated["bsos"] = valid_bsos.values()
    request.validated["invalid_bsos"] = invalid_bsos
Example #4
0
def parse_multiple_bsos(request):
    """Validator to parse a list of BSOs from the request body.

    This validator accepts a list of BSOs in either application/json or
    application/newlines format, parses and validates them.

    Valid BSOs are placed under the key "bsos".  Invalid BSOs are placed
    under the key "invalid_bsos".
    """
    content_type = request.content_type
    try:
        if content_type in ("application/json", None):
            bso_datas = json.loads(request.body)
        elif content_type == "application/newlines":
            bso_datas = [json.loads(ln) for ln in request.body.split("\n")]
        else:
            msg = "Unsupported Media Type: %s" % (content_type, )
            request.errors.add("header", "Content-Type", msg)
            request.errors.status = 415
            return
    except ValueError:
        request.errors.add("body", "bsos", "Invalid JSON in request body")
        return

    if not isinstance(bso_datas, (tuple, list)):
        request.errors.add("body", "bsos", "Input data was not a list")
        return

    valid_bsos = {}
    invalid_bsos = {}

    total_bytes = 0
    for count, bso_data in enumerate(bso_datas):
        try:
            bso = BSO(bso_data)
        except ValueError:
            msg = "Input data was not a list of BSOs"
            request.errors.add("body", "bsos", msg)
            return

        try:
            id = bso["id"]
        except KeyError:
            request.errors.add("body", "bsos", "Input BSO has no ID")
            return

        if id in valid_bsos:
            request.errors.add("body", "bsos", "Input BSO has duplicate ID")
            return

        consistent, msg = bso.validate()
        if not consistent:
            invalid_bsos[id] = msg
            continue

        if count >= BATCH_MAX_COUNT:
            invalid_bsos[id] = "retry bso"
            continue

        total_bytes += len(bso.get("payload", ""))
        if total_bytes >= BATCH_MAX_BYTES:
            invalid_bsos[id] = "retry bytes"
            continue

        valid_bsos[id] = bso

    request.validated["bsos"] = valid_bsos.values()
    request.validated["invalid_bsos"] = invalid_bsos
Example #5
0
def parse_multiple_bsos(request):
    """Validator to parse a list of BSOs from the request body.

    This validator accepts a list of BSOs in either application/json or
    application/newlines format, parses and validates them.

    Valid BSOs are placed under the key "bsos".  Invalid BSOs are placed
    under the key "invalid_bsos".
    """
    content_type = request.content_type
    try:
        if content_type in ("application/json", "text/plain", None):
            bso_datas = json_loads(request.body)
        elif content_type == "application/newlines":
            bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
        else:
            msg = "Unsupported Media Type: %s" % (content_type,)
            request.errors.add("header", "Content-Type", msg)
            request.errors.status = 415
            return
    except ValueError:
        request.errors.add("body", "bsos", "Invalid JSON in request body")
        return

    if not isinstance(bso_datas, (tuple, list)):
        request.errors.add("body", "bsos", "Input data was not a list")
        return

    valid_bsos = {}
    invalid_bsos = {}

    total_bytes = 0
    for count, bso_data in enumerate(bso_datas):
        try:
            bso = BSO(bso_data)
        except ValueError:
            msg = "Input data was not a list of BSOs"
            request.errors.add("body", "bsos", msg)
            return

        try:
            id = bso["id"]
        except KeyError:
            request.errors.add("body", "bsos", "Input BSO has no ID")
            return

        if id in valid_bsos:
            request.errors.add("body", "bsos", "Input BSO has duplicate ID")
            return

        consistent, msg = bso.validate()
        if not consistent:
            invalid_bsos[id] = msg
            continue

        if count >= BATCH_MAX_COUNT:
            invalid_bsos[id] = "retry bso"
            continue

        total_bytes += len(bso.get("payload", ""))
        if total_bytes >= BATCH_MAX_BYTES:
            invalid_bsos[id] = "retry bytes"
            continue

        valid_bsos[id] = bso

    request.validated["bsos"] = valid_bsos.values()
    request.validated["invalid_bsos"] = invalid_bsos
Example #6
0
    def test_validation(self):
        bso = BSO()
        result, failure = bso.validate()
        self.assertTrue(result)

        data = {'id': 'bigid' * 30}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'id': u'I AM A \N{SNOWMAN}'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'sortindex': 9999999999}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'sortindex': '9999'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)
        self.assertTrue(bso['sortindex'], 9999)

        data = {'sortindex': 'ok'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'sortindex': '12'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)
        self.assertEquals(bso['sortindex'], 12)

        for bad_ttl in ('bouh', -1, 31537000):
            data = {'ttl': bad_ttl}
            bso = BSO(data)
            result, failure = bso.validate()
            self.assertFalse(result)

        data = {'ttl': 3600}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)

        data = {'payload': "X" * 30000}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)

        data = {'payload': "X" * 300000}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {"boooo": ""}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {42: 17}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)
    def test_validation(self):
        bso = BSO()
        result, failure = bso.validate()
        self.assertTrue(result)

        data = {'id': 'bigid' * 30}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'id': u'I AM A \N{SNOWMAN}'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'sortindex': 9999999999}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'sortindex': '9999'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)
        self.assertTrue(bso['sortindex'], 9999)

        data = {'sortindex': 'ok'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {'sortindex': '12'}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)
        self.assertEquals(bso['sortindex'], 12)

        for bad_ttl in ('bouh', -1, 31537000):
            data = {'ttl': bad_ttl}
            bso = BSO(data)
            result, failure = bso.validate()
            # XXX TODO: ttls that are too large are currently ignored.
            # See https://bugzilla.mozilla.org/show_bug.cgi?id=977397
            # self.assertFalse(result)
            if bad_ttl != 31537000:
                self.assertFalse(result)
            else:
                self.assertTrue(result)
                self.assertTrue('ttl' not in bso)

        data = {'ttl': 3600}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)

        data = {'payload': "X" * 30000}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertTrue(result)

        data = {'payload': "X" * 300000}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {"boooo": ""}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)

        data = {42: 17}
        bso = BSO(data)
        result, failure = bso.validate()
        self.assertFalse(result)
def parse_multiple_bsos(request):
    """Validator to parse a list of BSOs from the request body.

    This validator accepts a list of BSOs in either application/json or
    application/newlines format, parses and validates them.

    Valid BSOs are placed under the key "bsos".  Invalid BSOs are placed
    under the key "invalid_bsos".
    """
    content_type = request.content_type
    try:
        if content_type in ("application/json", "text/plain", None):
            bso_datas = json_loads(request.body)
        elif content_type == "application/newlines":
            bso_datas = []
            if request.body:
                for ln in request.body.split("\n"):
                    bso_datas.append(json_loads(ln))
        else:
            msg = "Unsupported Media Type: %s" % (content_type,)
            request.errors.add("header", "Content-Type", msg)
            request.errors.status = 415
            return
    except ValueError:
        request.errors.add("body", "bsos", "Invalid JSON in request body")
        return

    if not isinstance(bso_datas, (tuple, list)):
        request.errors.add("body", "bsos", "Input data was not a list")
        return

    BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
    BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")

    valid_bsos = {}
    invalid_bsos = {}

    total_bytes = 0
    count = 0
    for bso_data in bso_datas:
        try:
            bso = BSO(bso_data)
        except ValueError:
            msg = "Input data was not a list of BSOs"
            request.errors.add("body", "bsos", msg)
            return

        try:
            id = bso["id"]
        except KeyError:
            request.errors.add("body", "bsos", "Input BSO has no ID")
            return

        if id in valid_bsos:
            request.errors.add("body", "bsos", "Input BSO has duplicate ID")
            return

        consistent, msg = bso.validate()
        if not consistent:
            invalid_bsos[id] = msg
            # Log status on how many invalid BSOs we get, and why.
            logmsg = "Invalid BSO %s/%s/%s (%s): %s"
            userid = request.matchdict["userid"]
            collection = request.matchdict.get("collection")
            logger.info(logmsg, userid, collection, id, msg, bso)
            continue

        count += 1
        if count > BATCH_MAX_COUNT:
            invalid_bsos[id] = "retry bso"
            continue

        total_bytes += len(bso.get("payload", ""))
        if total_bytes > BATCH_MAX_BYTES:
            invalid_bsos[id] = "retry bytes"
            continue

        valid_bsos[id] = bso

    request.validated["bsos"] = valid_bsos.values()
    request.validated["invalid_bsos"] = invalid_bsos