def parse_single_bso(request):
"""Validator to parse a single BSO from the request body.
This validator accepts a single BSO in application/json format, parses
and validates it, and places it under the key "bso".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_data = json_loads(request.body)
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bso", "Invalid JSON in request body")
return
try:
bso = BSO(bso_data)
except ValueError:
request.errors.add("body", "bso", "Invalid BSO data")
return
consistent, msg = bso.validate()
if not consistent:
request.errors.add("body", "bso", "Invalid BSO: " + msg)
return
request.validated["bso"] = bso
def parse_single_bso(request):
"""Validator to parse a single BSO from the request body.
This validator accepts a single BSO in application/json format, parses
and validates it, and places it under the key "bso".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_data = json_loads(request.body)
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bso", "Invalid JSON in request body")
return
try:
bso = BSO(bso_data)
except ValueError:
request.errors.add("body", "bso", "Invalid BSO data")
return
consistent, msg = bso.validate()
if not consistent:
request.errors.add("body", "bso", "Invalid BSO: " + msg)
return
request.validated["bso"] = bso
def _row_to_bso(self, row, timestamp):
"""Convert a database table row into a BSO object."""
item = dict(row)
for key in ("userid", "collection"):
item.pop(key, None)
ts = item.get("modified")
if ts is not None:
item["modified"] = dt2ts(ts)
# Convert the ttl back into an offset from the current time.
ttl = item.get("ttl")
if ttl is not None:
item["ttl"] = int(dt2ts(ttl) - timestamp)
return BSO(item)
def _row_to_bso(self, row):
"""Convert a database table row into a BSO object."""
item = dict(row)
for key in (
"userid",
"collection",
"payload_size",
"ttl",
):
item.pop(key, None)
ts = item.get("modified")
if ts is not None:
item["modified"] = bigint2ts(ts)
return BSO(item)
def apply_batch(self, session, user, collection, batchid):
userid = user_key(user)
collectionid = self._get_collection_id(collection)
batchid = ts2dt(batchid / 1000.0)
q = "SELECT bsos FROM batches WHERE userid=@userid AND " \
"collection=@collectionid AND id=@id"
result_set = session.transaction.execute_sql(q,
params={
"userid": userid,
"collectionid":
collectionid,
"id": batchid
},
param_types={
"userid":
param_types.STRING,
"collectionid":
param_types.INT64,
"id":
param_types.TIMESTAMP
})
result = result_set.one_or_none()
if not result:
self._touch_collection(session, userid, collectionid)
return session.timestamp
bsos = [BSO(json.loads(x)) for x in result[0].split("\n") if x]
session.transaction.execute_sql(
"DELETE FROM batches "
"WHERE userid=@userid AND collection=@collectionid "
"AND id=@id",
params={
"userid": userid,
"collectionid": collectionid,
"id": batchid
},
param_types={
"userid": param_types.STRING,
"collectionid": param_types.INT64,
"id": param_types.TIMESTAMP
})
return self.set_items(user, collection, bsos)
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
# Log status on how many invalid BSOs we get, and why.
logmsg = "Invalid BSO %s/%s/%s (%s): %s"
userid = request.matchdict["userid"]
collection = request.matchdict.get("collection")
logger.info(logmsg, userid, collection, id, msg, bso)
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", None):
bso_datas = json.loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json.loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type, )
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = [json_loads(ln) for ln in request.body.split("\n")]
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
for count, bso_data in enumerate(bso_datas):
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
continue
if count >= BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes >= BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def test_validation(self):
bso = BSO()
result, failure = bso.validate()
self.assertTrue(result)
data = {'id': 'bigid' * 30}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'id': u'I AM A \N{SNOWMAN}'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': 9999999999}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '9999'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertTrue(bso['sortindex'], 9999)
data = {'sortindex': 'ok'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '12'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertEquals(bso['sortindex'], 12)
for bad_ttl in ('bouh', -1, 31537000):
data = {'ttl': bad_ttl}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'ttl': 3600}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 30000}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 300000}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {"boooo": ""}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {42: 17}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
def test_validation(self):
bso = BSO()
result, failure = bso.validate()
self.assertTrue(result)
data = {'id': 'bigid' * 30}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'id': u'I AM A \N{SNOWMAN}'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': 9999999999}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '9999'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertTrue(bso['sortindex'], 9999)
data = {'sortindex': 'ok'}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {'sortindex': '12'}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
self.assertEquals(bso['sortindex'], 12)
for bad_ttl in ('bouh', -1, 31537000):
data = {'ttl': bad_ttl}
bso = BSO(data)
result, failure = bso.validate()
# XXX TODO: ttls that are too large are currently ignored.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=977397
# self.assertFalse(result)
if bad_ttl != 31537000:
self.assertFalse(result)
else:
self.assertTrue(result)
self.assertTrue('ttl' not in bso)
data = {'ttl': 3600}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 30000}
bso = BSO(data)
result, failure = bso.validate()
self.assertTrue(result)
data = {'payload': "X" * 300000}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {"boooo": ""}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
data = {42: 17}
bso = BSO(data)
result, failure = bso.validate()
self.assertFalse(result)
def parse_multiple_bsos(request):
"""Validator to parse a list of BSOs from the request body.
This validator accepts a list of BSOs in either application/json or
application/newlines format, parses and validates them.
Valid BSOs are placed under the key "bsos". Invalid BSOs are placed
under the key "invalid_bsos".
"""
content_type = request.content_type
try:
if content_type in ("application/json", "text/plain", None):
bso_datas = json_loads(request.body)
elif content_type == "application/newlines":
bso_datas = []
if request.body:
for ln in request.body.split("\n"):
bso_datas.append(json_loads(ln))
else:
msg = "Unsupported Media Type: %s" % (content_type,)
request.errors.add("header", "Content-Type", msg)
request.errors.status = 415
return
except ValueError:
request.errors.add("body", "bsos", "Invalid JSON in request body")
return
if not isinstance(bso_datas, (tuple, list)):
request.errors.add("body", "bsos", "Input data was not a list")
return
BATCH_MAX_COUNT = get_limit_config(request, "max_post_records")
BATCH_MAX_BYTES = get_limit_config(request, "max_post_bytes")
valid_bsos = {}
invalid_bsos = {}
total_bytes = 0
count = 0
for bso_data in bso_datas:
try:
bso = BSO(bso_data)
except ValueError:
msg = "Input data was not a list of BSOs"
request.errors.add("body", "bsos", msg)
return
try:
id = bso["id"]
except KeyError:
request.errors.add("body", "bsos", "Input BSO has no ID")
return
if id in valid_bsos:
request.errors.add("body", "bsos", "Input BSO has duplicate ID")
return
consistent, msg = bso.validate()
if not consistent:
invalid_bsos[id] = msg
# Log status on how many invalid BSOs we get, and why.
logmsg = "Invalid BSO %s/%s/%s (%s): %s"
userid = request.matchdict["userid"]
collection = request.matchdict.get("collection")
logger.info(logmsg, userid, collection, id, msg, bso)
continue
count += 1
if count > BATCH_MAX_COUNT:
invalid_bsos[id] = "retry bso"
continue
total_bytes += len(bso.get("payload", ""))
if total_bytes > BATCH_MAX_BYTES:
invalid_bsos[id] = "retry bytes"
continue
valid_bsos[id] = bso
request.validated["bsos"] = valid_bsos.values()
request.validated["invalid_bsos"] = invalid_bsos
def _row_to_bso(self, row):
"""Convert a database table row into a BSO object."""
item = dict(row)
for key in ("userid", "collection", "payload_size", "ttl",):
item.pop(key, None)
return BSO(item)
