def set_item(self, request):
"""Sets a single WBO object."""
storage = self._get_storage(request)
if storage.use_quota:
left = self._check_quota(request)
else:
left = 0.
user_id = request.user['userid']
collection_name = request.sync_info['collection']
item_id = request.sync_info['item']
if self._was_modified(request, user_id, collection_name):
raise HTTPPreconditionFailed(collection_name)
try:
data = json.loads(request.body)
except ValueError:
raise HTTPJsonBadRequest(WEAVE_MALFORMED_JSON)
try:
wbo = WBO(data)
except ValueError:
raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)
consistent, msg = wbo.validate()
if not consistent:
raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)
if self._has_modifiers(wbo):
wbo['modified'] = request.server_time
try:
res = storage.set_item(user_id, collection_name, item_id,
storage_time=request.server_time, **wbo)
except StorageConflictError:
raise HTTPJsonBadRequest(WEAVE_INVALID_WRITE)
response = json_response(res)
if storage.use_quota and left <= _ONE_MEG:
response.headers['X-Weave-Quota-Remaining'] = str(left)
return response
def get_item(self, user_id, collection_name, item_id, fields=None):
"""returns one item"""
wbo = self._get_wbo_table(user_id)
collection_id = self._get_collection_id(user_id, collection_name)
if fields is None:
fields = [wbo]
else:
fields = [getattr(wbo.c, field) for field in fields]
where = self._get_query('ITEM_ID_COL_USER', user_id)
query = select(fields, where)
res = self._do_query_fetchone(query,
user_id=user_id,
item_id=item_id,
collection_id=collection_id,
ttl=_int_now())
if res is None:
return None
return WBO(res, {'modified': bigint2time})
def get_items(self,
user_id,
collection_name,
fields=None,
filters=None,
limit=None,
offset=None,
sort=None):
"""returns items from a collection
"filter" is a dict used to add conditions to the db query.
Its keys are the field names on which the condition operates.
Its values are the values the field should have.
It can be a single value, or a list. For the latter the in()
operator is used. For single values, the operator has to be provided.
"""
wbo = self._get_wbo_table(user_id)
collection_id = self._get_collection_id(user_id, collection_name)
if fields is None:
fields = [wbo]
else:
fields = [getattr(wbo.c, field) for field in fields]
# preparing the where statement
where = [wbo.c.username == user_id, wbo.c.collection == collection_id]
if filters is not None:
for field, value in filters.items():
field = getattr(wbo.c, field)
operator, value = value
if field.name == 'modified':
value = _roundedbigint(value)
if isinstance(value, (list, tuple)):
where.append(field.in_(value))
else:
if operator == '=':
where.append(field == value)
elif operator == '<':
where.append(field < value)
elif operator == '>':
where.append(field > value)
if filters is None or 'ttl' not in filters:
where.append(wbo.c.ttl > _int_now())
where = and_(*where)
query = select(fields, where)
if sort is not None:
if sort == 'oldest':
query = query.order_by(wbo.c.modified.asc())
elif sort == 'newest':
query = query.order_by(wbo.c.modified.desc())
else:
query = query.order_by(wbo.c.sortindex.desc())
if limit is not None and int(limit) > 0:
query = query.limit(int(limit))
if offset is not None and int(offset) > 0:
query = query.offset(int(offset))
res = self._do_query_fetchall(query)
converters = {'modified': bigint2time}
return [WBO(line, converters) for line in res]
def test_validation(self):
data = {'parentid': 'bigid' * 30}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 'id', 'sortindex': 9999999999}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 'id', 'sortindex': '9999.1'}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
self.assertTrue(wbo['sortindex'], 9999)
data = {'parentid': 'id', 'sortindex': 'ok'}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 33, 'sortindex': '12'}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
self.assertEquals(wbo['parentid'], '33')
self.assertEquals(wbo['sortindex'], 12)
for bad_ttl in ('bouh', -1, 31537000):
data = {'parentid': 33, 'ttl': bad_ttl}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 33, 'ttl': 3600}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
data = {'payload': "X" * 30000}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
data = {'payload': "X" * 300000}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
def test_validation(self):
data = {'parentid': 'bigid' * 30}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 'id', 'sortindex': 9999999999}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 'id', 'sortindex': '9999.1'}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
self.assertTrue(wbo['sortindex'], 9999)
data = {'parentid': 'id', 'sortindex': 'ok'}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 33, 'sortindex': '12'}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
self.assertEquals(wbo['parentid'], '33')
self.assertEquals(wbo['sortindex'], 12)
for bad_ttl in ('bouh', -1, 31537000):
data = {'parentid': 33, 'ttl': bad_ttl}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
data = {'parentid': 33, 'ttl': 3600}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
data = {'payload': "X" * 30000}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertTrue(result)
data = {'payload': "X" * 300000}
wbo = WBO(data)
result, failure = wbo.validate()
self.assertFalse(result)
def test_basic(self):
wbo = WBO()
wbo = WBO({'boooo': ''})
self.assertTrue('boooo' not in wbo)
def set_collection(self, request):
"""Sets a batch of WBO objects into a collection."""
user_id = request.user['userid']
collection_name = request.sync_info['collection']
if self._was_modified(request, user_id, collection_name):
raise HTTPPreconditionFailed(collection_name)
try:
wbos = json.loads(request.body)
except ValueError:
raise HTTPJsonBadRequest(WEAVE_MALFORMED_JSON)
if not isinstance(wbos, (tuple, list)):
# thats a batch of one
try:
id_ = str(wbos['id'])
except (KeyError, TypeError):
raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)
if '/' in id_:
raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)
request.sync_info['item'] = id_
return self.set_item(request)
res = {'success': [], 'failed': {}}
# Sanity-check each of the WBOs.
# Limit the batch based on both count and payload size.
kept_wbos = []
total_bytes = 0
for count, wbo in enumerate(wbos):
try:
wbo = WBO(wbo)
except ValueError:
res['failed'][''] = ['invalid wbo']
continue
if 'id' not in wbo:
res['failed'][''] = ['invalid id']
continue
consistent, msg = wbo.validate()
item_id = wbo['id']
if not consistent:
res['failed'][item_id] = [msg]
continue
if count >= self.batch_max_count:
res['failed'][item_id] = ['retry wbo']
continue
if 'payload' in wbo:
total_bytes += len(wbo['payload'])
if total_bytes >= self.batch_max_bytes:
res['failed'][item_id] = ['retry bytes']
continue
if self._has_modifiers(wbo):
wbo['modified'] = request.server_time
kept_wbos.append(wbo)
storage = self._get_storage(request)
if storage.use_quota:
left = self._check_quota(request)
else:
left = 0.
storage_time = request.server_time
for wbos in batch(kept_wbos, size=self.batch_size):
wbos = list(wbos) # to avoid exhaustion
try:
storage.set_items(user_id, collection_name,
wbos, storage_time=storage_time)
except Exception, e: # we want to swallow the 503 in that case
# something went wrong
self.logger.error('Could not set items')
self.logger.error(str(e))
for wbo in wbos:
res['failed'][wbo['id']] = "db error"
else:
res['success'].extend([wbo['id'] for wbo in wbos])
