def test_multiple_profiles(self):
"""Exercise multiple profiles and collections"""
expected_count_all = 0
profiles_count = 5
collection_names = ( 'testing', 'keys', 'tabs', 'history', 'bookmarks' )
collection_counts = {}
# Produce a set of Profiles in the datastore
profiles = []
for i in range(profiles_count):
profile = Profile(user_name='t-%s'%i, user_id='id-%s'%i, password='******'%i)
profile.put()
profiles.append(profile)
# Generate collections for each profile.
for p in profiles:
auth_header = self.build_auth_header(p.user_name, p.password)
collection_counts[p.user_name] = {}
# Run through several collections and make WBOs
for cn in collection_names:
curr_count = random.randint(1,10)
collection_counts[p.user_name][cn] = curr_count
expected_count_all += curr_count
# Generate a bunch of random-content WBOs
base_url = '/sync/1.0/%s/storage/%s' % (p.user_name, cn)
for i in range(curr_count):
resp = self.put_random_wbo(base_url, auth_header)
# Ensure the total number of WBOs is correct.
result_count_all = WBO.all().count()
self.assertEqual(expected_count_all, result_count_all)
# Ensure the counts for each profile collection matches inserts.
for profile in profiles:
counts = Collection.get_counts(profile)
for name in collection_names:
c = Collection.get_by_profile_and_name(profile, name)
self.assertEqual(
collection_counts[profile.user_name][name],
WBO.get_by_collection(c).count()
)
# Delete each of the collections for each user.
for profile in profiles:
auth_header = self.build_auth_header(
profile.user_name, profile.password
)
for name in collection_names:
url = '/sync/1.0/%s/storage/%s' % (profile.user_name, name)
resp = self.app.delete(url, headers=auth_header)
# Ensure the individual collection is now empty.
c = Collection.get_by_profile_and_name(profile, name)
self.assertEqual(0, WBO.get_by_collection(c).count())
# Ensure there are no more WBOs
result_count_all = WBO.all().count()
self.assertEqual(0, result_count_all)
def test_multiple_profiles(self):
"""Exercise multiple profiles and collections"""
expected_count_all = 0
profiles_count = 5
collection_names = ('testing', 'keys', 'tabs', 'history', 'bookmarks')
collection_counts = {}
# Produce a set of Profiles in the datastore
profiles = []
for i in range(profiles_count):
profile = Profile(user_name='t-%s' % i,
user_id='id-%s' % i,
password='******' % i)
profile.put()
profiles.append(profile)
# Generate collections for each profile.
for p in profiles:
auth_header = self.build_auth_header(p.user_name, p.password)
collection_counts[p.user_name] = {}
# Run through several collections and make WBOs
for cn in collection_names:
curr_count = random.randint(1, 10)
collection_counts[p.user_name][cn] = curr_count
expected_count_all += curr_count
# Generate a bunch of random-content WBOs
base_url = '/sync/1.0/%s/storage/%s' % (p.user_name, cn)
for i in range(curr_count):
resp = self.put_random_wbo(base_url, auth_header)
# Ensure the total number of WBOs is correct.
result_count_all = WBO.all().count()
self.assertEqual(expected_count_all, result_count_all)
# Ensure the counts for each profile collection matches inserts.
for profile in profiles:
counts = Collection.get_counts(profile)
for name in collection_names:
c = Collection.get_by_profile_and_name(profile, name)
self.assertEqual(collection_counts[profile.user_name][name],
WBO.get_by_collection(c).count())
# Delete each of the collections for each user.
for profile in profiles:
auth_header = self.build_auth_header(profile.user_name,
profile.password)
for name in collection_names:
url = '/sync/1.0/%s/storage/%s' % (profile.user_name, name)
resp = self.app.delete(url, headers=auth_header)
# Ensure the individual collection is now empty.
c = Collection.get_by_profile_and_name(profile, name)
self.assertEqual(0, WBO.get_by_collection(c).count())
# Ensure there are no more WBOs
result_count_all = WBO.all().count()
self.assertEqual(0, result_count_all)
def test_cascading_profile_delete(self):
"""Ensure that profile deletion cascades down to collections and WBOs"""
(p, c, ah) = (self.profile, self.collection, self.auth_header)
wbos = self.build_wbo_set()
self.assert_(WBO.all().count() > 0)
self.assert_(Collection.all().count() > 0)
self.assert_(Profile.all().count() > 0)
p.delete()
self.assertEquals(0, WBO.all().count())
self.assertEquals(0, Collection.all().count())
self.assertEquals(0, Profile.all().count())
def test_cascading_profile_delete(self):
"""Ensure that profile deletion cascades down to collections and WBOs"""
(p, c, ah) = (self.profile, self.collection, self.auth_header)
wbos = self.build_wbo_set()
self.assert_(WBO.all().count() > 0)
self.assert_(Collection.all().count() > 0)
self.assert_(Profile.all().count() > 0)
p.delete()
self.assertEquals(0, WBO.all().count())
self.assertEquals(0, Collection.all().count())
self.assertEquals(0, Profile.all().count())
def get(self, user_name, collection_name, wbo_id):
"""Get an item from the collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name)
wbo = WBO.get_by_collection_and_wbo_id(collection, wbo_id)
if not wbo: return self.error(404)
return wbo.to_dict()
def post(self, user_name, collection_name):
"""Bulk update of WBOs in a collection"""
out = { 'modified': None, 'success': [], 'failed': {} }
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name
)
wbos = []
for wbo_data in self.request.body_json:
if 'id' not in wbo_data: continue
wbo_data['collection'] = collection
wbo_id = wbo_data['id']
(wbo, errors) = WBO.from_json(wbo_data)
if wbo:
out['modified'] = wbo.modified
out['success'].append(wbo_id)
wbos.append(wbo)
else:
out['failed'][wbo_id] = errors
if (len(wbos) > 0):
db.put(wbos)
return out
def setUp(self):
"""Prepare for unit test"""
self.log = logging.getLogger()
self.log.setLevel(logging.DEBUG)
# There shouldn't already be a profile, but just in case...
profile = Profile.get_by_user_name(self.USER_NAME)
if profile: profile.delete()
# Create a new profile for tests.
self.profile = p = Profile(user_name=self.USER_NAME,
user_id='8675309',
password=self.PASSWD)
self.profile.put()
self.auth_header = self.build_auth_header(p.user_name, p.password)
self.collection = Collection.get_by_profile_and_name(p, 'testing')
self.wbo_values = [
dict(zip(self.value_keys, value_set))
for value_set in self.value_sets
]
for w in self.wbo_values:
w['payload'] = simplejson.dumps({'stuff': w['payload']})
# Build the app test harness.
self.app = webtest.TestApp(sync_api.application())
def setUp(self):
"""Prepare for unit test"""
self.log = logging.getLogger()
self.log.setLevel(logging.DEBUG)
# There shouldn't already be a profile, but just in case...
profile = Profile.get_by_user_name(self.USER_NAME)
if profile: profile.delete()
# Create a new profile for tests.
self.profile = p = Profile(
user_name = self.USER_NAME,
user_id = '8675309',
password = self.PASSWD
)
self.profile.put()
self.auth_header = self.build_auth_header(p.user_name, p.password)
self.collection = Collection.get_by_profile_and_name(p, 'testing')
self.wbo_values = [
dict(zip(self.value_keys, value_set))
for value_set in self.value_sets
]
for w in self.wbo_values:
w['payload'] = simplejson.dumps({ 'stuff': w['payload'] })
# Build the app test harness.
self.app = webtest.TestApp(sync_api.application())
def get(self, user_name, collection_name):
"""Filtered retrieval of WBOs from a collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name)
# TODO: Need a generator here?
# TODO: Find out how not to load everything into memory.
params = self.normalize_retrieval_parameters()
self.response.headers['X-Weave-Records'] = \
str(collection.retrieve(count=True, **params))
out = collection.retrieve(**params)
accept = ('Accept' not in self.request.headers and 'application/json'
or self.request.headers['Accept'])
if 'application/newlines' == accept:
self.response.headers['Content-Type'] = 'application/newlines'
for x in out:
self.response.out.write("%s\n" % simplejson.dumps(x))
elif 'application/whoisi' == accept:
self.response.headers['Content-Type'] = 'application/whoisi'
for x in out:
rec = simplejson.dumps(x)
self.response.out.write('%s%s' %
(struct.pack('!I', len(rec)), rec))
else:
self.response.headers['Content-Type'] = 'application/json'
rv = [x for x in out]
self.response.out.write(simplejson.dumps(rv))
def get(self, user_name, collection_name):
"""Filtered retrieval of WBOs from a collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name
)
# TODO: Need a generator here?
# TODO: Find out how not to load everything into memory.
params = self.normalize_retrieval_parameters()
self.response.headers['X-Weave-Records'] = \
str(collection.retrieve(count=True, **params))
out = collection.retrieve(**params)
accept = ('Accept' not in self.request.headers
and 'application/json' or self.request.headers['Accept'])
if 'application/newlines' == accept:
self.response.headers['Content-Type'] = 'application/newlines'
for x in out:
self.response.out.write("%s\n" % simplejson.dumps(x))
elif 'application/whoisi' == accept:
self.response.headers['Content-Type'] = 'application/whoisi'
for x in out:
rec = simplejson.dumps(x)
self.response.out.write('%s%s' % (
struct.pack('!I', len(rec)), rec
))
else:
self.response.headers['Content-Type'] = 'application/json'
rv = [x for x in out]
self.response.out.write(simplejson.dumps(rv))
def tearDown(self):
"""Clean up after unit test"""
# Is this actually needed, since storage is mocked?
self.profile.delete()
q = WBO.all()
for o in q: o.delete()
q = Collection.all()
for o in q: o.delete()
def get(self, user_name, collection_name, wbo_id):
"""Get an item from the collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name
)
wbo = WBO.get_by_collection_and_wbo_id(collection, wbo_id)
if not wbo: return self.error(404)
return wbo.to_dict()
def delete(self, user_name, collection_name, wbo_id):
"""Delete an item from the collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name)
wbo = WBO.get_by_collection_and_wbo_id(collection, wbo_id)
if not wbo: return self.error(404)
wbo.delete()
self.response.out.write('%s' % WBO.get_time_now())
def delete(self, user_name, collection_name):
"""Bulk deletion of WBOs from a collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name)
params = self.normalize_retrieval_parameters()
params['wbo'] = True
out = collection.retrieve(**params)
db.delete(out)
return WBO.get_time_now()
def delete(self, user_name, collection_name, wbo_id):
"""Delete an item from the collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name
)
wbo = WBO.get_by_collection_and_wbo_id(collection, wbo_id)
if not wbo: return self.error(404)
wbo.delete()
self.response.out.write('%s' % WBO.get_time_now())
def delete(self, user_name, collection_name):
"""Bulk deletion of WBOs from a collection"""
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name
)
params = self.normalize_retrieval_parameters()
params['wbo'] = True
out = collection.retrieve(**params)
db.delete(out)
return WBO.get_time_now()
def tearDown(self):
"""Clean up after unit test"""
# Is this actually needed, since storage is mocked?
self.profile.delete()
q = WBO.all()
for o in q:
o.delete()
q = Collection.all()
for o in q:
o.delete()
def test_cascading_collection_delete(self):
"""Ensure that collection deletion cascades down to WBOs"""
(p, c, ah) = (self.profile, self.collection, self.auth_header)
wbos = self.build_wbo_set()
count_all = WBO.all().count()
collections = [c for c in Collection.all().ancestor(p)]
for c in collections:
c_count = len([x for x in c.retrieve()])
c.delete()
count_all -= c_count
self.assertEqual(count_all, WBO.all().count())
self.assertEqual(0, WBO.all().count())
def test_cascading_collection_delete(self):
"""Ensure that collection deletion cascades down to WBOs"""
(p, c, ah) = (self.profile, self.collection, self.auth_header)
wbos = self.build_wbo_set()
count_all = WBO.all().count()
collections = [c for c in Collection.all().ancestor(p)]
for c in collections:
c_count = len([x for x in c.retrieve()])
c.delete()
count_all -= c_count
self.assertEqual(count_all, WBO.all().count())
self.assertEqual(0, WBO.all().count())
def post(self, user_name, collection_name):
"""Bulk update of WBOs in a collection"""
out = {'modified': None, 'success': [], 'failed': {}}
collection = Collection.get_by_profile_and_name(
self.request.profile, collection_name)
wbos = []
for wbo_data in self.request.body_json:
if 'id' not in wbo_data: continue
wbo_data['collection'] = collection
wbo_id = wbo_data['id']
(wbo, errors) = WBO.from_json(wbo_data)
if wbo:
out['modified'] = wbo.modified
out['success'].append(wbo_id)
wbos.append(wbo)
else:
out['failed'][wbo_id] = errors
if (len(wbos) > 0):
db.put(wbos)
return out
def get(self, user_name):
"""Get counts for a user's collections"""
return Collection.get_counts(self.request.profile)
def get(self, user_name):
"""List user's collections and last modified times"""
return Collection.get_timestamps(self.request.profile)
def get(self, user_name):
"""Get counts for a user's collections"""
return Collection.get_counts(self.request.profile)
def test_item_validation(self):
"""Exercise WBO data validation"""
(p, c, ah) = (self.profile, self.collection, self.auth_header)
too_long_id = ''.join('x' for x in range(100))
self.assert_('invalid id' in WBO.validate({'id': ''}))
self.assert_('invalid id' in WBO.validate({'id': 'foo/bar'}))
self.assert_('invalid id' in WBO.validate({'id': too_long_id}))
self.assert_('invalid id' not in WBO.validate({'id': 'abcd'}))
self.assert_('invalid collection' in WBO.validate({}))
self.assert_('invalid collection' in WBO.validate(
{'collection': Collection(name=too_long_id, profile=p)}))
self.assert_('invalid predecessorid' in WBO.validate(
{
'collection': c,
'predecessorid': too_long_id
}))
self.assert_(
'invalid predecessorid' in WBO.validate({
'collection': c,
'predecessorid': 'abcdef'
}))
w = WBO(parent=c,
collection=c,
wbo_id='abcdef',
modified=WBO.get_time_now(),
payload='test')
w.put()
self.assert_('invalid predecessorid' not in WBO.validate(
{
'collection': c,
'predecessorid': 'abcdef'
}))
self.assert_('invalid predecessorid' in WBO.validate(
{
'collection': c,
'predecessorid': too_long_id
}))
self.assert_(
'invalid predecessorid' in WBO.validate({
'collection': c,
'predecessorid': 'defghi'
}))
w = WBO(parent=c,
collection=c,
wbo_id='defghi',
modified=WBO.get_time_now(),
payload='test')
w.put()
self.assert_('invalid predecessorid' not in WBO.validate(
{
'collection': c,
'predecessorid': 'abcdef'
}))
self.assert_(
'invalid modified date' in WBO.validate({'modified': 'abc'}))
self.assert_('no modification date' in WBO.validate({}))
self.assert_('no modification date' in WBO.validate({'modified': ''}))
self.assert_(
'invalid sortindex' in WBO.validate({'sortindex': 'abcd'}))
self.assert_(
'invalid sortindex' in WBO.validate({'sortindex': -1000000000}))
self.assert_(
'invalid sortindex' in WBO.validate({'sortindex': 1000000000}))
self.assert_('payload needs to be json-encoded' in WBO.validate(
{'payload': 'abcd'}))
self.assert_('payload too large' in WBO.validate(
{'payload': 'x'.join('x' for x in range(500000))}))
def get(self, user_name):
"""List user's collections and last modified times"""
return Collection.get_timestamps(self.request.profile)