def db_collection_insert(res_id, collection_name):
# TODO: Ensure request.json is not None.
if 'document' in request.json:
document = request.json['document']
else:
error = '\'document\' argument not found in the insert request.'
raise MWSServerError(400, error)
# Check quota
size = get_collection_size(res_id, collection_name)
# Handle inserting both a list of docs or a single doc
if isinstance(document, list):
req_size = 0
for d in document:
req_size += len(BSON.encode(d))
else:
req_size = len(BSON.encode(document))
if size + req_size > current_app.config['QUOTA_COLLECTION_SIZE']:
raise MWSServerError(403, 'Collection size exceeded')
# Insert document
with UseResId(res_id):
get_db()[collection_name].insert(document)
return empty_success()
def load_data_from_mongoexport(res_id,
export_location,
collection_name,
remove_id=False):
"""
This file should come from mongoexport, with or without the --jsonArray
flag. That is to say, it should either be a series of documents, each on
its own line, or a single array of documents. All documents will be
inserted into the given collection.
"""
with open(export_location) as export:
first_char = export.read(1)
export.seek(0, SEEK_SET)
if first_char == '[':
# Data is already in an array
documents = loads(export.read())
else:
# Each line of data is an object
documents = []
for line in export:
documents.append(loads(line))
if remove_id:
_remove_id(documents)
with UseResId(res_id):
get_db()[collection_name].insert(documents)
def test_loads_exported_data(self, open_mock):
documents = [
{'_id': 1, 'msg': 'my test string'},
{'_id': 2, 'message': 'my other string'},
{'_id': 3, 'foo': 'bar', 'greeting': 'hi there'},
]
file_contents = '\n'.join([dumps(doc) for doc in documents])
self.mock_open(open_mock, file_contents)
with self.real_app.app_context():
db = get_db()
# Test normally (keeping the _id)
load_data_from_mongoexport('myresid.', 'my/file/location',
'mycoll')
open_mock.assert_called_with('my/file/location')
collection_contents = list(db['myresid.mycoll'].find())
self.assertItemsEqual(collection_contents, documents)
db['myresid.mycoll'].drop()
# Test removing the _id
load_data_from_mongoexport('myresid.', 'my/file/location',
'mycoll', True)
collection_contents = list(db['myresid.mycoll'].find())
for doc in collection_contents:
# Should not be any of the given _id's
self.assertNotIn(doc['_id'], (1, 2, 3))
db['myresid.mycoll'].drop()
def test_keep_mws_alive(self, datetime_mock):
first = datetime.datetime(2012, 7, 4)
second = first + datetime.timedelta(days=1)
datetime_mock.now.return_value = first
db = get_db()
# get a session to keep alive
rv = self.app.post('/mws/')
res_id = loads(rv.data)['res_id']
with self.app.session_transaction() as sess:
session_id = sess['session_id']
res = db.clients.find({
'res_id': res_id,
'session_id': session_id
}, {'timestamp': 1})
_id = res[0]['_id']
old_ts = res[0]['timestamp']
self.assertEqual(old_ts, first)
datetime_mock.now.return_value = second
url = '/mws/' + res_id + '/keep-alive'
rv = self.app.post(url)
self.assertEqual(rv.status_code, 204)
newres = db.clients.find({'_id': _id}, {'timestamp': 1})
self.assertEqual(newres[0]['timestamp'], second)
def test_loads_exported_array_data(self, open_mock):
documents = [
{
'_id': 1,
'msg': 'my test string'
},
{
'_id': 2,
'message': 'my other string'
},
{
'_id': 3,
'foo': 'bar',
'greeting': 'hi there'
},
]
file_contents = dumps(documents)
self.mock_open(open_mock, file_contents)
with self.real_app.app_context():
db = get_db()
load_data_from_mongoexport('myresid.', 'my/file/location',
'mycoll')
open_mock.assert_called_with('my/file/location')
collection_contents = list(db['myresid.mycoll'].find())
self.assertItemsEqual(collection_contents, documents)
db['myresid.mycoll'].drop()
def ensure_indices(app):
with app.app_context():
db = get_db()
db.ratelimit.ensure_index([('session_id', 1), ('timestamp', 1)])
db.ratelimit.ensure_index('timestamp',
background=True,
expireAfterSeconds=60)
def db_collection_update(res_id, collection_name):
query = update = None
if request.json:
query = request.json.get('query')
update = request.json.get('update')
upsert = request.json.get('upsert', False)
multi = request.json.get('multi', False)
if query is None or update is None:
error = 'update requires spec and document arguments'
raise MWSServerError(400, error)
# Check quota
size = get_collection_size(res_id, collection_name)
with UseResId(res_id):
# Computation of worst case size increase - update size * docs affected
# It would be nice if we were able to make a more conservative estimate
# of the space difference that an update will cause. (especially if it
# results in smaller documents)
db = get_db()
affected = db[collection_name].find(query).count()
req_size = len(BSON.encode(update)) * affected
if size + req_size > current_app.config['QUOTA_COLLECTION_SIZE']:
raise MWSServerError(403, 'Collection size exceeded')
db[collection_name].update(query, update, upsert, multi=multi)
return empty_success()
def db_drop(res_id):
DB = get_db()
collections = get_collection_names(res_id)
with UseResId(res_id):
for c in collections:
DB.drop_collection(c)
return empty_success()
def test_mangles_collection_names_automatically(self):
with self.real_app.app_context():
db = get_db()
with UseResId('myresid.'):
coll = db.foo
self.assertEqual(coll.name, 'myresid.foo')
coll = db.foo
self.assertEqual(coll.name, 'foo')
def get_collection_names(res_id):
"""
Get the collection names associated with a given resource id. Should not be
called from within a 'with UseResId(res_id)' block.
"""
return get_db()[mongows.mws.views.CLIENTS_COLLECTION].find({"res_id": res_id}, {"collections": 1, "_id": 0})[0][
"collections"
]
def get_collection_names(res_id):
"""
Get the collection names associated with a given resource id. Should not be
called from within a 'with UseResId(res_id)' block.
"""
return get_db()[mongows.mws.views.CLIENTS_COLLECTION].find(
{'res_id': res_id}, {'collections': 1, '_id': 0}
)[0]['collections']
def test_mangles_collection_names_automatically(self):
with self.real_app.app_context():
db = get_db()
with UseResId('myresid.'):
coll = db.foo
self.assertEqual(coll.name, 'myresid.foo')
coll = db.foo
self.assertEqual(coll.name, 'foo')
def setUp(self):
class ValidationTestCase(ValidationTest):
def run(self):
pass
self.db = get_db()
self.validator = ValidationTestCase('test_')
self.coll = 'test_coll'
self.db.drop_collection(self.coll)
def db_collection_aggregate(res_id, collection_name):
parse_get_json(request)
try:
with UseResId(res_id):
coll = get_db()[collection_name]
result = coll.aggregate(request.json)
return to_json(result)
except OperationFailure as e:
raise MWSServerError(400, e.message)
def get_collection_size(res_id, collection_name):
coll = get_internal_coll_name(res_id, collection_name)
try:
return get_db().command({'collstats': coll})['size']
except OperationFailure as e:
if 'ns not found' in e.message:
return 0
else:
raise MWSServerError(500, e.message)
def setUp(self):
super(ExpireSessionsTestCase, self).setUp()
self.context = self.real_app.app_context()
self.context.push()
self.db = get_db()
# All test data should be before this date
# We can assume we don't have any real data before this date
self.test_before_date = datetime(2012, 7, 6)
self.db.clients.remove({'timestamp': {'$lt': self.test_before_date}})
def setUp(self):
super(QuotaCollectionsTestCase, self).setUp()
self.old_quota = self.real_app.config['QUOTA_NUM_COLLECTIONS']
self.res_id = 'myresid.'
with self.real_app.app_context():
self.db = get_db()
collections = get_collection_names(self.res_id)
with UseResId(self.res_id):
for c in collections:
self.db.drop_collection(c)
def setUp(self):
super(QuotaCollectionsTestCase, self).setUp()
self.old_quota = self.real_app.config['QUOTA_NUM_COLLECTIONS']
self.res_id = 'myresid.'
with self.real_app.app_context():
self.db = get_db()
collections = get_collection_names(self.res_id)
with UseResId(self.res_id):
for c in collections:
self.db.drop_collection(c)
def setUp(self):
super(DBCollectionTestCase, self).setUp()
self.coll_name = 'test_collection'
self.internal_coll_name = get_internal_coll_name(self.res_id,
self.coll_name)
self.db = get_db()
self.db_collection = self.db[self.internal_coll_name]
self.make_request_url = '/mws/%s/db/%s/%%s' % \
(self.res_id, self.coll_name)
def db_collection_remove(res_id, collection_name):
constraint = request.json.get('constraint') if request.json else {}
just_one = request.json and request.json.get('just_one', False)
with UseResId(res_id):
collection = get_db()[collection_name]
if just_one:
collection.find_and_modify(constraint, remove=True)
else:
collection.remove(constraint)
return empty_success()
def setUp(self):
super(DBCollectionTestCase, self).setUp()
self.coll_name = 'test_collection'
self.internal_coll_name = get_internal_coll_name(
self.res_id, self.coll_name)
self.db = get_db()
self.db_collection = self.db[self.internal_coll_name]
self.make_request_url = '/mws/%s/db/%s/%%s' % \
(self.res_id, self.coll_name)
def db_collection_count(res_id, collection_name):
parse_get_json(request)
query = request.json.get('query')
skip = request.json.get('skip', 0)
limit = request.json.get('limit', 0)
use_skip_limit = bool(skip or limit)
with UseResId(res_id):
coll = get_db()[collection_name]
cursor = coll.find(query, skip=skip, limit=limit)
count = cursor.count(use_skip_limit)
return to_json({'count': count})
def load_data_from_json(res_id, file_name, remove_id=False):
"""
The top level of this file should be an object who's keys are collection
names which map to an array of documents to be inserted into the collection
"""
with open(file_name) as json_file:
collections = loads(json_file.read())
db = get_db()
with UseResId(res_id):
for collection, documents in collections.iteritems():
if remove_id:
_remove_id(documents)
db[collection].insert(documents)
def load_data_from_json(res_id, file_name, remove_id=False):
"""
The top level of this file should be an object who's keys are collection
names which map to an array of documents to be inserted into the collection
"""
with open(file_name) as json_file:
collections = loads(json_file.read())
db = get_db()
with UseResId(res_id):
for collection, documents in collections.iteritems():
if remove_id:
_remove_id(documents)
db[collection].insert(documents)
def setUp(self):
super(DBTestCase, self).setUp()
# Todo: For stuff that isn't checking authentication,
# we probably don't want to rely on/use the authentication code
rv = self.app.post('/mws/')
response_dict = loads(rv.data)
self.assertIn('res_id', response_dict)
self.res_id = response_dict['res_id']
self.assertIsNotNone(self.res_id)
self.db = get_db()
self.make_request_url = '/mws/%s/db/%%s' % (self.res_id)
def setUp(self):
super(DBTestCase, self).setUp()
# Todo: For stuff that isn't checking authentication,
# we probably don't want to rely on/use the authentication code
rv = self.app.post('/mws/')
response_dict = loads(rv.data)
self.assertIn('res_id', response_dict)
self.res_id = response_dict['res_id']
self.assertIsNotNone(self.res_id)
self.db = get_db()
self.make_request_url = '/mws/%s/db/%%s' % (self.res_id)
def test_loads_json_data(self, open_mock):
documents = {
'first_coll': [
{'_id': 1, 'msg': 'my test string'},
{'_id': 2, 'message': 'my other string'},
{'_id': 3, 'foo': 'bar', 'greeting': 'hi there'},
],
'viewing_preferences': [
{'_id': 2, 'tv_shows': ['archer', 'firefly']},
{'_id': 3, 'tv_shows': ['arrested development'],
'movies': ['shawshank redemption']},
],
}
file_contents = dumps(documents)
with self.real_app.app_context():
db = get_db()
# Test normally (keeping the _id)
self.mock_open(open_mock, file_contents)
load_data_from_json('myresid.', 'my/file/location')
open_mock.assert_called_with('my/file/location')
first_coll_contents = list(db['myresid.first_coll'].find())
self.assertItemsEqual(first_coll_contents, documents['first_coll'])
viewing_prefs_contents = list(
db['myresid.viewing_preferences'].find()
)
self.assertItemsEqual(viewing_prefs_contents,
documents['viewing_preferences'])
db['myresid.first_coll'].drop()
db['myresid.viewing_preferences'].drop()
# Test removing the _id's
self.mock_open(open_mock, file_contents)
load_data_from_json('myresid.', 'my/file/location', True)
first_coll_contents = list(db['myresid.first_coll'].find())
for doc in first_coll_contents:
self.assertNotIn(doc['_id'], (1, 2, 3))
viewing_prefs_contents = list(
db['myresid.viewing_preferences'].find()
)
for doc in viewing_prefs_contents:
self.assertNotIn(doc['_id'], (1, 2, 3))
db['myresid.first_coll'].drop()
db['myresid.viewing_preferences'].drop()
def load_data_from_mongoexport(res_id, export_location, collection_name,
remove_id=False):
"""
This file should come from mongoexport, with or without the --jsonArray
flag. That is to say, it should either be a series of documents, each on
its own line, or a single array of documents. All documents will be
inserted into the given collection.
"""
with open(export_location) as export:
first_char = export.read(1)
export.seek(0, SEEK_SET)
if first_char == '[':
# Data is already in an array
documents = loads(export.read())
else:
# Each line of data is an object
documents = []
for line in export:
documents.append(loads(line))
if remove_id:
_remove_id(documents)
with UseResId(res_id):
get_db()[collection_name].insert(documents)
def db_collection_find(res_id, collection_name):
# TODO: Should we specify a content type? Then we have to use an options
# header, and we should probably get the return type from the content-type
# header.
parse_get_json(request)
query = request.json.get('query')
projection = request.json.get('projection')
skip = request.json.get('skip', 0)
limit = request.json.get('limit', 0)
with UseResId(res_id):
coll = get_db()[collection_name]
cursor = coll.find(query, projection, skip, limit)
documents = list(cursor)
return to_json({'result': documents})
def wrapped_function(*args, **kwargs):
session_id = session.get('session_id')
if session_id is None:
error = 'Cannot rate limit without session_id cookie'
raise MWSServerError(401, error)
config = current_app.config
coll = get_db()[config['RATELIMIT_COLLECTION']]
coll.insert({'session_id': session_id, 'timestamp': datetime.now()})
delta = timedelta(seconds=config['RATELIMIT_EXPIRY'])
expiry = datetime.now() - delta
accesses = coll.find({'session_id': session_id,
'timestamp': {'$gt': expiry}})
if accesses.count() > config['RATELIMIT_QUOTA']:
raise MWSServerError(429, 'Rate limit exceeded')
return f(*args, **kwargs)
def test_updates_collection_list(self):
with self.real_app.app_context():
db = get_db()
res_id = 'myresid.'
# Setup resource id record
clients_collection = db[CLIENTS_COLLECTION]
clients_collection.remove({'res_id': res_id})
clients_collection.insert({'res_id': res_id, 'collections': []})
def get_collections():
# Can't use the util function because we would be using it
# inside the with, so the collection name would be mangled
return clients_collection.find({'res_id': res_id}, {
'_id': 0,
'collections': 1
})[0]['collections']
with UseResId(res_id):
self.assertItemsEqual(get_collections(), [])
db.foo.insert({'message': 'test'})
self.assertItemsEqual(get_collections(), ['foo'])
self.assertItemsEqual(list(db.foo.find({}, {'_id': 0})),
[{
'message': 'test'
}])
db.bar.update({}, {'message': 'test'})
self.assertItemsEqual(get_collections(), ['foo'])
db.bar.update({}, {'message': 'test'}, upsert=True)
self.assertItemsEqual(get_collections(), ['foo', 'bar'])
self.assertItemsEqual(list(db.bar.find({}, {'_id': 0})),
[{
'message': 'test'
}])
db.foo.drop()
self.assertItemsEqual(get_collections(), ['bar'])
self.assertNotIn(res_id + 'foo', db.collection_names())
db.drop_collection('bar')
self.assertItemsEqual(get_collections(), [])
self.assertNotIn(res_id + 'bar', db.collection_names())
def test_loads_exported_array_data(self, open_mock):
documents = [
{'_id': 1, 'msg': 'my test string'},
{'_id': 2, 'message': 'my other string'},
{'_id': 3, 'foo': 'bar', 'greeting': 'hi there'},
]
file_contents = dumps(documents)
self.mock_open(open_mock, file_contents)
with self.real_app.app_context():
db = get_db()
load_data_from_mongoexport('myresid.',
'my/file/location', 'mycoll')
open_mock.assert_called_with('my/file/location')
collection_contents = list(db['myresid.mycoll'].find())
self.assertItemsEqual(collection_contents, documents)
db['myresid.mycoll'].drop()
def test_updates_collection_list(self):
with self.real_app.app_context():
db = get_db()
res_id = 'myresid.'
# Setup resource id record
clients_collection = db[CLIENTS_COLLECTION]
clients_collection.remove({'res_id': res_id})
clients_collection.insert({
'res_id': res_id,
'collections': []
})
def get_collections():
# Can't use the util function because we would be using it
# inside the with, so the collection name would be mangled
return clients_collection.find(
{'res_id': res_id},
{'_id': 0, 'collections': 1}
)[0]['collections']
with UseResId(res_id):
self.assertItemsEqual(get_collections(), [])
db.foo.insert({'message': 'test'})
self.assertItemsEqual(get_collections(), ['foo'])
self.assertItemsEqual(list(db.foo.find({}, {'_id': 0})),
[{'message': 'test'}])
db.bar.update({}, {'message': 'test'})
self.assertItemsEqual(get_collections(), ['foo'])
db.bar.update({}, {'message': 'test'}, upsert=True)
self.assertItemsEqual(get_collections(), ['foo', 'bar'])
self.assertItemsEqual(list(db.bar.find({}, {'_id': 0})),
[{'message': 'test'}])
db.foo.drop()
self.assertItemsEqual(get_collections(), ['bar'])
self.assertNotIn(res_id + 'foo', db.collection_names())
db.drop_collection('bar')
self.assertItemsEqual(get_collections(), [])
self.assertNotIn(res_id + 'bar', db.collection_names())
def create_mws_resource():
session_id = session.get('session_id', str(uuid.uuid4()))
session['session_id'] = session_id
clients = get_db()[CLIENTS_COLLECTION]
cursor = clients.find({'session_id': session_id}, {'res_id': 1, '_id': 0})
if cursor.count():
# TODO: handle multiple res_id per session
res_id = cursor[0]['res_id']
is_new = False
else:
res_id = generate_res_id()
clients.insert({
'version': 1,
'res_id': res_id,
'collections': [],
'session_id': session_id,
'timestamp': datetime.now()
})
is_new = True
return to_json({'res_id': res_id, 'is_new': is_new})
def test_loads_exported_data(self, open_mock):
documents = [
{
'_id': 1,
'msg': 'my test string'
},
{
'_id': 2,
'message': 'my other string'
},
{
'_id': 3,
'foo': 'bar',
'greeting': 'hi there'
},
]
file_contents = '\n'.join([dumps(doc) for doc in documents])
self.mock_open(open_mock, file_contents)
with self.real_app.app_context():
db = get_db()
# Test normally (keeping the _id)
load_data_from_mongoexport('myresid.', 'my/file/location',
'mycoll')
open_mock.assert_called_with('my/file/location')
collection_contents = list(db['myresid.mycoll'].find())
self.assertItemsEqual(collection_contents, documents)
db['myresid.mycoll'].drop()
# Test removing the _id
load_data_from_mongoexport('myresid.', 'my/file/location',
'mycoll', True)
collection_contents = list(db['myresid.mycoll'].find())
for doc in collection_contents:
# Should not be any of the given _id's
self.assertNotIn(doc['_id'], (1, 2, 3))
db['myresid.mycoll'].drop()
def test_keep_mws_alive(self, datetime_mock):
first = datetime.datetime(2012, 7, 4)
second = first + datetime.timedelta(days=1)
datetime_mock.now.return_value = first
db = get_db()
# get a session to keep alive
rv = self.app.post('/mws/')
res_id = loads(rv.data)['res_id']
with self.app.session_transaction() as sess:
session_id = sess['session_id']
res = db.clients.find({'res_id': res_id, 'session_id': session_id},
{'timestamp': 1})
_id = res[0]['_id']
old_ts = res[0]['timestamp']
self.assertEqual(old_ts, first)
datetime_mock.now.return_value = second
url = '/mws/' + res_id + '/keep-alive'
rv = self.app.post(url)
self.assertEqual(rv.status_code, 204)
newres = db.clients.find({'_id': _id}, {'timestamp': 1})
self.assertEqual(newres[0]['timestamp'], second)
def __init__(self, res_id):
self.res_id = res_id
self.db = get_db()
def db_collection_drop(res_id, collection_name):
with UseResId(res_id):
get_db().drop_collection(collection_name)
return empty_success()
def user_has_access(res_id, session_id):
query = {'res_id': res_id, 'session_id': session_id}
coll = get_db()[CLIENTS_COLLECTION]
return_value = coll.find_one(query)
return False if return_value is None else True
def run(res_id, data):
with UseResId(res_id):
db = get_db()
collections = data['collections']
for collection, documents in collections.iteritems():
db[collection].insert(documents)
def __init__(self, res_id):
self.res_id = str(res_id)
self.id_length = len(self.res_id)
self.client_collection = get_db()[mongows.mws.views.CLIENTS_COLLECTION]
def keep_mws_alive(res_id):
clients = get_db()[CLIENTS_COLLECTION]
clients.update({'session_id': session.get('session_id'), 'res_id': res_id},
{'$set': {'timestamp': datetime.now()}})
return empty_success()
def test_loads_json_data(self, open_mock):
documents = {
'first_coll': [
{
'_id': 1,
'msg': 'my test string'
},
{
'_id': 2,
'message': 'my other string'
},
{
'_id': 3,
'foo': 'bar',
'greeting': 'hi there'
},
],
'viewing_preferences': [
{
'_id': 2,
'tv_shows': ['archer', 'firefly']
},
{
'_id': 3,
'tv_shows': ['arrested development'],
'movies': ['shawshank redemption']
},
],
}
file_contents = dumps(documents)
with self.real_app.app_context():
db = get_db()
# Test normally (keeping the _id)
self.mock_open(open_mock, file_contents)
load_data_from_json('myresid.', 'my/file/location')
open_mock.assert_called_with('my/file/location')
first_coll_contents = list(db['myresid.first_coll'].find())
self.assertItemsEqual(first_coll_contents, documents['first_coll'])
viewing_prefs_contents = list(
db['myresid.viewing_preferences'].find())
self.assertItemsEqual(viewing_prefs_contents,
documents['viewing_preferences'])
db['myresid.first_coll'].drop()
db['myresid.viewing_preferences'].drop()
# Test removing the _id's
self.mock_open(open_mock, file_contents)
load_data_from_json('myresid.', 'my/file/location', True)
first_coll_contents = list(db['myresid.first_coll'].find())
for doc in first_coll_contents:
self.assertNotIn(doc['_id'], (1, 2, 3))
viewing_prefs_contents = list(
db['myresid.viewing_preferences'].find())
for doc in viewing_prefs_contents:
self.assertNotIn(doc['_id'], (1, 2, 3))
db['myresid.first_coll'].drop()
db['myresid.viewing_preferences'].drop()
def __init__(self, res_id):
self.res_id = str(res_id)
self.id_length = len(self.res_id)
self.client_collection = get_db()[mongows.mws.views.CLIENTS_COLLECTION]
def __init__(self, res_id):
self.res_id = res_id
self.db = get_db()