def load_data_from_mongoexport(res_id,
export_location,
collection_name,
remove_id=False):
"""
This file should come from mongoexport, with or without the --jsonArray
flag. That is to say, it should either be a series of documents, each on
its own line, or a single array of documents. All documents will be
inserted into the given collection.
"""
with open(export_location) as export:
first_char = export.read(1)
export.seek(0, SEEK_SET)
if first_char == '[':
# Data is already in an array
documents = loads(export.read())
else:
# Each line of data is an object
documents = []
for line in export:
documents.append(loads(line))
if remove_id:
_remove_id(documents)
with UseResId(res_id):
get_db()[collection_name].insert(documents)
def test_mangles_collection_names_automatically(self):
with self.real_app.app_context():
db = get_db()
with UseResId('myresid.'):
coll = db.foo
self.assertEqual(coll.name, 'myresid.foo')
coll = db.foo
self.assertEqual(coll.name, 'foo')
def __precompute(self, collection, data_only, data, check_id):
with UseResId(self.res_id):
query = {'$or': data} if data_only else {}
projection = None if check_id else {'_id': 0}
result = self.db[collection].find(query, projection)
data = (self.__hashable(x) for x in data)
result = (self.__hashable(x) for x in result)
return data, result
def setUp(self):
super(QuotaCollectionsTestCase, self).setUp()
self.old_quota = self.real_app.config['QUOTA_NUM_COLLECTIONS']
self.res_id = 'myresid.'
with self.real_app.app_context():
self.db = get_db()
collections = get_collection_names(self.res_id)
with UseResId(self.res_id):
for c in collections:
self.db.drop_collection(c)
def test_quota_collections_zero(self):
self.real_app.config['QUOTA_NUM_COLLECTIONS'] = 0
with self.real_app.app_context(), UseResId(self.res_id):
with self.assertRaises(MWSServerError) as cm:
self.db.a.insert({'a': 1})
self.assertEqual(cm.exception.error, 429)
self.db.drop_collection('a')
def load_data_from_json(res_id, file_name, remove_id=False):
"""
The top level of this file should be an object who's keys are collection
names which map to an array of documents to be inserted into the collection
"""
with open(file_name) as json_file:
collections = loads(json_file.read())
db = get_db()
with UseResId(res_id):
for collection, documents in collections.iteritems():
if remove_id:
_remove_id(documents)
db[collection].insert(documents)
def load_data_from_mongodump(res_id, dump_location, collection_name):
"""
The dump location should point to a .bson file, not a directory structure
as created by mongodump. Instead, use the .bson files inside this
directory structure.
"""
if not os.path.exists(dump_location):
raise NotFound('Unable to find dump file')
p = Popen(('mongorestore', '-d', 'mws', '-c',
'%s%s' % (res_id, collection_name), dump_location))
p.communicate() # Wait for process to finish
if p.poll() != 0:
raise InternalServerError('Loading dumped data failed')
UseResId(res_id).insert_client_collection(collection_name)
def test_updates_collection_list(self):
with self.real_app.app_context():
db = get_db()
res_id = 'myresid.'
# Setup resource id record
clients_collection = db[CLIENTS_COLLECTION]
clients_collection.remove({'res_id': res_id})
clients_collection.insert({'res_id': res_id, 'collections': []})
def get_collections():
# Can't use the util function because we would be using it
# inside the with, so the collection name would be mangled
return clients_collection.find({'res_id': res_id}, {
'_id': 0,
'collections': 1
})[0]['collections']
with UseResId(res_id):
self.assertItemsEqual(get_collections(), [])
db.foo.insert({'message': 'test'})
self.assertItemsEqual(get_collections(), ['foo'])
self.assertItemsEqual(list(db.foo.find({}, {'_id': 0})),
[{
'message': 'test'
}])
db.bar.update({}, {'message': 'test'})
self.assertItemsEqual(get_collections(), ['foo'])
db.bar.update({}, {'message': 'test'}, upsert=True)
self.assertItemsEqual(get_collections(), ['foo', 'bar'])
self.assertItemsEqual(list(db.bar.find({}, {'_id': 0})),
[{
'message': 'test'
}])
db.foo.drop()
self.assertItemsEqual(get_collections(), ['bar'])
self.assertNotIn(res_id + 'foo', db.collection_names())
db.drop_collection('bar')
self.assertItemsEqual(get_collections(), [])
self.assertNotIn(res_id + 'bar', db.collection_names())
def run(res_id, data):
with UseResId(res_id):
db = get_db()
collections = data['collections']
for collection, documents in collections.iteritems():
db[collection].insert(documents)