def load_data_from_mongoexport(res_id, export_location, collection_name, remove_id=False): """ This file should come from mongoexport, with or without the --jsonArray flag. That is to say, it should either be a series of documents, each on its own line, or a single array of documents. All documents will be inserted into the given collection. """ with open(export_location) as export: first_char = export.read(1) export.seek(0, SEEK_SET) if first_char == '[': # Data is already in an array documents = loads(export.read()) else: # Each line of data is an object documents = [] for line in export: documents.append(loads(line)) if remove_id: _remove_id(documents) with UseResId(res_id): get_db()[collection_name].insert(documents)
def test_mangles_collection_names_automatically(self): with self.real_app.app_context(): db = get_db() with UseResId('myresid.'): coll = db.foo self.assertEqual(coll.name, 'myresid.foo') coll = db.foo self.assertEqual(coll.name, 'foo')
def __precompute(self, collection, data_only, data, check_id): with UseResId(self.res_id): query = {'$or': data} if data_only else {} projection = None if check_id else {'_id': 0} result = self.db[collection].find(query, projection) data = (self.__hashable(x) for x in data) result = (self.__hashable(x) for x in result) return data, result
def setUp(self): super(QuotaCollectionsTestCase, self).setUp() self.old_quota = self.real_app.config['QUOTA_NUM_COLLECTIONS'] self.res_id = 'myresid.' with self.real_app.app_context(): self.db = get_db() collections = get_collection_names(self.res_id) with UseResId(self.res_id): for c in collections: self.db.drop_collection(c)
def test_quota_collections_zero(self): self.real_app.config['QUOTA_NUM_COLLECTIONS'] = 0 with self.real_app.app_context(), UseResId(self.res_id): with self.assertRaises(MWSServerError) as cm: self.db.a.insert({'a': 1}) self.assertEqual(cm.exception.error, 429) self.db.drop_collection('a')
def load_data_from_json(res_id, file_name, remove_id=False): """ The top level of this file should be an object who's keys are collection names which map to an array of documents to be inserted into the collection """ with open(file_name) as json_file: collections = loads(json_file.read()) db = get_db() with UseResId(res_id): for collection, documents in collections.iteritems(): if remove_id: _remove_id(documents) db[collection].insert(documents)
def load_data_from_mongodump(res_id, dump_location, collection_name): """ The dump location should point to a .bson file, not a directory structure as created by mongodump. Instead, use the .bson files inside this directory structure. """ if not os.path.exists(dump_location): raise NotFound('Unable to find dump file') p = Popen(('mongorestore', '-d', 'mws', '-c', '%s%s' % (res_id, collection_name), dump_location)) p.communicate() # Wait for process to finish if p.poll() != 0: raise InternalServerError('Loading dumped data failed') UseResId(res_id).insert_client_collection(collection_name)
def test_updates_collection_list(self): with self.real_app.app_context(): db = get_db() res_id = 'myresid.' # Setup resource id record clients_collection = db[CLIENTS_COLLECTION] clients_collection.remove({'res_id': res_id}) clients_collection.insert({'res_id': res_id, 'collections': []}) def get_collections(): # Can't use the util function because we would be using it # inside the with, so the collection name would be mangled return clients_collection.find({'res_id': res_id}, { '_id': 0, 'collections': 1 })[0]['collections'] with UseResId(res_id): self.assertItemsEqual(get_collections(), []) db.foo.insert({'message': 'test'}) self.assertItemsEqual(get_collections(), ['foo']) self.assertItemsEqual(list(db.foo.find({}, {'_id': 0})), [{ 'message': 'test' }]) db.bar.update({}, {'message': 'test'}) self.assertItemsEqual(get_collections(), ['foo']) db.bar.update({}, {'message': 'test'}, upsert=True) self.assertItemsEqual(get_collections(), ['foo', 'bar']) self.assertItemsEqual(list(db.bar.find({}, {'_id': 0})), [{ 'message': 'test' }]) db.foo.drop() self.assertItemsEqual(get_collections(), ['bar']) self.assertNotIn(res_id + 'foo', db.collection_names()) db.drop_collection('bar') self.assertItemsEqual(get_collections(), []) self.assertNotIn(res_id + 'bar', db.collection_names())
def run(res_id, data): with UseResId(res_id): db = get_db() collections = data['collections'] for collection, documents in collections.iteritems(): db[collection].insert(documents)