def get_empty_db_da(_database_name, _json_schema_folders=None, _uri_handlers = None): """ Create an empty database. Drops any existing. :param _database_name: The name of the database :return: A database access object for the database :param _json_schema_folders: A list of application specific JSON schema folders """ _client = MongoClient() if _database_name in _client.database_names(): _client.drop_database(_client[_database_name]) _database = _client[_database_name] return DatabaseAccess(_database=_database, _json_schema_folders=_json_schema_folders, _uri_handlers=_uri_handlers)
class TestMongosLoadBalancing(HATestCase): def setUp(self): super(TestMongosLoadBalancing, self).setUp() seed_list = ha_tools.create_sharded_cluster() self.assertIsNotNone(seed_list) self.dbname = 'pymongo_mongos_ha' self.client = MongoClient( seed_list, serverSelectionTimeoutMS=self.server_selection_timeout) self.client.drop_database(self.dbname) def test_mongos_load_balancing(self): wait_until(lambda: len(ha_tools.routers) == len(self.client.nodes), 'discover all mongoses') # Can't access "address" when load balancing. with self.assertRaises(InvalidOperation): self.client.address coll = self.client[self.dbname].test coll.insert_one({'foo': 'bar'}) live_routers = list(ha_tools.routers) ha_tools.kill_mongos(live_routers.pop()) while live_routers: try: self.assertEqual(1, coll.count()) except ConnectionFailure: # If first attempt happened to select the dead mongos. self.assertEqual(1, coll.count()) wait_until(lambda: len(live_routers) == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.kill_mongos(live_routers.pop()) # Make sure the last one's really dead. time.sleep(1) # I'm alone. self.assertRaises(ConnectionFailure, coll.count) wait_until(lambda: 0 == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.restart_mongos(one(ha_tools.routers)) # Find new mongos self.assertEqual(1, coll.count())
def __init__(self, mongo, db_name="news", new=False): if not mongo or not ("ip" in mongo and "port" in mongo): raise Exception if "user" in mongo and "pass" in mongo and mongo["user"] and mongo["pass"]: client = MongoClient( "mongodb://{}:{}@{}:{}".format( mongo["user"], mongo["pass"], mongo["ip"], mongo["port"] ) ) else: client = MongoClient(mongo["ip"], mongo["port"]) self._db = client[db_name] if new and not db_name == "news": client.drop_database(db_name) self._db.news.create_index([("title", HASHED)], name="news_title_index") self._db.news.create_index([("text", HASHED)], name="news_text_index") self._db.news.create_index([("link", HASHED)], name="news_link_index")
class TestMongosHighAvailability(unittest.TestCase): def setUp(self): seed_list = ha_tools.create_sharded_cluster() self.dbname = 'pymongo_mongos_ha' self.client = MongoClient(seed_list) self.client.drop_database(self.dbname) def test_mongos_ha(self): coll = self.client[self.dbname].test self.assertTrue(coll.insert({'foo': 'bar'})) first = '%s:%d' % (self.client.host, self.client.port) ha_tools.kill_mongos(first) # Fail first attempt self.assertRaises(AutoReconnect, coll.count) # Find new mongos self.assertEqual(1, coll.count()) second = '%s:%d' % (self.client.host, self.client.port) self.assertNotEqual(first, second) ha_tools.kill_mongos(second) # Fail first attempt self.assertRaises(AutoReconnect, coll.count) # Find new mongos self.assertEqual(1, coll.count()) third = '%s:%d' % (self.client.host, self.client.port) self.assertNotEqual(second, third) ha_tools.kill_mongos(third) # Fail first attempt self.assertRaises(AutoReconnect, coll.count) # We've killed all three, restart one. ha_tools.restart_mongos(first) # Find new mongos self.assertEqual(1, coll.count()) def tearDown(self): self.client.drop_database(self.dbname) ha_tools.kill_all_members()
class TestMongosHighAvailability(HATestCase): def setUp(self): seed_list = ha_tools.create_sharded_cluster() self.dbname = 'pymongo_mongos_ha' self.client = MongoClient(seed_list) self.client.drop_database(self.dbname) def test_mongos_ha(self): coll = self.client[self.dbname].test self.assertTrue(coll.insert({'foo': 'bar'})) first = '%s:%d' % (self.client.host, self.client.port) ha_tools.kill_mongos(first) # Fail first attempt self.assertRaises(AutoReconnect, coll.count) # Find new mongos self.assertEqual(1, coll.count()) second = '%s:%d' % (self.client.host, self.client.port) self.assertNotEqual(first, second) ha_tools.kill_mongos(second) # Fail first attempt self.assertRaises(AutoReconnect, coll.count) # Find new mongos self.assertEqual(1, coll.count()) third = '%s:%d' % (self.client.host, self.client.port) self.assertNotEqual(second, third) ha_tools.kill_mongos(third) # Fail first attempt self.assertRaises(AutoReconnect, coll.count) # We've killed all three, restart one. ha_tools.restart_mongos(first) # Find new mongos self.assertEqual(1, coll.count()) def tearDown(self): self.client.drop_database(self.dbname) super(TestMongosHighAvailability, self).tearDown()
def test_drop_database(self): client = MongoClient(host, port) self.assertRaises(TypeError, client.drop_database, 5) self.assertRaises(TypeError, client.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") client.pymongo_test.test.save({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) client.drop_database("pymongo_test") dbs = client.database_names() self.assertTrue("pymongo_test" not in dbs) client.pymongo_test.test.save({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) client.drop_database(client.pymongo_test) dbs = client.database_names() self.assertTrue("pymongo_test" not in dbs)
def test_copy_db(self): c = MongoClient(host, port) # Due to SERVER-2329, databases may not disappear # from a master in a master-slave pair. if server_is_master_with_slave(c): raise SkipTest("SERVER-2329") ctx = catch_warnings() try: warnings.simplefilter("ignore", DeprecationWarning) self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.pymongo_test.test.insert({"foo": "bar"}) c.drop_database("pymongo_test1") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.drop_database("pymongo_test1") # XXX - SERVER-15318 if not (version.at_least(c, (2, 6, 4)) and is_mongos(c)): self.assertFalse(c.in_request()) c.copy_database("pymongo_test", "pymongo_test1", "%s:%d" % (host, port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.drop_database("pymongo_test1") finally: ctx.exit()
def test_copy_db(self): c = MongoClient(host, port) # We test copy twice; once starting in a request and once not. In # either case the copy should succeed (because it starts a request # internally) and should leave us in the same state as before the copy. c.start_request() self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.pymongo_test.test.insert({"foo": "bar"}) # Due to SERVER-2329, databases may not disappear from a master in a # master-slave pair if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") # copy_database() didn't accidentally end the request self.assertTrue(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.end_request() self.assertFalse(c.in_request()) c.copy_database("pymongo_test", "pymongo_test2", "%s:%d" % (host, port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) if version.at_least(c, (1, 3, 3, 1)): c.drop_database("pymongo_test1") c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) if not is_mongos(c): # See SERVER-6427 c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
def clear(self): client = MongoClient(ARCTIC_SRV) client.drop_database("arctic")
def setUpClass(cls): mongo = MongoClient("mongodb://localhost/") mongo.drop_database('ermak-test') cls.db = mongo['ermak-test']
def get_empty_db_da(_database_name, _json_schema_folder): _client = MongoClient() if _database_name in _client.database_names(): _client.drop_database(_client[_database_name]) _database = _client[_database_name] return DatabaseAccess(_json_schema_folder=None, _database=_database)
def test_copy_db(self): c = MongoClient(host, port) # Due to SERVER-2329, databases may not disappear # from a master in a master-slave pair. if server_is_master_with_slave(c): raise SkipTest("SERVER-2329") # We test copy twice; once starting in a request and once not. In # either case the copy should succeed (because it starts a request # internally) and should leave us in the same state as before the copy. c.start_request() self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.pymongo_test.test.insert({"foo": "bar"}) c.copy_database("pymongo_test", "pymongo_test1") # copy_database() didn't accidentally end the request self.assertTrue(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.end_request() self.assertFalse(c.in_request()) c.copy_database("pymongo_test", "pymongo_test2", "%s:%d" % (host, port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) # See SERVER-6427 for mongos if (version.at_least(c, (1, 3, 3, 1)) and not is_mongos(c) and server_started_with_auth(c)): c.drop_database("pymongo_test1") c.admin.add_user("admin", "password") c.admin.authenticate("admin", "password") try: c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) finally: # Cleanup remove_all_users(c.pymongo_test) c.admin.remove_user("admin") c.disconnect()
def test_copy_db(self): c = MongoClient(host, port) # Due to SERVER-2329, databases may not disappear # from a master in a master-slave pair. if server_is_master_with_slave(c): raise SkipTest("SERVER-2329") if (not version.at_least(c, (2, 6, 0)) and is_mongos(c) and server_started_with_auth(c)): raise SkipTest("Need mongos >= 2.6.0 to test with authentication") # We test copy twice; once starting in a request and once not. In # either case the copy should succeed (because it starts a request # internally) and should leave us in the same state as before the copy. c.start_request() self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.pymongo_test.test.insert({"foo": "bar"}) c.copy_database("pymongo_test", "pymongo_test1") # copy_database() didn't accidentally end the request self.assertTrue(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.end_request() self.assertFalse(c.in_request()) if not c.is_mongos: # On mongos, this would try to copydb from the mongos itself # to the mongod that's the target. The mongod would try to # begin a transaction on the mongos, which would fail. c.copy_database("pymongo_test", "pymongo_test2", "%s:%d" % (host, port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) # See SERVER-6427 for mongos if not is_mongos(c) and server_started_with_auth(c): c.drop_database("pymongo_test1") c.admin.add_user("admin", "password") c.admin.authenticate("admin", "password") try: c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) finally: # Cleanup remove_all_users(c.pymongo_test) c.admin.remove_user("admin") c.disconnect()
def test_copy_db(self): c = MongoClient(self.host, self.port) # We test copy twice; once starting in a request and once not. In # either case the copy should succeed (because it starts a request # internally) and should leave us in the same state as before the copy. c.start_request() self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.pymongo_test.test.insert({"foo": "bar"}) # Due to SERVER-2329, databases may not disappear from a master in a # master-slave pair if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") # copy_database() didn't accidentally end the request self.assertTrue(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.end_request() self.assertFalse(c.in_request()) c.copy_database("pymongo_test", "pymongo_test2", "%s:%d" % (self.host, self.port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) if version.at_least(c, (1, 3, 3, 1)): c.drop_database("pymongo_test1") c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) if not is_mongos(c): # See SERVER-6427 c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
def setUp(self): # drop the database before we run the test client = MongoClient() client.drop_database(settings.MONGODB_DATABASE)
if __name__ == '__main__': return root_logger = logging.getLogger('') root_logger.setLevel(logging.INFO) logFormat = '%(asctime)s %(name)-10s %(levelname)-6s %(message)s' ch = logging.StreamHandler() ch.setFormatter(logging.Formatter(logFormat)) root_logger.addHandler(ch) fh = logging.FileHandler('extract.log', mode='w') fh.setFormatter(logging.Formatter(logFormat)) root_logger.addHandler(fh) logging.getLogger("requests").setLevel(logging.WARNING) #print parse_document('http://acropolis.org.uk/fb16889287d541ec85534ee7129b85c2.nq') #print parse_document('http://acropolis.org.uk/407c1c08ecde41159f1b1b6e91eeaf50.nq') #sys.exit(-1) client = MongoClient() client.drop_database('RESplorer') db = client['RESplorer'] for (name, uri) in CLASSES.iteritems(): logger.info('Processing {}'.format(name)) process_classes(db, uri) session_thread.close()
class MongoController: """ The main MongoDB controller class. Attributes: port - the port for the MongoDB service. temp_dir - the location of the MongoDB data and logs. client - a pymongo client pointed at the server. db_version - the version of the mongod executable. index_version - the version of the indexes created by the mongod executable - 1 for < 3.4.0, 2 otherwise. includes_system_indexes - true if system indexes will be included when listing database indexes, false otherwise. """ def __init__(self, mongoexe: Path, root_temp_dir: Path, use_wired_tiger: bool=False) -> None: ''' Create and start a new MongoDB database. An unused port will be selected for the server. :param mongoexe: The path to the MongoDB server executable (e.g. mongod) to run. :param root_temp_dir: A temporary directory in which to store MongoDB data and log files. The files will be stored inside a child directory that is unique per invocation. :param use_wired_tiger: For MongoDB versions > 3.0, specify that the Wired Tiger storage engine should be used. Setting this to true for other versions will cause an error. ''' if not mongoexe or not os.access(mongoexe, os.X_OK): raise test_util.TestException('mongod executable path {} does not exist or is not executable.' .format(mongoexe)) if not root_temp_dir: raise ValueError('root_temp_dir is None') # make temp dirs root_temp_dir = root_temp_dir.absolute() os.makedirs(root_temp_dir, exist_ok=True) self.temp_dir = Path(tempfile.mkdtemp(prefix='MongoController-', dir=str(root_temp_dir))) data_dir = self.temp_dir.joinpath('data') os.makedirs(data_dir) self.port = test_util.find_free_port() command = [str(mongoexe), '--port', str(self.port), '--dbpath', str(data_dir), '--nojournal'] if use_wired_tiger: command.extend(['--storageEngine', 'wiredTiger']) self._outfile = open(self.temp_dir.joinpath('mongo.log'), 'w') self._proc = subprocess.Popen(command, stdout=self._outfile, stderr=subprocess.STDOUT) time.sleep(1) # wait for server to start up self.client = MongoClient('localhost', self.port) # check that the server is up. See # https://api.mongodb.com/python/3.7.0/api/pymongo/mongo_client.html # #pymongo.mongo_client.MongoClient self.client.admin.command('ismaster') # get some info about the db self.db_version = self.client.server_info()['version'] self.index_version = 2 if (semver.compare(self.db_version, '3.4.0') >= 0) else 1 self.includes_system_indexes = (semver.compare(self.db_version, '3.2.0') < 0 and not use_wired_tiger) def destroy(self, delete_temp_files: bool) -> None: """ Shut down the MongoDB server. :param delete_temp_files: delete all the MongoDB data files and logs generated during the test. """ if self.client: self.client.close() if self._proc: self._proc.terminate() if self._outfile: self._outfile.close() if delete_temp_files and self.temp_dir: shutil.rmtree(self.temp_dir) def clear_database(self, db_name, drop_indexes=False): ''' Remove all data from a database. :param db_name: the name of the db to clear. :param drop_indexes: drop all indexes if true, retain indexes (which will be empty) if false. ''' if drop_indexes: self.client.drop_database(db_name) else: db = self.client[db_name] for name in db.list_collection_names(): if not name.startswith('system.'): # don't drop collection since that drops indexes db.get_collection(name).delete_many({})
from dataclasses import asdict from app.models.domain.training_data_set import TrainingDataSet from app.models.domain.sensor import Sensor from bson.objectid import ObjectId from pymongo.mongo_client import MongoClient from app.core.config import DATABASE_NAME, WORKSPACE_COLLECTION_NAME from app.models.domain.workspace import Workspace uri = 'mongodb://0.0.0.0/' client = MongoClient(uri, 27017) client.drop_database(DATABASE_NAME) db = client[DATABASE_NAME] coll = db[WORKSPACE_COLLECTION_NAME] test_workspace = asdict( Workspace( _id=ObjectId("666f6f2d6261722d71757578"), user_id=ObjectId("666f6f2d6261722d71757578"), sensors={ "Accelerometer": Sensor(sampling_rate=50, components=[ "x_Accelerometer", "y_Accelerometer", "z_Accelerometer" ]), "Gyroscope": Sensor(sampling_rate=50, components=["x_Gyroscope", "y_Gyroscope", "z_Gyroscope"]) }, training_data_set=TrainingDataSet(last_modified=1),