def test_unquote_after_parsing(self): quoted_val = "val%21%40%23%24%25%5E%26%2A%28%29_%2B%2C%3A+etc" unquoted_val = "val!@#$%^&*()_+,: etc" uri = ("mongodb://*****:*****@localhost/?authMechanism=MONGODB-AWS" "&authMechanismProperties=AWS_SESSION_TOKEN:" + quoted_val) res = parse_uri(uri) options = { 'authmechanism': 'MONGODB-AWS', 'authmechanismproperties': { 'AWS_SESSION_TOKEN': unquoted_val } } self.assertEqual(options, res['options']) uri = (("mongodb://localhost/foo?readpreference=secondary&" "readpreferencetags=dc:west," + quoted_val + ":" + quoted_val + "&" "readpreferencetags=dc:east,use:" + quoted_val)) res = parse_uri(uri) options = { 'readpreference': ReadPreference.SECONDARY.mongos_mode, 'readpreferencetags': [{ 'dc': 'west', unquoted_val: unquoted_val }, { 'dc': 'east', 'use': unquoted_val }] } self.assertEqual(options, res['options'])
def test_parse_ssl_paths(self): # Turn off "validate" since these paths don't exist on filesystem. self.assertEqual( { 'collection': None, 'database': None, 'nodelist': [('/MongoDB.sock', None)], 'options': { 'ssl_certfile': '/a/b' }, 'password': '******', 'username': '******' }, parse_uri( 'mongodb://*****:*****@%2FMongoDB.sock/?ssl_certfile=/a/b', validate=False)) self.assertEqual( { 'collection': None, 'database': None, 'nodelist': [('/MongoDB.sock', None)], 'options': { 'ssl_certfile': 'a/b' }, 'password': '******', 'username': '******' }, parse_uri( 'mongodb://*****:*****@%2FMongoDB.sock/?ssl_certfile=a/b', validate=False))
def test_waitQueueMultiple_deprecated(self): uri = "mongodb://example.com/?waitQueueMultiple=5" with warnings.catch_warnings(record=True) as ctx: warnings.simplefilter('always') parse_uri(uri) self.assertEqual(len(ctx), 1) self.assertTrue(issubclass(ctx[0].category, DeprecationWarning))
def test_redact_AWS_SESSION_TOKEN(self): unquoted_colon = "token:" uri = ("mongodb://*****:*****@localhost/?authMechanism=MONGODB-AWS" "&authMechanismProperties=AWS_SESSION_TOKEN:"+unquoted_colon) with self.assertRaisesRegex( ValueError, 'auth mechanism properties must be key:value pairs like ' 'SERVICE_NAME:mongodb, not AWS_SESSION_TOKEN:<redacted token>' ', did you forget to percent-escape the token with ' 'quote_plus?'): parse_uri(uri)
def __init__(self, db_config): self.config = db_config self.force_quit = False # sanitize the options if self.config["target_collections"] is not None: self.config["target_collections"] = set( [coll.strip() for coll in self.config["target_collections"]]) if 'auto_config' in self.config and self.config['auto_config'] is True: if 'auth_db' not in self.config['auto_config_options']: try: self.config['auto_config_options']['auth_db'] = self.config['auth_db'] except Exception: pass if 'user' not in self.config['auto_config_options']: try: self.config['auto_config_options']['user'] = self.config['user'] except Exception: pass if 'password' not in self.config['auto_config_options']: try: self.config['auto_config_options']['password'] = self.config['password'] except Exception: pass self.get_topology(self.config['auto_config_options']) oplog_servers = self.build_oplog_servers(self.config['auto_config_options']) profiler_servers = self.build_profiler_servers(self.config['auto_config_options']) else: oplog_servers = self.config["oplog_servers"] profiler_servers = self.config["profiler_servers"] if len(oplog_servers) < 1 or len(profiler_servers) < 1: utils.log.error("Detected either no profile or oplog servers, bailing") sys.exit(1) self.oplog_clients = {} for index, server in enumerate(oplog_servers): mongodb_uri = server['mongodb_uri'] nodelist = uri_parser.parse_uri(mongodb_uri)["nodelist"] server_string = "%s:%s" % (nodelist[0][0], nodelist[0][1]) self.oplog_clients[server_string] = self.connect_mongo(server) utils.LOG.info("oplog server %d: %s", index, self.sanatize_server(server)) # create a mongo client for each profiler server self.profiler_clients = {} for index, server in enumerate(profiler_servers): mongodb_uri = server['mongodb_uri'] nodelist = uri_parser.parse_uri(mongodb_uri)["nodelist"] server_string = "%s:%s" % (nodelist[0][0], nodelist[0][1]) self.profiler_clients[server_string] = self.connect_mongo(server) utils.LOG.info("profiling server %d: %s", index, self.sanatize_server(server))
def validate(self, clean=True): if (Utility.check_empty_string(self.type) or Utility.check_empty_string(self.url) or Utility.check_empty_string(self.db)): raise ValidationError( "Type, Url and DB cannot be blank or empty spaces") else: if self.type == "mongo": try: parse_uri(self.url) except InvalidURI: raise AppException("Invalid tracker url!")
def test_parse_tls_insecure_options(self): # tlsInsecure is expanded correctly. uri = "mongodb://example.com/?tlsInsecure=true" res = get_validated_options( { "ssl_match_hostname": False, "ssl_cert_reqs": ssl.CERT_NONE, "tlsinsecure": True }, warn=False) self.assertEqual(res, parse_uri(uri)["options"]) # tlsAllow* specified AFTER tlsInsecure. # tlsAllow* options warns and overrides values implied by tlsInsecure. uri = ("mongodb://example.com/?tlsInsecure=true" "&tlsAllowInvalidCertificates=false" "&tlsAllowInvalidHostnames=false") res = get_validated_options( { "ssl_match_hostname": True, "ssl_cert_reqs": ssl.CERT_REQUIRED, "tlsinsecure": True }, warn=False) with warnings.catch_warnings(record=True) as ctx: warnings.simplefilter('always') self.assertEqual(res, parse_uri(uri)["options"]) for warning in ctx: self.assertRegexpMatches( warning.message.args[0], ".*tlsAllowInvalid.*overrides.*tlsInsecure.*") clear_warning_registry() # tlsAllow* specified BEFORE tlsInsecure. # tlsAllow* options warns and overrides values implied by tlsInsecure. uri = ("mongodb://example.com/" "?tlsAllowInvalidCertificates=false" "&tlsAllowInvalidHostnames=false" "&tlsInsecure=true") res = get_validated_options( { "ssl_match_hostname": True, "ssl_cert_reqs": ssl.CERT_REQUIRED, "tlsinsecure": True }, warn=False) with warnings.catch_warnings(record=True) as ctx: warnings.simplefilter('always') self.assertEqual(res, parse_uri(uri)["options"]) for warning in ctx: self.assertRegexpMatches( warning.message.args[0], ".*tlsAllowInvalid.*overrides.*tlsInsecure.*")
def run_test(self): if not _HAVE_DNSPYTHON: raise unittest.SkipTest("DNS tests require the dnspython module") uri = test_case['uri'] seeds = test_case['seeds'] hosts = test_case['hosts'] options = test_case.get('options') if seeds: seeds = split_hosts(','.join(seeds)) if hosts: hosts = frozenset(split_hosts(','.join(hosts))) if options: for key, value in options.items(): # Convert numbers / booleans to strings for comparison if isinstance(value, bool): options[key] = 'true' if value else 'false' elif isinstance(value, (int, float)): options[key] = str(value) if seeds: result = parse_uri(uri, validate=False) self.assertEqual(sorted(result['nodelist']), sorted(seeds)) if options: opts = result['options'] if 'readpreferencetags' in opts: rpts = validate_read_preference_tags( 'readPreferenceTags', opts.pop('readpreferencetags')) opts['readPreferenceTags'] = rpts self.assertEqual(result['options'], options) hostname = next(iter(client_context.client.nodes))[0] # The replica set members must be configured as 'localhost'. if hostname == 'localhost': copts = client_context.default_client_options.copy() if client_context.ssl is True: # Our test certs don't support the SRV hosts used in these tests. copts['ssl_match_hostname'] = False client = MongoClient(uri, **copts) # Force server selection client.admin.command('ismaster') wait_until( lambda: hosts == client.nodes, 'match test hosts to client nodes') else: try: parse_uri(uri) except (ConfigurationError, ValueError): pass else: self.fail("failed to raise an exception")
def __init__(self, db_config): self.config = db_config self.force_quit = False # Sanitize the config options self.config["target_collections"] = set( [coll.strip() for coll in self.config.get("target_collections", [])]) if self.config.get('auto_config'): if 'auto_config_options' not in self.config: self.config['auto_config_options'] = {} if 'auth_db' not in self.config['auto_config_options'] and 'auth_db' in self.config: self.config['auto_config_options']['auth_db'] = self.config['auth_db'] if 'user' not in self.config['auto_config_options'] and 'user' in self.config: self.config['auto_config_options']['user'] = self.config['user'] if 'password' not in self.config['auto_config_options'] and 'password' in self.config: self.config['auto_config_options']['password'] = self.config['password'] self.get_topology(self.config['auto_config_options']) oplog_servers = self.build_oplog_servers(self.config['auto_config_options']) profiler_servers = self.build_profiler_servers(self.config['auto_config_options']) else: oplog_servers = self.config["oplog_servers"] profiler_servers = self.config["profiler_servers"] if len(oplog_servers) < 1 or len(profiler_servers) < 1: utils.log.error("Detected either no profile or oplog servers, bailing") sys.exit(1) # Connect to each MongoDB server that we want to get the oplog data from self.oplog_clients = {} for index, server in enumerate(oplog_servers): mongodb_uri = server['mongodb_uri'] nodelist = uri_parser.parse_uri(mongodb_uri)["nodelist"] server_string = "%s:%s" % (nodelist[0][0], nodelist[0][1]) self.oplog_clients[server_string] = self.connect_mongo(server) utils.LOG.info("oplog server %d: %s", index, self.sanitize_server(server)) # Connect to each MongoDB server that we want to get the profile data from self.profiler_clients = {} for index, server in enumerate(profiler_servers): mongodb_uri = server['mongodb_uri'] nodelist = uri_parser.parse_uri(mongodb_uri)["nodelist"] server_string = "%s:%s" % (nodelist[0][0], nodelist[0][1]) self.profiler_clients[server_string] = self.connect_mongo(server) utils.LOG.info("profiling server %d: %s", index, self.sanitize_server(server)) utils.LOG.info('Successfully connected to %d oplog server(s) and %d profiler server(s)', len(self.oplog_clients), len(self.profiler_clients))
def register_connection(alias, name=None, host=None, port=None, read_preference=READ_PREFERENCE, username=None, password=None, authentication_source=None, **kwargs): """Add a connection. :param alias: the name that will be used to refer to this connection throughout MongoEngine :param name: the name of the specific database to use :param host: the host name of the :program:`mongod` instance to connect to :param port: the port that the :program:`mongod` instance is running on :param read_preference: The read preference for the collection ** Added pymongo 2.1 :param username: username to authenticate with :param password: password to authenticate with :param authentication_source: database to authenticate against :param is_mock: explicitly use mongomock for this connection (can also be done by using `mongomock://` as db host prefix) :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver .. versionchanged:: 0.10.6 - added mongomock support """ global _connection_settings conn_settings = { 'name': name or 'test', 'host': host or 'localhost', 'port': port or 27017, 'read_preference': read_preference, 'username': username, 'password': password, 'authentication_source': authentication_source } # Handle uri style connections conn_host = conn_settings['host'] if conn_host.startswith('mongomock://'): conn_settings['is_mock'] = True # `mongomock://` is not a valid url prefix and must be replaced by `mongodb://` conn_settings['host'] = conn_host.replace('mongomock://', 'mongodb://', 1) elif '://' in conn_host: uri_dict = uri_parser.parse_uri(conn_host) conn_settings.update({ 'name': uri_dict.get('database') or name, 'username': uri_dict.get('username'), 'password': uri_dict.get('password'), 'read_preference': read_preference, }) uri_options = uri_dict['options'] if 'replicaset' in uri_options: conn_settings['replicaSet'] = True if 'authsource' in uri_options: conn_settings['authentication_source'] = uri_options['authsource'] # Deprecated parameters that should not be passed on kwargs.pop('slaves', None) kwargs.pop('is_slave', None) conn_settings.update(kwargs) _connection_settings[alias] = conn_settings
def get_mongodb(): db_inst = getattr(g, 'mongodb', None) if not db_inst: mongo_uri = current_app.config.get('MONGO_URI', None) parsed = uri_parser.parse_uri(mongo_uri) db_inst = g.mongodb = MongoClient(mongo_uri)[parsed.get('database')] return db_inst
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params): NamespaceManager.__init__(self, namespace) if lock_dir: self.lock_dir = lock_dir elif data_dir: self.lock_dir = data_dir + "/container_mongodb_gridfs_lock" if self.lock_dir: verify_directory(self.lock_dir) if not url: raise MissingCacheParameter("MongoDB url is required") for k, v in parse_uri(url).iteritems(): setattr(self, "url_%s" % k, v) if not self.url_database or not self.url_nodelist: raise MissingCacheParameter("Invalid MongoDB url.") data_key = "mongodb_gridfs:%s:%s" % (self.url_database, self.url_collection) self.gridfs = MongoDBGridFSNamespaceManager.clients.get( data_key, self._create_mongo_connection)
def _create_connection(conn_settings, testing=False): # Handle multiple connections recursively if isinstance(conn_settings, list): connections = {} for conn in conn_settings: connections[conn.get('alias')] = _create_connection(conn, testing) return connections # Ugly dict comprehention in order to support python 2.6 conn = dict((k.lower(), v) for k, v in conn_settings.items() if v is not None) if 'replicaset' in conn: conn['replicaSet'] = conn.pop('replicaset') if (StrictVersion(mongoengine.__version__) >= StrictVersion('0.10.6') and testing and conn.get('host', '').startswith('mongomock://')): pass # Handle uri style connections elif "://" in conn.get('host', ''): uri_dict = uri_parser.parse_uri(conn['host']) conn['db'] = uri_dict['database'] if conn['db'] is None: raise ValueError('Mongo host URI must contain database name') return mongoengine.connect(conn.pop('db', 'test'), **conn)
def __init__(self, db_url, collection_name): client = MongoClient(db_url) parsed = uri_parser.parse_uri(db_url) if 'database' not in parsed: raise ValueError('database not in uri: {}', db_url) db = client[parsed['database']] self.c = db[collection_name]
def __init__(self, uri='mongodb://127.0.0.1:27017', pool_size=1, ssl_context_factory=None): assert isinstance(uri, basestring) assert isinstance(pool_size, int) assert pool_size >= 1 if not uri.startswith('mongodb://'): uri = 'mongodb://' + uri self.cred_cache = {} self.__uri = parse_uri(uri) self.__pool_size = pool_size self.__pool = [ _Connection(self, self.__uri, i) for i in xrange(pool_size) ] host, port = self.__uri['nodelist'][0] for factory in self.__pool: if ssl_context_factory: factory.connector = reactor.connectSSL(host, port, factory, ssl_context_factory) else: factory.connector = reactor.connectTCP(host, port, factory)
def run_test(self): if not _HAVE_DNSPYTHON: raise unittest.SkipTest("DNS tests require the dnspython module") uri = test_case['uri'] seeds = test_case['seeds'] hosts = test_case['hosts'] options = test_case.get('options') if seeds: seeds = split_hosts(','.join(seeds)) if hosts: hosts = frozenset(split_hosts(','.join(hosts))) if options: for key, value in options.items(): # Convert numbers to strings for comparison options[key] = str(value) if seeds: result = parse_uri(uri, validate=False) self.assertEqual(sorted(result['nodelist']), sorted(seeds)) if options: self.assertEqual(result['options'], options) hostname = next(iter(client_context.client.nodes))[0] # The replica set members must be configured as 'localhost'. if hostname == 'localhost': client = MongoClient(uri, **_SSL_OPTS) # Force server selection client.admin.command('ismaster') wait_until( lambda: hosts == client.nodes, 'match test hosts to client nodes') else: self.assertRaises( ConfigurationError, parse_uri, uri, validate=False)
def open(url=None, task=None): from mongo_util import get_collection query = son.SON(json.loads(url, object_hook=json_util.object_hook)) uri = query['inputURI'] uri_info = uri_parser.parse_uri(uri) spec = query['query'] fields = query['fields'] skip = query['skip'] limit = query['limit'] timeout = query['timeout'] sort = query['sort'] slave_ok = query['slave_ok'] #go around: connect to the sonnection then choose db by ['dbname'] collection = get_collection(uri) cursor = collection.find(spec=spec, fields=fields, skip=skip, limit=limit, sort=sort, timeout=timeout, slave_okay=slave_ok) wrapper = MongoWrapper(cursor) return wrapper
def init_db(mongo_uri): global _conn global _db dbname = uri_parser.parse_uri(mongo_uri)['database'] _conn = MongoClient(host=mongo_uri) _conn.register([model.Instance]) _db = _conn[dbname]
def connect(mongodb_uri, alias=DEFAULT_CONNECTION_ALIAS): """Register a connection to MongoDB, optionally providing a name for it. :parameters: - `mongodb_uri`: A MongoDB connection string. Any options may be passed within the string that are supported by PyMongo. `mongodb_uri` must specify a database, which will be used by any :class:`~pymodm.MongoModel` that uses this connection. - `alias`: An optional name for this connection, backed by a :class:`~pymongo.mongo_client.MongoClient` instance that is cached under this name. You can specify what connection a MongoModel uses by specifying the connection's alias via the `connection_alias` attribute inside their `Meta` class. Switching connections is also possible using the :class:`~pymodm.context_managers.switch_connection` context manager. Note that calling `connect()` multiple times with the same alias will replace any previous connections. """ # Make sure the database is provided. parsed_uri = uri_parser.parse_uri(mongodb_uri) if not parsed_uri.get('database'): raise ValueError('Connection must specify a database.') _CONNECTIONS[alias] = ConnectionInfo( parsed_uri=parsed_uri, conn_string=mongodb_uri, database=MongoClient(mongodb_uri)[parsed_uri['database']])
def main(): # Connect to database Database try: parser = uri_parser.parse_uri(os.environ['MONGODB_URI']) db = MongoClient(os.environ['MONGODB_URI'])[parser['database']] except Exception as e: print "Unable to connected to database" raise e settings = { "template_path": os.path.join(os.path.dirname(__file__), "templates"), "static_path": os.path.join(os.path.dirname(__file__), "static"), "db": db, "debug": True } app = web.Application([ (r'/', IndexHandler), (r'/api/v1/landfills', LandfillHandler), ], **settings) port = int(os.environ.get("PORT", 5000)) print "Listening on port: %s" % (port) app.listen(port) ioloop.IOLoop.instance().start()
def get_mongo_uri(dasconfig): """ read dasconfig and return mongodb host and port (as dict) """ # use debug=False to suppress printouts about DAS config uri = dasconfig['mongodb']['dburi'][0] parsed_uri = parse_uri(uri) host, port = parsed_uri['nodelist'][0] return dict(mongo_host=host, mongo_port=port)
def __init__(self, uri="mongodb://127.0.0.1:27017", pool_size=1, ssl_context_factory=None, **kwargs): assert isinstance(uri, basestring) assert isinstance(pool_size, int) assert pool_size >= 1 if not uri.startswith("mongodb://"): uri = "mongodb://" + uri self.__uri = parse_uri(uri) wc_options = self.__uri['options'].copy() wc_options.update(kwargs) wc_options = dict((k, v) for k, v in wc_options.items() if k in self.__wc_possible_options) self.__write_concern = WriteConcern(**wc_options) self.__pool_size = pool_size self.__pool = [_Connection(self, self.__uri, i) for i in range(pool_size)] if self.__uri['database'] and self.__uri['username'] and self.__uri['password']: self.authenticate(self.__uri['database'], self.__uri['username'], self.__uri['password'], self.__uri['options'].get('authmechanism', 'DEFAULT')) host, port = self.__uri['nodelist'][0] for factory in self.__pool: if ssl_context_factory: factory.connector = reactor.connectSSL(host, port, factory, ssl_context_factory) else: factory.connector = reactor.connectTCP(host, port, factory)
def __init__(self, uri: str, loop: asyncio.AbstractEventLoop, check_primary_period: int = 1): self._check_primary_period = check_primary_period uri_info = parse_uri(uri=uri) self.nodes = [{ 'host': node[0], 'port': node[1], } for node in uri_info['nodelist']] self.options = ClientOptions( uri_info['username'], uri_info['password'], uri_info['database'], uri_info['options'] ) self.loop = loop self._pools = {} self._primary_pool = None self.__default_database_name = uri_info['database'] self._check_primary_tasks = []
def init_db(mongo_uri): global _conn global _db dbname = uri_parser.parse_uri(mongo_uri)["database"] _conn = MongoClient(host=mongo_uri) _conn.register([model.Instance]) _db = _conn[dbname]
def copy_database_to(source_database, destination_database, private_config_xml_file, dump_dir): mongo_params = parse_uri( get_mongo_uri_for_eva_profile("production", private_config_xml_file)) # nodelist is in format: [(host1,port1), (host2,port2)]. Just choose one. # Mongo is smart enough to fallback to secondaries automatically. mongo_host = mongo_params["nodelist"][0][0] logger.info("Beginning data copy for: " + source_database) dump_output_dir = "{0}/dump_{1}".format(dump_dir, source_database.replace(".", "_")) mongodump_args = { "db": source_database, "host": mongo_host, "username": mongo_params["username"], "password": mongo_params["password"], "authenticationDatabase": "admin", "out": dump_output_dir } mongorestore_args = { "db": destination_database, "host": mongo_host, "username": mongo_params["username"], "password": mongo_params["password"], "dir": "{0}/{1}/".format(dump_output_dir, source_database) } logger.info("Running export to {0} against {2} in production".format( dump_output_dir, source_database)) copy_db(mongodump_args, mongorestore_args)
def __init__(self, db_config): self.config = db_config self.force_quit = False # sanitize the options if self.config["target_collections"] is not None: self.config["target_collections"] = set( [coll.strip() for coll in self.config["target_collections"]]) oplog_server = self.config["oplog_server"] profiler_servers = self.config["profiler_servers"] mongodb_uri = oplog_server["mongodb_uri"] self.oplog_client = MongoClient(mongodb_uri) # create a mongo client for each profiler server self.profiler_clients = {} for index, server in enumerate(profiler_servers): mongodb_uri = server['mongodb_uri'] nodelist = uri_parser.parse_uri(mongodb_uri)["nodelist"] server_string = "%s:%s" % (nodelist[0][0], nodelist[0][1]) self.profiler_clients[server_string] = MongoClient(mongodb_uri, slaveOk=True) utils.LOG.info("profiling server %d: %s", index, str(server)) utils.LOG.info("oplog server: %s", str(oplog_server))
def get_collection(uri): uri_info = uri_parser.parse_uri(uri) col = uri_info["collection"] database = get_database(uri) return database[col]
def _create_connection(conn_settings, testing=False): # Handle multiple connections recursively if isinstance(conn_settings, list): connections = {} for conn in conn_settings: connections[conn.get('alias')] = _create_connection(conn, testing) return connections # Ugly dict comprehention in order to support python 2.6 conn = dict( (k.lower(), v) for k, v in conn_settings.items() if v is not None) if 'replicaset' in conn: conn['replicaSet'] = conn.pop('replicaset') if (StrictVersion(mongoengine.__version__) >= StrictVersion('0.10.6') and testing and conn.get('host', '').startswith('mongomock://')): pass # Handle uri style connections elif "://" in conn.get('host', ''): uri_dict = uri_parser.parse_uri(conn['host']) conn['db'] = uri_dict['database'] if conn['db'] is None: raise ValueError('Mongo host URI must contain database name') return mongoengine.connect(conn.pop('db', 'test'), **conn)
def __init__(self, location, params, mongodb=None): """ @type mongodb: инстанс подключения к монго """ self.collection_name = params['collection'] options = params.get('OPTIONS', {}) self.write_concern = options.get('WRITE_CONCERN', 1) self.json_serializeable_values = options.get( 'VALUES_ARE_JSON_SERIALIZEABLE', True) strategy = options.get('STRATEGY', 'NEAREST') assert self.write_concern > -1 assert isinstance(self.collection_name, basestring) if not location.startswith('mongodb://'): raise ImproperlyConfigured('connection to mongo should start with mongodb://') database = uri_parser.parse_uri(location)['database'] if not database: raise ImproperlyConfigured('Specify DB like that mongodb://hosts/database_name') self.mongodb = mongodb or MongoDBWrapper( hosts=location, strategy=strategy, replica_set=options['replica_set'], database_name=database ) self.logger = logging.getLogger('mongo_requests') super(MongoDBCache, self).__init__(params) self._ensure_ttl_collection()
def __init__(self, email: None, mongo_uri_std_cits=None): self.email = email if mongo_uri_std_cits: try: self.persist_mode = 'mongo' mongo_col = uri_parser.parse_uri(mongo_uri_std_cits).get( 'collection') if not mongo_col: mongo_col = MONGO_STDCITS_COLLECTION self.standardizer = MongoClient( mongo_uri_std_cits).get_database().get_collection( mongo_col) total_docs = self.standardizer.count_documents({}) logging.info( 'There are {0} documents in the collection {1}'.format( total_docs, mongo_col)) except ConnectionError as e: logging.error('ConnectionError %s' % mongo_uri_std_cits) logging.error(e) else: self.persist_mode = 'json' file_name_results = 'crossref-results-' + str( time.time()) + '.json' self.path_results = os.path.join(DIR_DATA, file_name_results)
def _create_connection(conn_settings): # Handle multiple connections recursively if isinstance(conn_settings, list): connections = {} for conn in conn_settings: connections[conn.get('alias')] = _create_connection(conn) return connections # Ugly dict comprehention in order to support python 2.6 conn = dict((k.lower(), v) for k, v in conn_settings.items() if v is not None) if 'replicaset' in conn: conn['replicaSet'] = conn.pop('replicaset') if (StrictVersion(mongoengine.__version__) >= StrictVersion('0.10.6') and current_app.config['TESTING'] == True and conn.get('host', '').startswith('mongomock://')): pass # Handle uri style connections elif "://" in conn.get('host', ''): uri_dict = uri_parser.parse_uri(conn['host']) conn['db'] = uri_dict['database'] return mongoengine.connect(conn.pop('db', 'test'), **conn)
def register_connection( alias, name, host="localhost", port=27017, is_slave=False, read_preference=False, slaves=None, username=None, password=None, **kwargs ): """Add a connection. :param alias: the name that will be used to refer to this connection throughout MongoEngine :param name: the name of the specific database to use :param host: the host name of the :program:`mongod` instance to connect to :param port: the port that the :program:`mongod` instance is running on :param is_slave: whether the connection can act as a slave ** Depreciated pymongo 2.0.1+ :param read_preference: The read preference for the collection ** Added pymongo 2.1 :param slaves: a list of aliases of slave connections; each of these must be a registered connection that has :attr:`is_slave` set to ``True`` :param username: username to authenticate with :param password: password to authenticate with :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver """ global _connection_settings conn_settings = { "name": name, "host": host, "port": port, "is_slave": is_slave, "slaves": slaves or [], "username": username, "password": password, "read_preference": read_preference, } # Handle uri style connections if "://" in host: uri_dict = uri_parser.parse_uri(host) conn_settings.update( { "host": host, "name": uri_dict.get("database") or name, "username": uri_dict.get("username"), "password": uri_dict.get("password"), "read_preference": read_preference, } ) if "replicaSet" in host: conn_settings["replicaSet"] = True conn_settings.update(kwargs) _connection_settings[alias] = conn_settings
def _get_db(): """ Returns the connection to the database using the settings. This function should not be called outside of this file. Use db instead. """ from .settings import settings mongo = settings.MONGODB if 'URI' in mongo and mongo['URI']: uri = mongo['URI'] else: uri = 'mongodb://' if all(mongo.get(key) for key in ('USERNAME', 'PASSWORD')): uri += '{0}:{1}@'.format(mongo['USERNAME'], mongo['PASSWORD']) if 'HOSTS' in mongo and mongo['HOSTS']: uri += ','.join( '{0}:{1}'.format(host, port) for (host, port) in zip(mongo['HOSTS'], mongo['PORTS']), ) else: uri += '{0}:{1}'.format(mongo['HOST'], mongo.get('PORT', 27017)) uri += '/' + mongo['DATABASE'] if 'OPTIONS' in mongo and mongo['OPTIONS']: uri += '?{0}'.format('&'.join(mongo['OPTIONS'])) client = ConnectionFailureProxy(MongoClient(uri, connect=False)) database = client[parse_uri(uri)['database']] return database
def __init__(self, uri, db, keyword='mongodb', json_mongo=False, post_create=None, **kwargs): if not uri: raise PluginError("MongoDB uri is required") self.uri_params = parse_uri(uri) if not len(self.uri_params['nodelist']): raise PluginError("MongoDB hostname and port not configured") self.uri_params['database'] = self.uri_params['database'] or db if not self.uri_params['database']: raise PluginError("MongoDB database name not configured") self.uri_params['options'].update(kwargs) self.keyword = keyword self.json_mongo = json_mongo self.mongo_db = None self.name = "mongo:" + keyword self.post_create = post_create
def test_parse_uri_unicode(self): # Ensure parsing a unicode returns option names that can be passed # as kwargs. In Python 2.4, keyword argument names must be ASCII. # In all Pythons, str is the type of valid keyword arg names. res = parse_uri(unicode("mongodb://localhost/?fsync=true")) for key in res['options']: self.assertTrue(isinstance(key, str))
def db_connect(): """ connect to the database :return:database connection object """ from pymongo import Connection, uri_parser # Connect to the database if 'MONGO_URL' not in os.environ: print "[***] Missing 'MONGO_URL' Environment Variable [***]" raise EnvironmentError mongo_options = uri_parser.parse_uri(os.environ['MONGO_URL']) (host, port) = mongo_options['nodelist'][0] ssl = mongo_options['options'].get('ssl', False) print "[+] Attempting connection to database '{0}:{1}/{2}'".format( host, str(port), mongo_options['database'] ) conn = Connection(host, port, ssl=ssl) db = conn[mongo_options['database']] if mongo_options['username'] or mongo_options['password']: db.authenticate(mongo_options['username'], mongo_options['password']) print "[+] Connection successful." return db
def DB(uri): global _db_set if uri not in _db_set: _client = MongoClient(uri) _db_set[uri] = _client[uri_parser.parse_uri(uri)['database']] return _db_set[uri]
def test_tlsinsecure_simple(self): # check that tlsInsecure is expanded correctly. uri = "mongodb://example.com/?tlsInsecure=true" res = { "ssl_match_hostname": False, "ssl_cert_reqs": ssl.CERT_NONE, "tlsinsecure": True} self.assertEqual(res, parse_uri(uri)["options"])
def test_parse_uri_unicode(self): # Ensure parsing a unicode returns option names that can be passed # as kwargs. In Python 2.4, keyword argument names must be ASCII. # In all Pythons, str is the type of valid keyword arg names. res = parse_uri(_unicode("mongodb://localhost/?fsync=true")) for key in res['options']: self.assertTrue(isinstance(key, str))
def __init__(self, uri: str, collection_name: str, database: str = None, **kwargs): """ Args: uri: MongoDB+SRV URI database: database to connect to collection_name: The collection name """ self.uri = uri # parse the dbname from the uri if database is None: d_uri = uri_parser.parse_uri(uri) if d_uri["database"] is None: raise ConfigurationError( "If database name is not supplied, a database must be set in the uri" ) self.database = d_uri["database"] else: self.database = database self.collection_name = collection_name self.kwargs = kwargs self._collection = None super(MongoStore, self).__init__(**kwargs) # lgtm
def _initialize_(self, do_connect): super(Mongo, self)._initialize_(do_connect) #: uri parse from pymongo import uri_parser m = uri_parser.parse_uri(self.uri) if isinstance(m, tuple): m = {"database": m[1]} if m.get('database') is None: raise SyntaxError("Database is required!") self._driver_db = m['database'] #: mongodb imports and utils from bson.objectid import ObjectId from bson.son import SON from pymongo.write_concern import WriteConcern self.epoch = datetime.fromtimestamp(0) self.SON = SON self.ObjectId = ObjectId self.WriteConcern = WriteConcern #: options self.db_codec = 'UTF-8' # this is the minimum amount of replicates that it should wait # for on insert/update self.minimumreplication = self.adapter_args.get( 'minimumreplication', 0) # by default all inserts and selects are performed asynchronous, # but now the default is # synchronous, except when overruled by either this default or # function parameter self.safe = 1 if self.adapter_args.get('safe', True) else 0
def collect(self): server = self.config.get('server') if server == None: self.error("Missing 'server' in mongo config") return parsed = uri_parser.parse_uri(server) username = parsed.get('username') password = parsed.get('password') db_name = parsed.get('database') slowms = self.config.get('slowms', 25) if not db_name: self.log.info('No MongoDB database found in URI. Defaulting to admin.') db_name = 'admin' do_auth = True if username is None or password is None: self.log.debug("Mongo: cannot extract username and password from config %s" % server) do_auth = False try: self.conn = pymongo.Connection(server, network_timeout=self.DEFAULT_TIMEOUT) except Exception, e: self.error(e) return
def _initialize_(self, do_connect): super(Mongo, self)._initialize_(do_connect) #: uri parse from pymongo import uri_parser m = uri_parser.parse_uri(self.uri) if isinstance(m, tuple): m = {"database": m[1]} if m.get('database') is None: raise SyntaxError("Database is required!") self._driver_db = m['database'] #: mongodb imports and utils from bson.objectid import ObjectId from bson.son import SON from pymongo.write_concern import WriteConcern self.epoch = datetime.fromtimestamp(0) self.SON = SON self.ObjectId = ObjectId self.WriteConcern = WriteConcern #: options self.db_codec = 'UTF-8' # this is the minimum amount of replicates that it should wait # for on insert/update self.minimumreplication = self.adapter_args.get( 'minimumreplication', 0) # by default all inserts and selects are performed asynchronous, # but now the default is # synchronous, except when overruled by either this default or # function parameter self.safe = 1 if self.adapter_args.get('safe', True) else 0 self._mock_reconnect()
def config_celery_for_mongo(settings): db_uri = settings['mongodb.uri'].strip('"\'') db_name = settings['celery.dbname'].strip('"\'') res = uri_parser.parse_uri(db_uri) host, port = res['nodelist'][0] global _modules_to_register global _celery_routes #print 'collected tasks' #print _celery_routes.keys() celery_config = { 'CELERY_RESULT_BACKEND' : 'mongodb', 'BROKER_TRANSPORT' : 'mongodb', 'CELERY_IMPORTS': tuple(_modules_to_register), 'BROKER_HOST' : host, 'BROKER_PORT' : port, 'BROKER_VHOST' : db_name, 'CELERY_MONGODB_BACKEND_SETTINGS' : { 'host': host, 'port': port, 'database': db_name }, 'CELERY_DISABLE_RATE_LIMITS': True, 'CELERY_ROUTES': _celery_routes, 'CELERYD_POOL': 'gevent', } return celery_config
def _get_db(): """ Returns the connection to the database using the settings. This function should not be called outside of this file. Use db instead. """ from .settings import settings mongo = settings.MONGODB if 'URI' in mongo and mongo['URI']: uri = mongo['URI'] else: uri = 'mongodb://' if all(mongo.get(key) for key in ('USERNAME', 'PASSWORD')): uri += '{0}:{1}@'.format(mongo['USERNAME'], mongo['PASSWORD']) if 'HOSTS' in mongo and mongo['HOSTS']: uri += ','.join( '{0}:{1}'.format(host, port) for (host, port) in zip(mongo['HOSTS'], mongo['PORTS']), ) else: uri += '{0}:{1}'.format(mongo['HOST'], mongo.get('PORT', 27017)) uri += '/' + mongo['DATABASE'] if 'OPTIONS' in mongo and mongo['OPTIONS']: uri += '?{0}'.format('&'.join(mongo['OPTIONS'])) return MongoClient(uri)[parse_uri(uri)['database']]
def db_connect(): """ connect to the database :return:database connection object """ from pymongo import Connection, uri_parser # Connect to the database if 'MONGO_URL' not in os.environ: print "[***] Missing 'MONGO_URL' Environment Variable [***]" raise EnvironmentError mongo_options = uri_parser.parse_uri(os.environ['MONGO_URL']) (host, port) = mongo_options['nodelist'][0] ssl = mongo_options['options'].get('ssl', False) print "[+] Attempting connection to database '{0}:{1}/{2}'".format( host, str(port), mongo_options['database']) conn = Connection(host, port, ssl=ssl) db = conn[mongo_options['database']] if mongo_options['username'] or mongo_options['password']: db.authenticate(mongo_options['username'], mongo_options['password']) print "[+] Connection successful." return db
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, skip_pickle=False, sparse_collection=False, **params): NamespaceManager.__init__(self, namespace) if not url: raise MissingCacheParameter("MongoDB url is required") if skip_pickle: log.info("Disabling pickling for namespace: %s" % self.namespace) self._pickle = False if sparse_collection: log.info(("Separating data to one row per key (sparse collection)" " for ns %s .") % self.namespace) self._sparse = True # Temporarily uses a local copy of the functions until pymongo upgrades to new parser code url_components = parse_uri(url) host_list = url_components['nodelist'] username = url_components['username'] password = url_components['password'] collection = url_components['collection'] database = url_components['database'] options = url_components['options'] if database and host_list: data_key = "mongodb:%s" % (database) else: raise MissingCacheParameter("Invalid Cache URL. Cannot parse.") # Key will be db + collection if lock_dir: self.lock_dir = lock_dir elif data_dir: self.lock_dir = data_dir + "/container_mongodb_lock" if hasattr(self, 'lock_dir'): verify_directory(self.lock_dir) def _create_mongo_conn(): host_uri = 'mongodb://' for x in host_list: host_uri += '%s:%s' % x log.info("Host URI: %s" % host_uri) conn = Connection(url, slave_okay=options.get('slaveok', False)) db = conn[database] if username: log.info("Attempting to authenticate %s/%s " % (username, password)) if not db.authenticate(username, password): raise InvalidCacheBackendError('Cannot authenticate to ' ' MongoDB.') return db[collection] self.mongo = MongoDBNamespaceManager.clients.get( data_key, _create_mongo_conn)
def __init__(self): uri = uri_parser.parse_uri(MONGODB_URL) client = MongoClient(uri['nodelist'][0][0], uri['nodelist'][0][1]) self.mongo_db = client[uri['database']] self.redis = redis.StrictRedis(decode_responses=True) self.google_api_key = GOOGLE_API_KEY self.raven_client = RavenClient(SENTRY_URL) self.raven_client.tags_context({'provider': self.provider_name})
def database(): mongodb_uri = uri_parser.parse_uri(environ['MONGOLAB_URI']) conn = Connection(*mongodb_uri['nodelist'][0]) db = conn[mongodb_uri['database']] db.authenticate(mongodb_uri['username'], mongodb_uri['password']) return db
def register_connection( alias, name=None, host=None, port=None, read_preference=False, username=None, password=None, authentication_source=None, **kwargs ): """Add a connection. :param alias: the name that will be used to refer to this connection throughout MongoEngine :param name: the name of the specific database to use :param host: the host name of the :program:`mongod` instance to connect to :param port: the port that the :program:`mongod` instance is running on :param read_preference: The read preference for the collection ** Added pymongo 2.1 :param username: username to authenticate with :param password: password to authenticate with :param authentication_source: database to authenticate against :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver """ global _connection_settings conn_settings = { "name": name or "test", "host": host or "localhost", "port": port or 27017, "read_preference": read_preference, "username": username, "password": password, "authentication_source": authentication_source, } # Handle uri style connections if "://" in conn_settings["host"]: uri_dict = uri_parser.parse_uri(conn_settings["host"]) conn_settings.update( { "name": uri_dict.get("database") or name, "username": uri_dict.get("username"), "password": uri_dict.get("password"), "read_preference": read_preference, } ) if "replicaSet" in conn_settings["host"]: conn_settings["replicaSet"] = True # Deprecated parameters that should not be passed on kwargs.pop("slaves", None) kwargs.pop("is_slave", None) conn_settings.update(kwargs) _connection_settings[alias] = conn_settings
def _parseHosts(): """ Build a dict of all of the connections defined by DBHost Doesn't register a default connection yet. """ for k, v in DBHost.items(): parts = parse_uri(v["host"].replace("mongomock", "mongodb")) # hack for a parse_uri restriction DBHost[k]["db"] = parts["database"]
def get_database_name(self): """ extract database from connection string """ uri_dict = uri_parser.parse_uri(self.host) database = uri_dict.get('database', None) if not database: raise "database name is missing" return database
def mongo_from_uri(uri): config = parse_uri(uri) conn_settings = { 'db': config['database'], 'username': config['username'], 'password': config['password'], 'host': config['nodelist'][0][0], 'port': config['nodelist'][0][1] } return conn_settings