def mon_repl(self, host, port): ''' 监控repl集群,并时刻把拓扑结构回传给conf文件 ''' try: #初始化mongo链接,并获取副本状态 logging.info('connect repl start') conn = pymongo.MongoReplicaSetClient("%s:%s" % (host, port)) repl_status = conn.admin.command("replSetGetStatus") ReplSetName = repl_status['set'] logging.info('ReplSetName is %s' %(ReplSetName)) #解析所有成员状态 new_machine_list = '' for member in repl_status['members']: member_host, member_port = member['name'].split(':') if re.search(r"(reachable|healthy|RECOVERING)",member['stateStr']): #logging.error("%s:%s is offline" % (member_host, member_port)) push_data.push_data(member_host, member_port, 0) else: #logging.info("%s:%s is normal" % (member_host, member_port)) push_data.push_data(member_host, member_port, 1) new_machine_list += member['name'] + ";" new_machine_list = new_machine_list.strip(';') conf.set(self.section, 'mach_list', new_machine_list) conf.write(open(config_file, 'w')) except Exception, e: #截获链接异常 logging.critical(e)
def _get_db(self): if self.is_replica_set: db_connection = pymongo.MongoReplicaSetClient(self.configuration["connectionString"], replicaSet=self.configuration["replicaSetName"]) else: db_connection = pymongo.MongoClient(self.configuration["connectionString"]) return db_connection[self.db_name]
def add_user(user='******', manager=False, admin=False, mongo_options=None): token = str(uuid.uuid4()) password = '******' email = '*****@*****.**' db_body = { '_id': user, 'password': password, 'email': email, 'token': token } if not mongo_options: mdb = pymongo.MongoClient() else: if is_domain(mongo_options['host']): mongo_options['ssl'] = True if 'replicaSet' in mongo_options: mdb = pymongo.MongoReplicaSetClient(**mongo_options) mdb.read_preference = pymongo.ReadPreference.SECONDARY_PREFERRED else: mdb = pymongo.MongoClient(**mongo_options) mdb.users.all.insert(db_body) result = db_body result['user'] = user if manager: weight = 1 db_body = {'_id': user, 'weight': weight} mdb.users.managers.insert(db_body) result['weight'] = weight if admin: db_body = {'_id': user} mdb.users.admins.insert(db_body) return result
def test_auth_connection(self): self.assertTrue( isinstance(self.repl.connection().admin.collection_names(), list)) c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id) self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def __init__(self, collection="results"): """Initilize connection and check indeces""" try: # MongoDB URI uri = self.make_uri() # Database database = config["app_conf"]["mongo_db"] # Collection # Updated the mongo client connection for the collection # We could also use MongoReplicaSetClient for dealing with replica sets. replicate = config["app_conf"]["mongo_replicate"] if replicate == "true": self.collection = pymongo.MongoReplicaSetClient(host=uri,port=int(config["app_conf"]["mongo_port"]),replicaSet=(config["app_conf"]["mongo_replset"]))[database][collection] else: self.collection = pymongo.mongo_client.MongoClient(host=uri,port=int(config["app_conf"]["mongo_port"]))[database][collection] # Indecies self.ensure_index() except Exception as error: # Exception type: Exception message c.message = ": ".join([type(error).__name__, error.message])
def getDbConnection(uri=None, replicaSet=None): """ Get a MongoClient object that is connected to the configured database. We lazy-instantiate a module-level singleton, the MongoClient objects manage their own connection pools internally. :param uri: if specified, connect to this mongo db rather than the one in the config. :param replicaSet: if uri is specified, use this replica set. """ global _dbClients origKey = (uri, replicaSet) if origKey in _dbClients: return _dbClients[origKey] if uri is None or uri == '': dbConf = getDbConfig() uri = dbConf.get('uri') replicaSet = dbConf.get('replica_set') clientOptions = { 'connectTimeoutMS': 15000, # This is the maximum time between when we fetch data from a cursor. # If it times out, the cursor is lost and we can't reconnect. If it # isn't set, we have issues with replica sets when the primary goes # down. This value can be overridden in the mongodb uri connection # string with the socketTimeoutMS. 'socketTimeoutMS': 60000, } if uri is None: dbUriRedacted = 'mongodb://*****:*****@') if len(parts) == 2: dbUriRedacted = 'mongodb://' + parts[1] else: dbUriRedacted = uri if replicaSet: client = pymongo.MongoReplicaSetClient( uri, replicaSet=replicaSet, read_preference=ReadPreference.SECONDARY_PREFERRED, **clientOptions) else: client = pymongo.MongoClient(uri, **clientOptions) client = MongoProxy(client, logger=logger) _dbClients[origKey] = _dbClients[(uri, replicaSet)] = client desc = '' if replicaSet: desc += ', replica set: %s' % replicaSet print( TerminalColor.info('Connected to MongoDB: %s%s' % (dbUriRedacted, desc))) return client
def _initialize_collection(self): try: from gevent import monkey monkey.patch_socket() except ImportError: pass if self._rsname: self.connection = pymongo.MongoReplicaSetClient( self._rshosts, replicaSet=self._rsname, read_preference=getattr(pymongo.ReadPreference, self._read_preference, 'PRIMARY'), socketTimeoutMS=self._socket_timeout_ms, connectTimeoutMS=self._connect_timeout_ms, tag_sets=self._tag_sets) else: self.connection = pymongo.Connection(self._host, self._port) self._db = self.connection[self._database] if self._user and self._password: self._db.authenticate(self._user, self._password) self._coll = self._db[self._collection] #-----------------------------------------------------------------------------------------------------------------------
def connection(self, hostname=None, read_preference=pymongo.ReadPreference.PRIMARY, timeout=300): """return MongoReplicaSetClient object if hostname specified return MongoClient object if hostname doesn't specified Args: hostname - connection uri read_preference - default PRIMARY timeout - specify how long, in seconds, a command can take before server times out. """ logger.debug( "connection({hostname}, {read_preference}, {timeout})".format( **locals())) t_start = time.time() servers = hostname or ",".join(self.server_map.values()) while True: try: if hostname is None: c = pymongo.MongoReplicaSetClient( servers, replicaSet=self.repl_id, read_preference=read_preference, socketTimeoutMS=20000, **self.kwargs) if c.primary: try: self.login and self.password and c.admin.authenticate( self.login, self.password) except: pass return c raise pymongo.errors.AutoReconnect( "No replica set primary available") else: logger.debug( "connection to the {servers}".format(**locals())) c = pymongo.MongoClient(servers, socketTimeoutMS=20000, **self.kwargs) if self.login and self.password: try: c.admin.authenticate(self.login, self.password) except: pass return c except (pymongo.errors.PyMongoError): exc_type, exc_value, exc_tb = sys.exc_info() err_message = traceback.format_exception( exc_type, exc_value, exc_tb) logger.error( "Exception {exc_type} {exc_value}".format(**locals())) logger.error(err_message) if time.time() - t_start > timeout: raise pymongo.errors.AutoReconnect( "Couldn't connect while timeout {timeout} second". format(**locals())) time.sleep(10)
def get_conn(host_url, replica_set_name): try: mongoConn = pymongo.MongoReplicaSetClient( host=host_url, replicaSet=replica_set_name, read_preference=pymongo.ReadPreference.PRIMARY_PREFERRED) except AutoReconnect, e: print "init mongo connection failed.... " mongoConn = None
def __init__(self, uri, conn=None, dbname='eduid_api', **kwargs): if conn is not None: self.connection = conn else: if 'replicaSet=' in uri: self.connection = pymongo.MongoReplicaSetClient(uri, **kwargs) else: self.connection = pymongo.MongoClient(uri, **kwargs) self.db = self.connection[dbname]
def db(self): if self._database is None: if options.CFG.uri and 'replicaSet' in options.CFG.uri: conn = pymongo.MongoReplicaSetClient(options.CFG.uri) else: conn = pymongo.MongoClient(options.CFG.uri) self._database = conn[options.CFG.database] return self._database
def test_auth_admin(self): c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id) self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names) self.assertTrue(c.admin.authenticate('admin', 'admin')) self.assertTrue(isinstance(c.admin.collection_names(), list)) self.assertTrue(c.admin.logout() is None) self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def __init__(self, mongo_uri, dbname='helper', replset=0, pool_size=100): try: if replset: self.con = pymongo.MongoReplicaSetClient( mongo_uri, read_preference=pymongo.ReadPreference.SECONDARY_PREFERRED, max_pool_size=pool_size) else: self.con = pymongo.MongoClient(mongo_uri) self.db = self.con[dbname] except Exception as e: raise Exception('connect mongodb: %s, error: %s' % (mongo_uri, e))
def test_auth_collection(self): c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id) self.assertTrue(c.admin.authenticate('admin', 'admin')) db = c.test_auth db.add_user('user', 'userpass', roles=['readWrite']) c.admin.logout() self.assertTrue(db.authenticate('user', 'userpass')) self.assertTrue(db.foo.insert({'foo': 'bar'}, w=2, wtimeout=1000)) self.assertTrue(isinstance(db.foo.find_one(), dict)) db.logout() self.assertRaises(pymongo.errors.OperationFailure, db.foo.find_one)
def _mongo_connect(url): try: if cfg.CONF.database.mongodb_replica_set: client = MongoProxy( pymongo.MongoReplicaSetClient( url, replicaSet=cfg.CONF.database.mongodb_replica_set)) else: client = MongoProxy(pymongo.MongoClient(url, safe=True)) return client except pymongo.errors.ConnectionFailure as e: LOG.warn( _('Unable to connect to the database server: ' '%(errmsg)s.') % {'errmsg': e}) raise
def connect_db(db_uri, **kwargs): for x in range(0, 30): try: db_client = pymongo.MongoReplicaSetClient( db_uri, ** kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient( db_uri, **kwargs) except: time.sleep(1) pass else: break else: raise Exception("Could not connect to MongoDB") return db_client
def get_server_info(hostname="localhost", port="27017", replica_set=None): """ Get the mongod server build info and server status from the target mongod server :param hostname: the hostname the target database is running on (defaults to localhost) :param port: the port the target database is running on (defaults to 27017) :param replica_set: the replica set name the target database is using (defaults to none) :return: a tuple of the buildinfo and the server status """ if replica_set is None: client = pymongo.MongoClient("mongodb://%s:%s/test" % (hostname, port)) else: client = pymongo.MongoReplicaSetClient("mongodb://%s:%s/test?replicaSet=%s" % (hostname, port, replica_set)) db = client.test server_build_info = db.command("buildinfo") server_status = db.command("serverStatus") client.close() return server_build_info, server_status
def main(args): client = pymongo.MongoReplicaSetClient(URI) db = client.get_default_database() collection_name = _get_unique_tmp_collection_name() c = db[collection_name] try: while True: now = datetime.datetime.now() try: if client.primary: print("%s: connected to primary server '%s'..." % (now, client.primary[0] + ":" + str(client.primary[1]))) print("...about to perform safe insert.") c.insert({"t": now}) print("...about to read.") num_docs = c.find({}).count() print("...successfully inserted & read. There are %i " "documents in your temporary test collection." % num_docs) retry_total_time = 0 except Exception as e: print("*** EXCEPTION - %s" % e) print("*** ...will sleep for %s seconds before retrying " "[total of %s seconds]" % (RETRY_WAIT_IN_SEC, retry_total_time)) retry_total_time += RETRY_WAIT_IN_SEC print("\n") time.sleep(RETRY_WAIT_IN_SEC) except: print("An unexpected exception occurred. Performing cleanup " "if possible by dropping test collection '%s'..." % collection_name) c.drop() client.close()
def mongo_connection( mongo_host=None, mongo_port=None, mongo_replica_set=None, ): if mongo_replica_set is not None: with elle.log.log('connect to MongoDB replica set %s' % (mongo_replica_set, )): return pymongo.MongoReplicaSetClient(','.join(mongo_replica_set), replicaSet='fist-meta') else: with elle.log.log('connect to MongoDB on %s:%s' % (mongo_host, mongo_port)): db_args = {} if mongo_host is not None: db_args['host'] = mongo_host if mongo_port is not None: db_args['port'] = mongo_port return pymongo.MongoClient(**db_args)
def get_connection( host="127.0.0.1", port=27017, database="trunk", user="******", password="******", replica=None, **kwargs ): """Возвращает синхронное подключение к базе. :param host: Хост, к которому выполняется подключение :type host: str :param port: Порт базы данных :type port: int :param user: Имя пользователя базы данных :type user: str :param password: Пароль пользователя базы данных :type password: str :param replica: Название replicaSet (при наличии) :type replica: str :param database: Имя базы данных :type database: str """ if not replica: con = pymongo.MongoClient( "mongodb://{}:{}@{}:{}/{}".format( user, password, host, port, database )) else: con = pymongo.MongoReplicaSetClient( "mongodb://{}:{}@{}:{}/{}".format( user, password, host, port, database ), replicaSet=replica, connectTimeoutMS=1500, socketTimeoutMS=1500 ) return con[database]
def _get_db(self): # defer imports until backend is used global pymongo import pymongo if self.use_replica: connection = pymongo.MongoReplicaSetClient( host=self.hosts, replicaSet=self.replicaset_name, max_pool_size=self.max_pool_size, **self.conn_kwargs) else: # used for standalone node or mongos in sharded setup connection = pymongo.MongoClient( host=self.hosts, max_pool_size=self.max_pool_size, **self.conn_kwargs) database = getattr(connection, self.db_name) self._assign_data_mainpulator() database.add_son_manipulator(self._data_manipulator) if self.username and self.password: database.authenticate(self.username, self.password) return database
def __getattr__(self, item): if item in self.dbs and self._mongoclient is None: if mode == 'test': self._mongoclient = pymongo.MongoClient(host=self.localhost, port=self.port) else: self._mongoclient = pymongo.MongoReplicaSetClient( ','.join(mongo_machines), replicaSet=REPLICASET_NAME, ) #print "+++++++++authing++++++++++" self._mongoclient.fbt.authenticate(FBT_USER, FBT_PASSWD) #print "+++++++++authing OK++++++++++"+str(ok) self.dbs['fbt'] = self._mongoclient.fbt self.module.__dict__.update(self.dbs) return getattr(self.module, item)
def __init__(self, uri, logger, conn=None, db_name="eduid_api", retries=10, **kwargs): APIAuthStore.__init__(self) self._logger = logger if conn is not None: self.connection = conn else: if "replicaSet=" in uri: self.connection = pymongo.MongoReplicaSetClient(uri, **kwargs) else: self.connection = pymongo.MongoClient(uri, **kwargs) self.db = self.connection[db_name] self.coll = self.db['authusers'] for this in xrange(retries): try: self.coll.ensure_index([('authuser.factors.id', 1)], name='factor_id_idx', unique=True) break except (pymongo.errors.AutoReconnect, bson.errors.InvalidDocument) as exc: # InvalidDocument: When eduid-API starts at the same time as MongoDB (e.g. on reboot), # this error can be returned while MongoDB sorts out it's replica set status. if this == (retries - 1): logger.error( "Failed ensuring mongodb index, giving up after {!r} retries." .format(retries)) raise logger.debug( "Failed ensuring mongodb index, retrying ({!r})".format( exc)) time.sleep(1)
import datetime logging.basicConfig() arg_parser = argparse.ArgumentParser() arg_parser.add_argument('site_id', help='site_id to be added to authd instances') arg_parser.add_argument('--db_uri', help='DB uri', default='mongodb://127.0.0.1/central') args = arg_parser.parse_args() log = logging.getLogger('add_host') kwargs = dict(tz_aware=True) db_client = pymongo.MongoReplicaSetClient( args.db_uri, ** kwargs) if 'replicaSet' in args.db_uri else pymongo.MongoClient( args.db_uri, **kwargs) db = db_client.get_default_database() # is there already a host entry for this site_id? if not db.instances.find_one({'_id': args.site_id}): db.instances.insert({ '_id': args.site_id, 'date_added': datetime.datetime.now() }) log.info('entry created for host %s' % args.site_id) else: log.info('entry already exists for host %s' % args.site_id)
def run_query(self, query): if self.is_replica_set: db_connection = pymongo.MongoReplicaSetClient( self.configuration["connectionString"], replicaSet=self.configuration["replicaSetName"]) else: db_connection = pymongo.MongoClient( self.configuration["connectionString"]) db = db_connection[self.db_name] logger.debug("mongodb connection string: %s", self.configuration['connectionString']) logger.debug("mongodb got query: %s", query) try: query_data = parse_query_json(query) except ValueError: return None, "Invalid query format. The query is not a valid JSON." if "collection" not in query_data: return None, "'collection' must have a value to run a query" else: collection = query_data["collection"] q = query_data.get("query", None) f = None aggregate = query_data.get("aggregate", None) if aggregate: for step in aggregate: if "$sort" in step: sort_list = [] for sort_item in step["$sort"]: sort_list.append( (sort_item["name"], sort_item["direction"])) step["$sort"] = SON(sort_list) if not aggregate: s = None if "sort" in query_data and query_data["sort"]: s = [] for field in query_data["sort"]: s.append((field["name"], field["direction"])) if "fields" in query_data: f = query_data["fields"] s = None if "sort" in query_data and query_data["sort"]: s = [] for field_data in query_data["sort"]: s.append((field_data["name"], field_data["direction"])) columns = [] rows = [] cursor = None if q or (not q and not aggregate): if s: cursor = db[collection].find(q, f).sort(s) else: cursor = db[collection].find(q, f) if "skip" in query_data: cursor = cursor.skip(query_data["skip"]) if "limit" in query_data: cursor = cursor.limit(query_data["limit"]) elif aggregate: r = db[collection].aggregate(aggregate) # Backwards compatibility with older pymongo versions. # # Older pymongo version would return a dictionary from an aggregate command. # The dict would contain a "result" key which would hold the cursor. # Newer ones return pymongo.command_cursor.CommandCursor. if isinstance(r, dict): cursor = r["result"] else: cursor = r for r in cursor: for k in r: if self._get_column_by_name(columns, k) is None: columns.append({ "name": k, "friendly_name": k, "type": TYPES_MAP.get(type(r[k]), TYPE_STRING) }) rows.append(r) if f: ordered_columns = [] for k in sorted(f, key=f.get): ordered_columns.append(self._get_column_by_name(columns, k)) columns = ordered_columns data = {"columns": columns, "rows": rows} error = None json_data = json.dumps(data, cls=MongoDBJSONEncoder) return json_data, error
def getConnection(): if (settings.replica_set): return pymongo.MongoReplicaSetClient(settings.mongodb_host, replicaSet=settings.replica_set) else: return pymongo.Connection(settings.mongodb_host)
import bson.objectid import bson.json_util pymongo.objectid = bson.objectid pymongo.json_util = bson.json_util sys.modules['pymongo.objectid'] = bson.objectid sys.modules['pymongo.json_util'] = bson.json_util #MONGODB SETTINGS PRIM_DB = {'HOST': 'mm_243', 'PORT': 2011, 'USER': '', 'PASSWORD': ''} SCD_DB = {'HOST': 'mm_246', 'PORT': 2011, 'USER': '', 'PASSWORD': ''} host_url = '%s:%i,%s:%i' % (PRIM_DB['HOST'], PRIM_DB['PORT'], SCD_DB['HOST'], SCD_DB['PORT']) print host_url mongoConn = pymongo.MongoReplicaSetClient(host=host_url, replicaSet='user_center_replset') #利用mongodb 自带的connection poll 来管理数据库连接 #user_conn = Connection(host=MGDBS['user']['HOST'],port=MGDBS['user']['PORT']) sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) log_path = os.path.normpath( os.path.join(os.path.dirname(__file__), '../data_spider.log')) from user_center.conf import set_env set_env.getEnvReady() logger = logging.getLogger("data_spider") hdlr = logging.FileHandler(log_path) hdlr.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)-8s %(name)s:%(lineno)-15d %(message)s')
# -*- coding: utf-8 *-* import logging import pymongo import tornado.web import tornado.httpserver from tornado.ioloop import IOLoop from tornado.options import options as opts from selene import log, options, Selene, web if __name__ == '__main__': options.setup_options('selene.conf') if not opts.db_rs: db = pymongo.MongoClient(opts.db_uri)[opts.db_name] logging.info('Connected to a MongoDB standalone instance.') else: db = pymongo.MongoReplicaSetClient( opts.db_uri, replicaSet=opts.db_rs_name)[opts.db_name] logging.info('Connected to a MongoDB replica set.') if opts.logging_db: log.configure_mongolog() http_server = tornado.httpserver.HTTPServer(Selene(db)) tornado.web.ErrorHandler = web.ErrorHandler http_server.listen(opts.port) logging.info('Web server listening on %s port.' % opts.port) if opts.use_pyuv: from tornado_pyuv import UVLoop IOLoop.configure(UVLoop) loop = IOLoop.instance() loop.start()
help='time to sleep between is alive signals') arg_parser.add_argument('--debug', help='enable default mode', action='store_true', default=False) arg_parser.add_argument('--log_level', help='log level [info]', default='info') args = arg_parser.parse_args() args.site_name = ' '.join( args.site_name) if args.site_name else None # site_name as string logging.basicConfig() log.setLevel(getattr(logging, args.log_level.upper())) db = (pymongo.MongoReplicaSetClient(args.db_uri) if 'replicaSet' in args.db_uri else pymongo.MongoClient( args.db_uri)).get_default_database() fail_count = 0 while True: if not update(db, args.api_uri, args.site_name, args.site_id, args.ssl_cert, args.central_url): fail_count += 1 else: fail_count = 0 if fail_count == 3: log.debug('scitran central unreachable, purging all remotes info') clean_remotes(db) time.sleep(args.sleeptime)
from pymongo import MongoReplicaSetClient from pymongo.read_preferences import ReadPreference from time import sleep import datetime import pymongo import sys import bson def log(msg): timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print('%s %s' % (timestamp, msg)) log("Connecting to replica set...") client = pymongo.MongoReplicaSetClient('localhost:27017,localhost:27018', replicaset='replset') # connectTimeoutMS=50) db = client.testDB db.read_preference = ReadPreference.PRIMARY_PREFERRED while True: for retries in range(1, 11): try: doc = db.testRW.insert({'a': 1}) log('Inserted doc: %s' % doc) except (pymongo.errors.AutoReconnect, bson.errors.InvalidDocument, AssertionError): log('Insert failed, retrying #%s' % retries) if retries == 10: log('Max retries reached, giving up')