def mon_repl(self, host, port):
'''
监控repl集群,并时刻把拓扑结构回传给conf文件
'''
try:
#初始化mongo链接，并获取副本状态
logging.info('connect repl start')
conn = pymongo.MongoReplicaSetClient("%s:%s" % (host, port))
repl_status = conn.admin.command("replSetGetStatus")
ReplSetName = repl_status['set']
logging.info('ReplSetName is %s' %(ReplSetName))
#解析所有成员状态
new_machine_list = ''
for member in repl_status['members']:
member_host, member_port = member['name'].split(':')
if re.search(r"(reachable|healthy|RECOVERING)",member['stateStr']):
#logging.error("%s:%s is offline" % (member_host, member_port))
push_data.push_data(member_host, member_port, 0)
else:
#logging.info("%s:%s is normal" % (member_host, member_port))
push_data.push_data(member_host, member_port, 1)
new_machine_list += member['name'] + ";"
new_machine_list = new_machine_list.strip(';')
conf.set(self.section, 'mach_list', new_machine_list)
conf.write(open(config_file, 'w'))
except Exception, e:
#截获链接异常
logging.critical(e)
def _get_db(self):
if self.is_replica_set:
db_connection = pymongo.MongoReplicaSetClient(self.configuration["connectionString"], replicaSet=self.configuration["replicaSetName"])
else:
db_connection = pymongo.MongoClient(self.configuration["connectionString"])
return db_connection[self.db_name]
def add_user(user='******', manager=False, admin=False, mongo_options=None):
token = str(uuid.uuid4())
password = '******'
email = '*****@*****.**'
db_body = {
'_id': user,
'password': password,
'email': email,
'token': token
}
if not mongo_options:
mdb = pymongo.MongoClient()
else:
if is_domain(mongo_options['host']):
mongo_options['ssl'] = True
if 'replicaSet' in mongo_options:
mdb = pymongo.MongoReplicaSetClient(**mongo_options)
mdb.read_preference = pymongo.ReadPreference.SECONDARY_PREFERRED
else:
mdb = pymongo.MongoClient(**mongo_options)
mdb.users.all.insert(db_body)
result = db_body
result['user'] = user
if manager:
weight = 1
db_body = {'_id': user, 'weight': weight}
mdb.users.managers.insert(db_body)
result['weight'] = weight
if admin:
db_body = {'_id': user}
mdb.users.admins.insert(db_body)
return result
def test_auth_connection(self):
self.assertTrue(
isinstance(self.repl.connection().admin.collection_names(), list))
c = pymongo.MongoReplicaSetClient(self.repl.primary(),
replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure,
c.admin.collection_names)
def __init__(self, collection="results"):
"""Initilize connection and check indeces"""
try:
# MongoDB URI
uri = self.make_uri()
# Database
database = config["app_conf"]["mongo_db"]
# Collection
# Updated the mongo client connection for the collection
# We could also use MongoReplicaSetClient for dealing with replica sets.
replicate = config["app_conf"]["mongo_replicate"]
if replicate == "true":
self.collection = pymongo.MongoReplicaSetClient(host=uri,port=int(config["app_conf"]["mongo_port"]),replicaSet=(config["app_conf"]["mongo_replset"]))[database][collection]
else:
self.collection = pymongo.mongo_client.MongoClient(host=uri,port=int(config["app_conf"]["mongo_port"]))[database][collection]
# Indecies
self.ensure_index()
except Exception as error:
# Exception type: Exception message
c.message = ": ".join([type(error).__name__, error.message])
def getDbConnection(uri=None, replicaSet=None):
"""
Get a MongoClient object that is connected to the configured database.
We lazy-instantiate a module-level singleton, the MongoClient objects
manage their own connection pools internally.
:param uri: if specified, connect to this mongo db rather than the one in
the config.
:param replicaSet: if uri is specified, use this replica set.
"""
global _dbClients
origKey = (uri, replicaSet)
if origKey in _dbClients:
return _dbClients[origKey]
if uri is None or uri == '':
dbConf = getDbConfig()
uri = dbConf.get('uri')
replicaSet = dbConf.get('replica_set')
clientOptions = {
'connectTimeoutMS': 15000,
# This is the maximum time between when we fetch data from a cursor.
# If it times out, the cursor is lost and we can't reconnect. If it
# isn't set, we have issues with replica sets when the primary goes
# down. This value can be overridden in the mongodb uri connection
# string with the socketTimeoutMS.
'socketTimeoutMS': 60000,
}
if uri is None:
dbUriRedacted = 'mongodb://*****:*****@')
if len(parts) == 2:
dbUriRedacted = 'mongodb://' + parts[1]
else:
dbUriRedacted = uri
if replicaSet:
client = pymongo.MongoReplicaSetClient(
uri,
replicaSet=replicaSet,
read_preference=ReadPreference.SECONDARY_PREFERRED,
**clientOptions)
else:
client = pymongo.MongoClient(uri, **clientOptions)
client = MongoProxy(client, logger=logger)
_dbClients[origKey] = _dbClients[(uri, replicaSet)] = client
desc = ''
if replicaSet:
desc += ', replica set: %s' % replicaSet
print(
TerminalColor.info('Connected to MongoDB: %s%s' %
(dbUriRedacted, desc)))
return client
def _initialize_collection(self):
try:
from gevent import monkey
monkey.patch_socket()
except ImportError:
pass
if self._rsname:
self.connection = pymongo.MongoReplicaSetClient(
self._rshosts,
replicaSet=self._rsname,
read_preference=getattr(pymongo.ReadPreference,
self._read_preference, 'PRIMARY'),
socketTimeoutMS=self._socket_timeout_ms,
connectTimeoutMS=self._connect_timeout_ms,
tag_sets=self._tag_sets)
else:
self.connection = pymongo.Connection(self._host, self._port)
self._db = self.connection[self._database]
if self._user and self._password:
self._db.authenticate(self._user, self._password)
self._coll = self._db[self._collection]
#-----------------------------------------------------------------------------------------------------------------------
def connection(self,
hostname=None,
read_preference=pymongo.ReadPreference.PRIMARY,
timeout=300):
"""return MongoReplicaSetClient object if hostname specified
return MongoClient object if hostname doesn't specified
Args:
hostname - connection uri
read_preference - default PRIMARY
timeout - specify how long, in seconds, a command can take before server times out.
"""
logger.debug(
"connection({hostname}, {read_preference}, {timeout})".format(
**locals()))
t_start = time.time()
servers = hostname or ",".join(self.server_map.values())
while True:
try:
if hostname is None:
c = pymongo.MongoReplicaSetClient(
servers,
replicaSet=self.repl_id,
read_preference=read_preference,
socketTimeoutMS=20000,
**self.kwargs)
if c.primary:
try:
self.login and self.password and c.admin.authenticate(
self.login, self.password)
except:
pass
return c
raise pymongo.errors.AutoReconnect(
"No replica set primary available")
else:
logger.debug(
"connection to the {servers}".format(**locals()))
c = pymongo.MongoClient(servers,
socketTimeoutMS=20000,
**self.kwargs)
if self.login and self.password:
try:
c.admin.authenticate(self.login, self.password)
except:
pass
return c
except (pymongo.errors.PyMongoError):
exc_type, exc_value, exc_tb = sys.exc_info()
err_message = traceback.format_exception(
exc_type, exc_value, exc_tb)
logger.error(
"Exception {exc_type} {exc_value}".format(**locals()))
logger.error(err_message)
if time.time() - t_start > timeout:
raise pymongo.errors.AutoReconnect(
"Couldn't connect while timeout {timeout} second".
format(**locals()))
time.sleep(10)
def get_conn(host_url, replica_set_name):
try:
mongoConn = pymongo.MongoReplicaSetClient(
host=host_url,
replicaSet=replica_set_name,
read_preference=pymongo.ReadPreference.PRIMARY_PREFERRED)
except AutoReconnect, e:
print "init mongo connection failed.... "
mongoConn = None
def __init__(self, uri, conn=None, dbname='eduid_api', **kwargs):
if conn is not None:
self.connection = conn
else:
if 'replicaSet=' in uri:
self.connection = pymongo.MongoReplicaSetClient(uri, **kwargs)
else:
self.connection = pymongo.MongoClient(uri, **kwargs)
self.db = self.connection[dbname]
def db(self):
if self._database is None:
if options.CFG.uri and 'replicaSet' in options.CFG.uri:
conn = pymongo.MongoReplicaSetClient(options.CFG.uri)
else:
conn = pymongo.MongoClient(options.CFG.uri)
self._database = conn[options.CFG.database]
return self._database
def test_auth_admin(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(),
replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure,
c.admin.collection_names)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
self.assertTrue(isinstance(c.admin.collection_names(), list))
self.assertTrue(c.admin.logout() is None)
self.assertRaises(pymongo.errors.OperationFailure,
c.admin.collection_names)
def __init__(self, mongo_uri, dbname='helper', replset=0, pool_size=100):
try:
if replset:
self.con = pymongo.MongoReplicaSetClient(
mongo_uri,
read_preference=pymongo.ReadPreference.SECONDARY_PREFERRED,
max_pool_size=pool_size)
else:
self.con = pymongo.MongoClient(mongo_uri)
self.db = self.con[dbname]
except Exception as e:
raise Exception('connect mongodb: %s, error: %s' % (mongo_uri, e))
def test_auth_collection(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(),
replicaSet=self.repl.repl_id)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
db = c.test_auth
db.add_user('user', 'userpass', roles=['readWrite'])
c.admin.logout()
self.assertTrue(db.authenticate('user', 'userpass'))
self.assertTrue(db.foo.insert({'foo': 'bar'}, w=2, wtimeout=1000))
self.assertTrue(isinstance(db.foo.find_one(), dict))
db.logout()
self.assertRaises(pymongo.errors.OperationFailure, db.foo.find_one)
def _mongo_connect(url):
try:
if cfg.CONF.database.mongodb_replica_set:
client = MongoProxy(
pymongo.MongoReplicaSetClient(
url, replicaSet=cfg.CONF.database.mongodb_replica_set))
else:
client = MongoProxy(pymongo.MongoClient(url, safe=True))
return client
except pymongo.errors.ConnectionFailure as e:
LOG.warn(
_('Unable to connect to the database server: '
'%(errmsg)s.') % {'errmsg': e})
raise
def connect_db(db_uri, **kwargs):
for x in range(0, 30):
try:
db_client = pymongo.MongoReplicaSetClient(
db_uri, **
kwargs) if 'replicaSet' in db_uri else pymongo.MongoClient(
db_uri, **kwargs)
except:
time.sleep(1)
pass
else:
break
else:
raise Exception("Could not connect to MongoDB")
return db_client
def get_server_info(hostname="localhost", port="27017", replica_set=None):
"""
Get the mongod server build info and server status from the target mongod server
:param hostname: the hostname the target database is running on (defaults to localhost)
:param port: the port the target database is running on (defaults to 27017)
:param replica_set: the replica set name the target database is using (defaults to none)
:return: a tuple of the buildinfo and the server status
"""
if replica_set is None:
client = pymongo.MongoClient("mongodb://%s:%s/test" % (hostname, port))
else:
client = pymongo.MongoReplicaSetClient("mongodb://%s:%s/test?replicaSet=%s" % (hostname, port, replica_set))
db = client.test
server_build_info = db.command("buildinfo")
server_status = db.command("serverStatus")
client.close()
return server_build_info, server_status
def main(args):
client = pymongo.MongoReplicaSetClient(URI)
db = client.get_default_database()
collection_name = _get_unique_tmp_collection_name()
c = db[collection_name]
try:
while True:
now = datetime.datetime.now()
try:
if client.primary:
print("%s: connected to primary server '%s'..." %
(now,
client.primary[0] + ":" + str(client.primary[1])))
print("...about to perform safe insert.")
c.insert({"t": now})
print("...about to read.")
num_docs = c.find({}).count()
print("...successfully inserted & read. There are %i "
"documents in your temporary test collection." %
num_docs)
retry_total_time = 0
except Exception as e:
print("*** EXCEPTION - %s" % e)
print("*** ...will sleep for %s seconds before retrying "
"[total of %s seconds]" %
(RETRY_WAIT_IN_SEC, retry_total_time))
retry_total_time += RETRY_WAIT_IN_SEC
print("\n")
time.sleep(RETRY_WAIT_IN_SEC)
except:
print("An unexpected exception occurred. Performing cleanup "
"if possible by dropping test collection '%s'..." %
collection_name)
c.drop()
client.close()
def mongo_connection(
mongo_host=None,
mongo_port=None,
mongo_replica_set=None,
):
if mongo_replica_set is not None:
with elle.log.log('connect to MongoDB replica set %s' %
(mongo_replica_set, )):
return pymongo.MongoReplicaSetClient(','.join(mongo_replica_set),
replicaSet='fist-meta')
else:
with elle.log.log('connect to MongoDB on %s:%s' %
(mongo_host, mongo_port)):
db_args = {}
if mongo_host is not None:
db_args['host'] = mongo_host
if mongo_port is not None:
db_args['port'] = mongo_port
return pymongo.MongoClient(**db_args)
def get_connection(
host="127.0.0.1",
port=27017,
database="trunk",
user="******",
password="******",
replica=None,
**kwargs
):
"""Возвращает синхронное подключение к базе.
:param host: Хост, к которому выполняется подключение
:type host: str
:param port: Порт базы данных
:type port: int
:param user: Имя пользователя базы данных
:type user: str
:param password: Пароль пользователя базы данных
:type password: str
:param replica: Название replicaSet (при наличии)
:type replica: str
:param database: Имя базы данных
:type database: str
"""
if not replica:
con = pymongo.MongoClient(
"mongodb://{}:{}@{}:{}/{}".format(
user, password,
host, port,
database
))
else:
con = pymongo.MongoReplicaSetClient(
"mongodb://{}:{}@{}:{}/{}".format(
user, password,
host, port,
database
),
replicaSet=replica,
connectTimeoutMS=1500,
socketTimeoutMS=1500
)
return con[database]
def _get_db(self):
# defer imports until backend is used
global pymongo
import pymongo
if self.use_replica:
connection = pymongo.MongoReplicaSetClient(
host=self.hosts, replicaSet=self.replicaset_name,
max_pool_size=self.max_pool_size, **self.conn_kwargs)
else: # used for standalone node or mongos in sharded setup
connection = pymongo.MongoClient(
host=self.hosts, max_pool_size=self.max_pool_size,
**self.conn_kwargs)
database = getattr(connection, self.db_name)
self._assign_data_mainpulator()
database.add_son_manipulator(self._data_manipulator)
if self.username and self.password:
database.authenticate(self.username, self.password)
return database
def __getattr__(self, item):
if item in self.dbs and self._mongoclient is None:
if mode == 'test':
self._mongoclient = pymongo.MongoClient(host=self.localhost,
port=self.port)
else:
self._mongoclient = pymongo.MongoReplicaSetClient(
','.join(mongo_machines),
replicaSet=REPLICASET_NAME,
)
#print "+++++++++authing++++++++++"
self._mongoclient.fbt.authenticate(FBT_USER, FBT_PASSWD)
#print "+++++++++authing OK++++++++++"+str(ok)
self.dbs['fbt'] = self._mongoclient.fbt
self.module.__dict__.update(self.dbs)
return getattr(self.module, item)
def __init__(self,
uri,
logger,
conn=None,
db_name="eduid_api",
retries=10,
**kwargs):
APIAuthStore.__init__(self)
self._logger = logger
if conn is not None:
self.connection = conn
else:
if "replicaSet=" in uri:
self.connection = pymongo.MongoReplicaSetClient(uri, **kwargs)
else:
self.connection = pymongo.MongoClient(uri, **kwargs)
self.db = self.connection[db_name]
self.coll = self.db['authusers']
for this in xrange(retries):
try:
self.coll.ensure_index([('authuser.factors.id', 1)],
name='factor_id_idx',
unique=True)
break
except (pymongo.errors.AutoReconnect,
bson.errors.InvalidDocument) as exc:
# InvalidDocument: When eduid-API starts at the same time as MongoDB (e.g. on reboot),
# this error can be returned while MongoDB sorts out it's replica set status.
if this == (retries - 1):
logger.error(
"Failed ensuring mongodb index, giving up after {!r} retries."
.format(retries))
raise
logger.debug(
"Failed ensuring mongodb index, retrying ({!r})".format(
exc))
time.sleep(1)
import datetime
logging.basicConfig()
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('site_id',
help='site_id to be added to authd instances')
arg_parser.add_argument('--db_uri',
help='DB uri',
default='mongodb://127.0.0.1/central')
args = arg_parser.parse_args()
log = logging.getLogger('add_host')
kwargs = dict(tz_aware=True)
db_client = pymongo.MongoReplicaSetClient(
args.db_uri, **
kwargs) if 'replicaSet' in args.db_uri else pymongo.MongoClient(
args.db_uri, **kwargs)
db = db_client.get_default_database()
# is there already a host entry for this site_id?
if not db.instances.find_one({'_id': args.site_id}):
db.instances.insert({
'_id': args.site_id,
'date_added': datetime.datetime.now()
})
log.info('entry created for host %s' % args.site_id)
else:
log.info('entry already exists for host %s' % args.site_id)
def run_query(self, query):
if self.is_replica_set:
db_connection = pymongo.MongoReplicaSetClient(
self.configuration["connectionString"],
replicaSet=self.configuration["replicaSetName"])
else:
db_connection = pymongo.MongoClient(
self.configuration["connectionString"])
db = db_connection[self.db_name]
logger.debug("mongodb connection string: %s",
self.configuration['connectionString'])
logger.debug("mongodb got query: %s", query)
try:
query_data = parse_query_json(query)
except ValueError:
return None, "Invalid query format. The query is not a valid JSON."
if "collection" not in query_data:
return None, "'collection' must have a value to run a query"
else:
collection = query_data["collection"]
q = query_data.get("query", None)
f = None
aggregate = query_data.get("aggregate", None)
if aggregate:
for step in aggregate:
if "$sort" in step:
sort_list = []
for sort_item in step["$sort"]:
sort_list.append(
(sort_item["name"], sort_item["direction"]))
step["$sort"] = SON(sort_list)
if not aggregate:
s = None
if "sort" in query_data and query_data["sort"]:
s = []
for field in query_data["sort"]:
s.append((field["name"], field["direction"]))
if "fields" in query_data:
f = query_data["fields"]
s = None
if "sort" in query_data and query_data["sort"]:
s = []
for field_data in query_data["sort"]:
s.append((field_data["name"], field_data["direction"]))
columns = []
rows = []
cursor = None
if q or (not q and not aggregate):
if s:
cursor = db[collection].find(q, f).sort(s)
else:
cursor = db[collection].find(q, f)
if "skip" in query_data:
cursor = cursor.skip(query_data["skip"])
if "limit" in query_data:
cursor = cursor.limit(query_data["limit"])
elif aggregate:
r = db[collection].aggregate(aggregate)
# Backwards compatibility with older pymongo versions.
#
# Older pymongo version would return a dictionary from an aggregate command.
# The dict would contain a "result" key which would hold the cursor.
# Newer ones return pymongo.command_cursor.CommandCursor.
if isinstance(r, dict):
cursor = r["result"]
else:
cursor = r
for r in cursor:
for k in r:
if self._get_column_by_name(columns, k) is None:
columns.append({
"name":
k,
"friendly_name":
k,
"type":
TYPES_MAP.get(type(r[k]), TYPE_STRING)
})
rows.append(r)
if f:
ordered_columns = []
for k in sorted(f, key=f.get):
ordered_columns.append(self._get_column_by_name(columns, k))
columns = ordered_columns
data = {"columns": columns, "rows": rows}
error = None
json_data = json.dumps(data, cls=MongoDBJSONEncoder)
return json_data, error
def getConnection():
if (settings.replica_set):
return pymongo.MongoReplicaSetClient(settings.mongodb_host,
replicaSet=settings.replica_set)
else:
return pymongo.Connection(settings.mongodb_host)
import bson.objectid
import bson.json_util
pymongo.objectid = bson.objectid
pymongo.json_util = bson.json_util
sys.modules['pymongo.objectid'] = bson.objectid
sys.modules['pymongo.json_util'] = bson.json_util
#MONGODB SETTINGS
PRIM_DB = {'HOST': 'mm_243', 'PORT': 2011, 'USER': '', 'PASSWORD': ''}
SCD_DB = {'HOST': 'mm_246', 'PORT': 2011, 'USER': '', 'PASSWORD': ''}
host_url = '%s:%i,%s:%i' % (PRIM_DB['HOST'], PRIM_DB['PORT'], SCD_DB['HOST'],
SCD_DB['PORT'])
print host_url
mongoConn = pymongo.MongoReplicaSetClient(host=host_url,
replicaSet='user_center_replset')
#利用mongodb 自带的connection poll 来管理数据库连接
#user_conn = Connection(host=MGDBS['user']['HOST'],port=MGDBS['user']['PORT'])
sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
log_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), '../data_spider.log'))
from user_center.conf import set_env
set_env.getEnvReady()
logger = logging.getLogger("data_spider")
hdlr = logging.FileHandler(log_path)
hdlr.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)-8s %(name)s:%(lineno)-15d %(message)s')
# -*- coding: utf-8 *-*
import logging
import pymongo
import tornado.web
import tornado.httpserver
from tornado.ioloop import IOLoop
from tornado.options import options as opts
from selene import log, options, Selene, web
if __name__ == '__main__':
options.setup_options('selene.conf')
if not opts.db_rs:
db = pymongo.MongoClient(opts.db_uri)[opts.db_name]
logging.info('Connected to a MongoDB standalone instance.')
else:
db = pymongo.MongoReplicaSetClient(
opts.db_uri, replicaSet=opts.db_rs_name)[opts.db_name]
logging.info('Connected to a MongoDB replica set.')
if opts.logging_db:
log.configure_mongolog()
http_server = tornado.httpserver.HTTPServer(Selene(db))
tornado.web.ErrorHandler = web.ErrorHandler
http_server.listen(opts.port)
logging.info('Web server listening on %s port.' % opts.port)
if opts.use_pyuv:
from tornado_pyuv import UVLoop
IOLoop.configure(UVLoop)
loop = IOLoop.instance()
loop.start()
help='time to sleep between is alive signals')
arg_parser.add_argument('--debug',
help='enable default mode',
action='store_true',
default=False)
arg_parser.add_argument('--log_level',
help='log level [info]',
default='info')
args = arg_parser.parse_args()
args.site_name = ' '.join(
args.site_name) if args.site_name else None # site_name as string
logging.basicConfig()
log.setLevel(getattr(logging, args.log_level.upper()))
db = (pymongo.MongoReplicaSetClient(args.db_uri)
if 'replicaSet' in args.db_uri else pymongo.MongoClient(
args.db_uri)).get_default_database()
fail_count = 0
while True:
if not update(db, args.api_uri, args.site_name, args.site_id,
args.ssl_cert, args.central_url):
fail_count += 1
else:
fail_count = 0
if fail_count == 3:
log.debug('scitran central unreachable, purging all remotes info')
clean_remotes(db)
time.sleep(args.sleeptime)
from pymongo import MongoReplicaSetClient
from pymongo.read_preferences import ReadPreference
from time import sleep
import datetime
import pymongo
import sys
import bson
def log(msg):
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s %s' % (timestamp, msg))
log("Connecting to replica set...")
client = pymongo.MongoReplicaSetClient('localhost:27017,localhost:27018',
replicaset='replset')
# connectTimeoutMS=50)
db = client.testDB
db.read_preference = ReadPreference.PRIMARY_PREFERRED
while True:
for retries in range(1, 11):
try:
doc = db.testRW.insert({'a': 1})
log('Inserted doc: %s' % doc)
except (pymongo.errors.AutoReconnect, bson.errors.InvalidDocument,
AssertionError):
log('Insert failed, retrying #%s' % retries)
if retries == 10:
log('Max retries reached, giving up')
