def registerShard(dbHost, configHost, dbPort=DATABASE_SERVER_PORT, configPort=ROUTER_SERVER_PORT, block=True, timeout=TIMEOUT):
"""
Function to register a database server as a shard in the database cluster
"""
functionName = registerShard.__name__
helpers.entrylog(log, functionName, locals())
if not block:
timeout = 0
elif timeout <= 0:
timeout = sys.maxint
start = time.time()
stop = start + timeout
log.info("Trying to register %s:%d as a shard on %s:%d" %(dbHost, dbPort, configHost, configPort))
connection = getConnection(host=configHost, port=configPort, timeout=timeout) #check if mongos is up and connect to it
getConnection(host=dbHost, port=DATABASE_SERVER_PORT, timeout=timeout) #check if mongod is up
while time.time() < stop:
if call("""mongo --host %s --eval "sh.addShard('%s:%d')" """ %(configHost, dbHost, dbPort), shell=True):
log.debug("Failed to add shard. Will retry.")
time.sleep(1)
continue
if connection.config.shards.find({"host": "%s:%d" % (dbHost, dbPort)}).count() == 0:
log.debug("Failed to add shard. Will retry.")
time.sleep(1)
continue
log.info("Registered %s as a shard on %s" %(dbHost, configHost))
helpers.exitlog(log, functionName)
return
log.error("Cannot add the required shard")
helpers.exitlog(log, functionName)
raise pymongo.errors.PyMongoError("Cannot add the required shard")
def isDBRunning(host='localhost', port=None):
"""
Check if a database server is running on a given host and port
"""
try:
getConnection(host=host, port=port, block=False)
log.info("An instance of mongodb server is already running on %s:%d" %(host, port))
return True
except pymongo.errors.ConnectionFailure:
log.info("No instance of mongodb server is already running on %s:%d" %(host, port))
return False
def isDBRunning(host='localhost', port=None):
"""
Check if a database server is running on a given host and port
"""
try:
getConnection(host=host, port=port, block=False)
log.info("An instance of mongodb server is already running on %s:%d" %
(host, port))
return True
except pymongo.errors.ConnectionFailure:
log.info("No instance of mongodb server is already running on %s:%d" %
(host, port))
return False
def startShardServer(port=ROUTER_SERVER_PORT,
logPath=os.path.join(TEMP_DIR, "mongos.log"),
configHost=None,
configPort=CONFIG_SERVER_PORT,
block=True,
timeout=TIMEOUT):
"""
Function to start a database config server on the node
"""
if timeout <= 0:
timeout = sys.maxint
start = time.time()
stop = start + timeout
try:
log.info("Checking if an instance of mongos server is already running")
if isDBRunning(port=port):
return
log.info("Trying to connect to mongo config server")
getConnection(configHost,
port=configPort,
block=block,
timeout=timeout)
log.info("Trying to start mongo shard server")
mongos = [
'mongos', '--configdb',
'%s:%d' % (configHost, configPort), '--port',
str(port), '--noAutoSplit', '--logpath', logPath
]
log.info("Running %s", mongos)
while time.time() < stop:
p = Popen(mongos)
time.sleep(1)
if p.poll() is None:
log.info("Started mongo shard server with pid %s", p.pid)
return p
log.debug("Failed to start shard config server.")
if not block:
break
log.debug("Retrying to start mongo shard server")
log.error("Cannot start mongo shard server")
raise pymongo.errors.PyMongoError("Cannot start mongo shard server")
except Exception, e:
log.error("Exception while setting up mongo db shard server: %s", e)
raise
def registerShard(dbHost,
configHost,
dbPort=DATABASE_SERVER_PORT,
configPort=ROUTER_SERVER_PORT,
block=True,
timeout=TIMEOUT):
"""
Function to register a database server as a shard in the database cluster
"""
functionName = registerShard.__name__
helpers.entrylog(log, functionName, locals())
if not block:
timeout = 0
elif timeout <= 0:
timeout = sys.maxint
start = time.time()
stop = start + timeout
log.info("Trying to register %s:%d as a shard on %s:%d" %
(dbHost, dbPort, configHost, configPort))
connection = getConnection(
host=configHost, port=configPort,
timeout=timeout) #check if mongos is up and connect to it
getConnection(host=dbHost, port=DATABASE_SERVER_PORT,
timeout=timeout) #check if mongod is up
while time.time() < stop:
if call("""mongo --host %s --eval "sh.addShard('%s:%d')" """ %
(configHost, dbHost, dbPort),
shell=True):
log.debug("Failed to add shard. Will retry.")
time.sleep(1)
continue
if connection.config.shards.find({
"host": "%s:%d" % (dbHost, dbPort)
}).count() == 0:
log.debug("Failed to add shard. Will retry.")
time.sleep(1)
continue
log.info("Registered %s as a shard on %s" % (dbHost, configHost))
helpers.exitlog(log, functionName)
return
log.error("Cannot add the required shard")
helpers.exitlog(log, functionName)
raise pymongo.errors.PyMongoError("Cannot add the required shard")
def isShardRegistered(dbHost,
configHost,
dbPort=DATABASE_SERVER_PORT,
configPort=ROUTER_SERVER_PORT,
block=True):
"""
Check if given mongo db host is registered as a shard
"""
functionName = isShardRegistered.__name__
helpers.entrylog(log, functionName, locals())
connection = getConnection(host=configHost, port=configPort)
log.info("Checking if database server is registered as a shard")
while True:
try:
if connection.config.shards.find({
"host": "%s:%d" % (dbHost, dbPort)
}).count() != 0:
helpers.exitlog(log, functionName)
return True
except:
pass
if not block:
helpers.exitlog(log, functionName)
return False
time.sleep(1)
def moveChunk(db,
collection,
host,
collector,
configHost,
configPort=ROUTER_SERVER_PORT):
"""
Shard, split and move a given collection to the corresponding collector
"""
functionName = moveChunk.__name__
helpers.entrylog(log, functionName, locals())
adminConnection = getConnection(host=configHost, port=configPort)
log.info("Trying to move chunk %s:%s to %s" %
(host, collection, collector))
while True:
try:
log.info("Enabling sharding %s.%s" % (db, collection))
adminConnection.admin.command('enablesharding',
'%s.%s' % (db, collection))
log.info("Sharding enabled successfully.")
break
except pymongo.errors.OperationFailure, e:
log.error(str(e)) #sharding might already be enabled
if "already enabled" in str(e):
break
time.sleep(0.2)
def startShardServer(port=ROUTER_SERVER_PORT,
logPath= os.path.join(TEMP_DIR, "mongos.log"),
configHost=None,
configPort=CONFIG_SERVER_PORT,
block=True,
timeout=TIMEOUT):
"""
Function to start a database config server on the node
"""
if timeout <= 0:
timeout = sys.maxint
start = time.time()
stop = start + timeout
try:
log.info("Checking if an instance of mongos server is already running")
if isDBRunning(port=port):
return
log.info("Trying to connect to mongo config server")
getConnection(configHost, port=configPort, block=block, timeout=timeout)
log.info("Trying to start mongo shard server")
mongos = ['mongos', '--configdb', '%s:%d'%(configHost, configPort),
'--port', str(port),
'--noAutoSplit',
'--logpath', logPath]
log.info("Running %s", mongos)
while time.time() < stop:
p = Popen(mongos)
time.sleep(1)
if p.poll() is None:
log.info("Started mongo shard server with pid %s", p.pid)
return p
log.debug("Failed to start shard config server.")
if not block:
break
log.debug("Retrying to start mongo shard server")
log.error("Cannot start mongo shard server")
raise pymongo.errors.PyMongoError("Cannot start mongo shard server")
except Exception, e:
log.error("Exception while setting up mongo db shard server: %s", e)
raise
def setBalancerState(state, configHost, configPort=CONFIG_SERVER_PORT):
"""
Function to turn on/off data balancer
"""
connection = getConnection(configHost, configPort)
connection.config.settings.update({"_id": "balancer"},
{"$set": {
"stopped": not state
}}, True)
def __init__(self, sceneID, parent=None):
super().__init__(parent)
self.sceneID = sceneID
# Set up db transaction
self.db = getConnection()
self.ok = False
self.db.transaction()
self.initUI()
self.bind()
self.okCancelApply.applyBtn.setDisabled(True)
def __init__(self,
agentName,
hostName,
connection=None,
dbHost='localhost',
dbPort=DATABASE_SERVER_PORT):
if not connection:
connection = getConnection(host=dbHost, port=dbPort, block=False)
pymongo.collection.Collection.__init__(self, connection[DB_NAME],
COLLECTION_NAME)
self.agentName = agentName
self.hostName = hostName
def __init__(self, sceneID, parent=None):
super().__init__(parent)
self.sceneID = sceneID
# Set up db transaction
self.db = getConnection()
self.ok = False
self.db.transaction()
self.initUI()
self.bind()
self.okCancelApply.applyBtn.setDisabled(True)
def isShardRegistered(dbHost, configHost, dbPort=DATABASE_SERVER_PORT, configPort=ROUTER_SERVER_PORT, block=True):
"""
Check if given mongo db host is registered as a shard
"""
functionName = isShardRegistered.__name__
helpers.entrylog(log, functionName, locals())
connection = getConnection(host=configHost, port=configPort)
log.info("Checking if database server is registered as a shard")
while True:
try:
if connection.config.shards.find({"host": "%s:%d" %(dbHost, dbPort)}).count() != 0:
helpers.exitlog(log, functionName)
return True
except:
pass
if not block:
helpers.exitlog(log, functionName)
return False
time.sleep(1)
def moveChunk(db, collection, host, collector, configHost, configPort=ROUTER_SERVER_PORT):
"""
Shard, split and move a given collection to the corresponding collector
"""
functionName = moveChunk.__name__
helpers.entrylog(log, functionName, locals())
adminConnection = getConnection(host=configHost, port=configPort)
log.info("Trying to move chunk %s:%s to %s" %(host, collection, collector))
while True:
try:
log.info("Enabling sharding %s.%s" %(db, collection))
adminConnection.admin.command('enablesharding', '%s.%s' %(db, collection))
log.info("Sharding enabled successfully.")
break
except pymongo.errors.OperationFailure, e:
log.error(str(e)) #sharding might already be enabled
if "already enabled" in str(e):
break
time.sleep(0.2)
def __init__(self):
super().__init__()
self.db = getConnection()
self.initUI()
def __init__(self, agentName, hostName, connection=None, dbHost='localhost', dbPort=DATABASE_SERVER_PORT):
if not connection:
connection = getConnection(host=dbHost, port=dbPort, block=False)
pymongo.collection.Collection.__init__(self, connection[DB_NAME], COLLECTION_NAME)
self.agentName = agentName
self.hostName = hostName
def setBalancerState(state, configHost, configPort=CONFIG_SERVER_PORT):
"""
Function to turn on/off data balancer
"""
connection = getConnection(configHost, configPort)
connection.config.settings.update({ "_id": "balancer" }, { "$set" : { "stopped": not state } } , True )