def registerShard(dbHost, configHost, dbPort=DATABASE_SERVER_PORT, configPort=ROUTER_SERVER_PORT, block=True, timeout=TIMEOUT): """ Function to register a database server as a shard in the database cluster """ functionName = registerShard.__name__ helpers.entrylog(log, functionName, locals()) if not block: timeout = 0 elif timeout <= 0: timeout = sys.maxint start = time.time() stop = start + timeout log.info("Trying to register %s:%d as a shard on %s:%d" %(dbHost, dbPort, configHost, configPort)) connection = getConnection(host=configHost, port=configPort, timeout=timeout) #check if mongos is up and connect to it getConnection(host=dbHost, port=DATABASE_SERVER_PORT, timeout=timeout) #check if mongod is up while time.time() < stop: if call("""mongo --host %s --eval "sh.addShard('%s:%d')" """ %(configHost, dbHost, dbPort), shell=True): log.debug("Failed to add shard. Will retry.") time.sleep(1) continue if connection.config.shards.find({"host": "%s:%d" % (dbHost, dbPort)}).count() == 0: log.debug("Failed to add shard. Will retry.") time.sleep(1) continue log.info("Registered %s as a shard on %s" %(dbHost, configHost)) helpers.exitlog(log, functionName) return log.error("Cannot add the required shard") helpers.exitlog(log, functionName) raise pymongo.errors.PyMongoError("Cannot add the required shard")
def isDBRunning(host='localhost', port=None): """ Check if a database server is running on a given host and port """ try: getConnection(host=host, port=port, block=False) log.info("An instance of mongodb server is already running on %s:%d" %(host, port)) return True except pymongo.errors.ConnectionFailure: log.info("No instance of mongodb server is already running on %s:%d" %(host, port)) return False
def isDBRunning(host='localhost', port=None): """ Check if a database server is running on a given host and port """ try: getConnection(host=host, port=port, block=False) log.info("An instance of mongodb server is already running on %s:%d" % (host, port)) return True except pymongo.errors.ConnectionFailure: log.info("No instance of mongodb server is already running on %s:%d" % (host, port)) return False
def startShardServer(port=ROUTER_SERVER_PORT, logPath=os.path.join(TEMP_DIR, "mongos.log"), configHost=None, configPort=CONFIG_SERVER_PORT, block=True, timeout=TIMEOUT): """ Function to start a database config server on the node """ if timeout <= 0: timeout = sys.maxint start = time.time() stop = start + timeout try: log.info("Checking if an instance of mongos server is already running") if isDBRunning(port=port): return log.info("Trying to connect to mongo config server") getConnection(configHost, port=configPort, block=block, timeout=timeout) log.info("Trying to start mongo shard server") mongos = [ 'mongos', '--configdb', '%s:%d' % (configHost, configPort), '--port', str(port), '--noAutoSplit', '--logpath', logPath ] log.info("Running %s", mongos) while time.time() < stop: p = Popen(mongos) time.sleep(1) if p.poll() is None: log.info("Started mongo shard server with pid %s", p.pid) return p log.debug("Failed to start shard config server.") if not block: break log.debug("Retrying to start mongo shard server") log.error("Cannot start mongo shard server") raise pymongo.errors.PyMongoError("Cannot start mongo shard server") except Exception, e: log.error("Exception while setting up mongo db shard server: %s", e) raise
def registerShard(dbHost, configHost, dbPort=DATABASE_SERVER_PORT, configPort=ROUTER_SERVER_PORT, block=True, timeout=TIMEOUT): """ Function to register a database server as a shard in the database cluster """ functionName = registerShard.__name__ helpers.entrylog(log, functionName, locals()) if not block: timeout = 0 elif timeout <= 0: timeout = sys.maxint start = time.time() stop = start + timeout log.info("Trying to register %s:%d as a shard on %s:%d" % (dbHost, dbPort, configHost, configPort)) connection = getConnection( host=configHost, port=configPort, timeout=timeout) #check if mongos is up and connect to it getConnection(host=dbHost, port=DATABASE_SERVER_PORT, timeout=timeout) #check if mongod is up while time.time() < stop: if call("""mongo --host %s --eval "sh.addShard('%s:%d')" """ % (configHost, dbHost, dbPort), shell=True): log.debug("Failed to add shard. Will retry.") time.sleep(1) continue if connection.config.shards.find({ "host": "%s:%d" % (dbHost, dbPort) }).count() == 0: log.debug("Failed to add shard. Will retry.") time.sleep(1) continue log.info("Registered %s as a shard on %s" % (dbHost, configHost)) helpers.exitlog(log, functionName) return log.error("Cannot add the required shard") helpers.exitlog(log, functionName) raise pymongo.errors.PyMongoError("Cannot add the required shard")
def isShardRegistered(dbHost, configHost, dbPort=DATABASE_SERVER_PORT, configPort=ROUTER_SERVER_PORT, block=True): """ Check if given mongo db host is registered as a shard """ functionName = isShardRegistered.__name__ helpers.entrylog(log, functionName, locals()) connection = getConnection(host=configHost, port=configPort) log.info("Checking if database server is registered as a shard") while True: try: if connection.config.shards.find({ "host": "%s:%d" % (dbHost, dbPort) }).count() != 0: helpers.exitlog(log, functionName) return True except: pass if not block: helpers.exitlog(log, functionName) return False time.sleep(1)
def moveChunk(db, collection, host, collector, configHost, configPort=ROUTER_SERVER_PORT): """ Shard, split and move a given collection to the corresponding collector """ functionName = moveChunk.__name__ helpers.entrylog(log, functionName, locals()) adminConnection = getConnection(host=configHost, port=configPort) log.info("Trying to move chunk %s:%s to %s" % (host, collection, collector)) while True: try: log.info("Enabling sharding %s.%s" % (db, collection)) adminConnection.admin.command('enablesharding', '%s.%s' % (db, collection)) log.info("Sharding enabled successfully.") break except pymongo.errors.OperationFailure, e: log.error(str(e)) #sharding might already be enabled if "already enabled" in str(e): break time.sleep(0.2)
def startShardServer(port=ROUTER_SERVER_PORT, logPath= os.path.join(TEMP_DIR, "mongos.log"), configHost=None, configPort=CONFIG_SERVER_PORT, block=True, timeout=TIMEOUT): """ Function to start a database config server on the node """ if timeout <= 0: timeout = sys.maxint start = time.time() stop = start + timeout try: log.info("Checking if an instance of mongos server is already running") if isDBRunning(port=port): return log.info("Trying to connect to mongo config server") getConnection(configHost, port=configPort, block=block, timeout=timeout) log.info("Trying to start mongo shard server") mongos = ['mongos', '--configdb', '%s:%d'%(configHost, configPort), '--port', str(port), '--noAutoSplit', '--logpath', logPath] log.info("Running %s", mongos) while time.time() < stop: p = Popen(mongos) time.sleep(1) if p.poll() is None: log.info("Started mongo shard server with pid %s", p.pid) return p log.debug("Failed to start shard config server.") if not block: break log.debug("Retrying to start mongo shard server") log.error("Cannot start mongo shard server") raise pymongo.errors.PyMongoError("Cannot start mongo shard server") except Exception, e: log.error("Exception while setting up mongo db shard server: %s", e) raise
def setBalancerState(state, configHost, configPort=CONFIG_SERVER_PORT): """ Function to turn on/off data balancer """ connection = getConnection(configHost, configPort) connection.config.settings.update({"_id": "balancer"}, {"$set": { "stopped": not state }}, True)
def __init__(self, sceneID, parent=None): super().__init__(parent) self.sceneID = sceneID # Set up db transaction self.db = getConnection() self.ok = False self.db.transaction() self.initUI() self.bind() self.okCancelApply.applyBtn.setDisabled(True)
def __init__(self, agentName, hostName, connection=None, dbHost='localhost', dbPort=DATABASE_SERVER_PORT): if not connection: connection = getConnection(host=dbHost, port=dbPort, block=False) pymongo.collection.Collection.__init__(self, connection[DB_NAME], COLLECTION_NAME) self.agentName = agentName self.hostName = hostName
def __init__(self, sceneID, parent=None): super().__init__(parent) self.sceneID = sceneID # Set up db transaction self.db = getConnection() self.ok = False self.db.transaction() self.initUI() self.bind() self.okCancelApply.applyBtn.setDisabled(True)
def isShardRegistered(dbHost, configHost, dbPort=DATABASE_SERVER_PORT, configPort=ROUTER_SERVER_PORT, block=True): """ Check if given mongo db host is registered as a shard """ functionName = isShardRegistered.__name__ helpers.entrylog(log, functionName, locals()) connection = getConnection(host=configHost, port=configPort) log.info("Checking if database server is registered as a shard") while True: try: if connection.config.shards.find({"host": "%s:%d" %(dbHost, dbPort)}).count() != 0: helpers.exitlog(log, functionName) return True except: pass if not block: helpers.exitlog(log, functionName) return False time.sleep(1)
def moveChunk(db, collection, host, collector, configHost, configPort=ROUTER_SERVER_PORT): """ Shard, split and move a given collection to the corresponding collector """ functionName = moveChunk.__name__ helpers.entrylog(log, functionName, locals()) adminConnection = getConnection(host=configHost, port=configPort) log.info("Trying to move chunk %s:%s to %s" %(host, collection, collector)) while True: try: log.info("Enabling sharding %s.%s" %(db, collection)) adminConnection.admin.command('enablesharding', '%s.%s' %(db, collection)) log.info("Sharding enabled successfully.") break except pymongo.errors.OperationFailure, e: log.error(str(e)) #sharding might already be enabled if "already enabled" in str(e): break time.sleep(0.2)
def __init__(self): super().__init__() self.db = getConnection() self.initUI()
def __init__(self, agentName, hostName, connection=None, dbHost='localhost', dbPort=DATABASE_SERVER_PORT): if not connection: connection = getConnection(host=dbHost, port=dbPort, block=False) pymongo.collection.Collection.__init__(self, connection[DB_NAME], COLLECTION_NAME) self.agentName = agentName self.hostName = hostName
def setBalancerState(state, configHost, configPort=CONFIG_SERVER_PORT): """ Function to turn on/off data balancer """ connection = getConnection(configHost, configPort) connection.config.settings.update({ "_id": "balancer" }, { "$set" : { "stopped": not state } } , True )