def open(self): try: if self.replica_set: self.conn = pymongo.ReplicaSetConnection(self.mongodb_uri, replicaSet=self.replica_set, fsync=self.mongodb_fsync) else: # Old versions of pymongo do not known about fsync if ReplicaSetConnection: self.conn = pymongo.Connection(self.mongodb_uri, fsync=self.mongodb_fsync) else: self.conn = pymongo.Connection(self.mongodb_uri) self.db = self.conn[self.database] self.db[self.collection].ensure_index([('host_name', pymongo.ASCENDING), ('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='logs_idx') self.db[self.collection].ensure_index([('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='time_1_lineno_1') if self.replica_set: pass # This might be a future option prefer_secondary #self.db.read_preference = ReadPreference.SECONDARY self.is_connected = CONNECTED self.next_log_db_rotate = time.time() except AutoReconnect as err: # now what, ha? logger.error("[LogStoreMongoDB] LiveStatusLogStoreMongoDB.AutoReconnect %s" % err) # The mongodb is hopefully available until this module is restarted raise LiveStatusLogStoreError(err) except Exception as err: # If there is a replica_set, but the host is a simple standalone one # we get a "No suitable hosts found" here. # But other reasons are possible too. logger.error("[LogStoreMongoDB] Could not open the database: %s" % err) raise LiveStatusLogStoreError(err)
def main(argv): arg_parsing(argv) global connection global db print "Welcome to the HW 6.x replica Checker. My job is to make sure you started a replica set with three nodes" # connect to the db (mongostr was set in arg_parsing) try: connection = pymongo.ReplicaSetConnection(mongostr, replicaSet=rs_name, safe=True) db = connection[db_name] except: print "can't connect to MongoDB replica", rs_name, " set using", mongostr, ". Is it running?" exit(2) # no graceful exit if it is not connected if (not repl_set_running(3)): print "Sorry, the replica set does not seem to be running" graceful_exit(1) # if you are reading this in cleartext, you are violating the honor code. # You can still redeem yourself. Get it working and don't submit the validation code until you do. # All a man has at the end of the day is his word. print "Tests Passed for HW 6.5. Your HW 6.5 validation code is kjvjkl3290mf0m20f2kjjv" graceful_exit(0)
def open(self): try: if self.replica_set: self.conn = pymongo.ReplicaSetConnection( self.mongodb_uri, replicaSet=self.replica_set, fsync=self.mongodb_fsync) else: # Old versions of pymongo do not known about fsync if ReplicaSetConnection: self.conn = pymongo.Connection(self.mongodb_uri, fsync=self.mongodb_fsync) else: self.conn = pymongo.Connection(self.mongodb_uri) self.db = self.conn[self.database] self.db[self.collection].ensure_index( [('host_name', pymongo.ASCENDING), ('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='logs_idx') if self.replica_set: pass # This might be a future option prefer_secondary #self.db.read_preference = ReadPreference.SECONDARY self.is_connected = CONNECTED self.next_log_db_rotate = time.time() except AutoReconnect, exp: # now what, ha? logger.error( "[LogStoreMongoDB] LiveStatusLogStoreMongoDB.AutoReconnect %s" % (exp)) # The mongodb is hopefully available until this module is restarted raise LiveStatusLogStoreError
def __init__(self): # # TODO :: Add Error trapping / retries! # mongo = pymongo.ReplicaSetConnection( "data2:27017,data1:27017", replicaSet="linux",slaveOk=True, safe=True) self.db = mongo.fc_gluster
def database( config, read_preference=None, ): conn = _connection(config) host = conn['host'] replica_set = conn.get('replica-set') db = conn['database'] colls = conn['collections'] if replica_set: conn = pymongo.ReplicaSetConnection( host, replicaSet=replica_set, ) # ReadPreference.PRIMARY is the default if read_preference is not None: conn.read_preference = read_preference else: conn = pymongo.Connection(host) db = ProxyDatabase(conn, db, colls) return db
''' Extract the column number that needs updating ''' for index, c in enumerate(header): if columnName == c: columnNumber = index print "Column that needs updating is: " + str(columnNumber) ''' Querying mongo db, for now development ''' con = pymongo.ReplicaSetConnection(host='stsia0808.cluster.net', replicaSet = 'rs0') #constr = "con.development.test" constr ="con.preannotation."+str(CUR_CHROM) a = eval(constr) PreAnnotationDB = a line = invar.readline() while line: # print line """ If this is the first line skip """ lin = line.strip().split("\t") if (lin[1]=="Chromosome"): line = invar.readline()
def getMongoInfos(): for uri in config.mongoMongoUri: uristr = str(uri).strip() if len(uristr) <= 1: continue try: conn = pymongo.ReplicaSetConnection(uristr, replicaSet='SwallowCap') #conn.read_preference = pymongo.read_preferences.ReadPreference.SECONDARY_PREFERRED conn.read_preference= pymongo.ReadPreference.SECONDARY except: logger.error('[MongoInfo] Connect to ' + uristr + ' in replicaSet = SwallowCap failed. Try Normal connect.') # print ('[MongoInfo] Connect to ' + uristr + ' in replicaSet = SwallowCap failed. Try Nomal connect.') try: conn = pymongo.Connection(uri) except: logger.error('[MongoInfo] Connect to ' + uristr + ' failed.') # print ('[MongoInfo] Connect to ' + uristr + ' failed.') continue names = conn.database_names() MPMap = dict() for name in names: if str(name).startswith('msg#'): domains = str(name).split('#') topic = domains[1] info = next(conn[name]['c'].find().sort('_id', -1).limit(1), None) # 新增代码,如果已经记录过该topic上次最新消息,则计算产生了多少新消息,否则将该topic数量作为产生了多少新消息 if(config.logMongoTopic.__contains__(topic.lower())): lastPeroidCount = conn[name]['c'].find({'_id' : {'$gt' : config.logMongoTopic[topic.lower()]}}).count() else: lastPeroidCount = conn[name]['c'].count() mongoStatInfo = MongoStatInfo() mongoStatInfo.topicName = topic mongoStatInfo.lastPeriodAmount = lastPeroidCount mongoStatInfos.add(mongoStatInfo) # 新增结束 if info == None: MPMap[topic] = None config.logMongoTopic[topic.lower()] = Timestamp(int(time.time()), 1) else: MPMap[topic] = info['_id'] config.logMongoTopic[topic.lower()] = info['_id'] for name in names: if str(name).startswith('ack#'): domains = str(name).split('#') topic = domains[1] cid = domains[2] info = next(conn[name]['c'].find().sort('_id', -1).limit(1), None) if info == None: pass else: mi = MongoInfo() mi.topic = topic mi.consumerId = cid mid = info['_id'] strmid = str(time.ctime(mid.time)) + ' NO.' + str(mid.inc) mi.consumedMax = strmid if MPMap.has_key(topic): mid = MPMap[topic] if mid != None: strmid = str(time.ctime(mid.time)) + ' NO.' + str(mid.inc) mi.producedMax = strmid mi.delayTime = int(mid.time) - int(info['_id'].time) else: mi.producedMax = None mi.delayTime = 0 else: mi.producedMax = None mi.delayTime = 0 mongoInfos.add(mi) # 新增代码,用以统计mongo中consumer消费记录 lastPeroidAmount = 0 mongoConsumedStatInfo = MongoConsumedStatInfo() if config.logMongoConsumeStatus.__contains__(topic.lower()): if config.logMongoConsumeStatus[topic.lower()].__contains__(cid.lower()): lastPeroidMsgId = config.logMongoConsumeStatus[topic.lower()][cid.lower()] if names.__contains__('msg#' + topic): lastPeroidAmount = conn['msg#' + topic]['c'].find({'_id' : {'$gt' : lastPeroidMsgId, '$lte' : info['_id']}}).count() else: if names.__contains__('msg#' + topic): lastPeroidAmount = conn['msg#' + topic]['c'].find({'_id' : {'$gt' : Timestamp(int(info['_id'].time) - 360, 0), '$lte': Timestamp(int(info['_id'].time), info['_id'].inc)}}).count() else: if names.__contains__('msg#' + topic): lastPeroidAmount = conn['msg#' + topic]['c'].find({'_id' : {'$gt' : Timestamp(int(info['_id'].time) - 360, 0), '$lte': Timestamp(int(info['_id'].time), info['_id'].inc)}}).count() mongoConsumedStatInfo.consumerId = cid mongoConsumedStatInfo.topicName = topic mongoConsumedStatInfo.lastPeriodAmount = lastPeroidAmount mongoConsumeStatInfos.add(mongoConsumedStatInfo) if not dict(config.logMongoConsumeStatus).__contains__(topic.lower()): config.logMongoConsumeStatus[topic.lower()] = dict() config.logMongoConsumeStatus[topic.lower()][cid.lower()] = info['_id'] # 新增结束 for name in names: if str(name).startswith('heartbeat#'): ip = str(str(name)[10:]).strip().replace('_', '.') if config.consumerServersIp.__contains__(ip): latestMsgId = next(conn[name]['c'].find().sort('_id', -1).limit(1), None)['t'] lastHeartbeatTime = int(time.mktime(latestMsgId.timetuple())) now = int(time.time()) if now - lastHeartbeatTime > 60 * 60 * 8 + 60: consumerServerStat[ip] = False else: consumerServerStat[ip] = True MPMap.clear() conn.close()
__author__ = 'emilio.torrens' import pymongo connection = pymongo.ReplicaSetConnection('192.168.1.218:27017', replicaSet='devSet') instances = connection.instances print instances
except: temp.append(line[4:]) all_vars.append(temp) line = invar.readline() invar.close() var_it = 0 tracker = 0 content_it = 0 var_res = [] var_annot = [] content = [] #conn = pymongo.Connection(host='10.128.128.60')#host='10.128.143.255') #conn = pymongo.Connection(host='stsia0808.cluster.net') conn = pymongo.ReplicaSetConnection('hadoop00-adm.cluster.net',replicaSet='rs0') a =eval("conn.development."+str(CUR_CHROM)) PreAnnotationDB = a counter_found = 0 while var_it < len(all_vars): hap = None chrom = None #all_vars[var_it][0] begpos = None #int(all_vars[var_it][1]) endpos = None #int(all_vars[var_it][2]) vartype = None #all_vars[var_it][4] ref = None #all_vars[var_it][5] obs = None #all_vars[var_it][6]