def entityProperties(entity): # Conexion a Mongo conn = Connection() db = conn.grupo10_taller4 colTypeQueryCache = db.typeQueryCache colDescribeQueryCache = db.describeQueryCache # Revisar si la consulta existe en el cache describeCache = colDescribeQueryCache.find({"term":entity}) if (describeCache.count() > 0): print "Describe cache hit for %s" % entity for cacheResult in describeCache: return cacheResult["results"] sparql = SPARQLWrapper("http://dbpedia.org/sparql") sparql.setQuery(""" PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> DESCRIBE ?object WHERE { ?object rdfs:label '%s'@en . } """ % entity ) sparql.setReturnFormat(JSON) results = sparql.query().convert() queryRes = [{"term":entity,"results":results}] colDescribeQueryCache.insert(queryRes) conn.disconnect() return results
class DB(object): def __init__(self, uri): if not uri.startswith('mongodb://'): uri = environ[uri] p = urlparse(uri) self.URI = { 'host': p.hostname, 'port': p.port, 'username': p.username, 'password': p.password, 'database_name': p.path[1:] } self._uri = uri self._conn = None def __collection(self, collection): if not self._conn: self._conn = Connection(self._uri) return self._conn[self.URI['database_name']][collection] def __getattr__(self, collection): return self.__collection(collection) def __getitem__(self, collection): return self.__collection(collection) def __del__(self): if self._conn: self._conn.disconnect()
def setup_database(): # Disconnect from the default mongo db, and use a test db instead. conn = Connection() checkin_service = conn["checkin_service"] results = conn["results"] for collection in ['splice_server', 'marketing_product_usage']: if drop: print('DROPPING') checkin_service.drop_collection(collection) print 'importing %s collection' % collection call(['mongoimport', '--db', 'checkin_service', '-c', collection, '--file', '%s.json' % os.path.join(DUMP_DIR, collection)] ) """ for collection in ['marketing_report_data']: results.drop_collection(collection) print 'importing %s collection' % collection call(['mongoimport', '--db', 'results', '-c', collection, '--file', '%s.json' % os.path.join(DUMP_DIR, collection)] ) """ conn.disconnect()
def repair(conf, options): '''fixes issues created by myself. Currently, only orphan files and item links are detected and automatically removed.''' con = Connection(conf['MONGODB_HOST'], conf['MONGODB_PORT']) db = con[conf['MONGODB_NAME']] fs = GridFS(db) objs = [obj['_id'] for obj in fs._GridFS__files.find()] meta = [cur['_id'] for cur in db.items.find()] if objs != meta: # 1. metadata has some files missing, no repair possible diff1 = filter(lambda i: not i in objs, meta) diff2 = filter(lambda i: not i in meta, objs) for item in diff1: print 'removing metadata for `%s`' % item db.items.remove(item) # 2. metadata is missing, but file is there. Recover possible, but not implemented #win for item in diff2: print 'removing GridFS-File `%s`' % item objs.remove(item) # rebuild accounts items, when something changed for cur in db.accounts.find(): if str(cur['_id']).startswith('_'): continue _id, items = cur['_id'], cur['items'] items = filter(lambda i: i in objs, items) db.accounts.update({'_id': _id}, {'$set': {'items': items}}) con.disconnect()
def dropCollectionInDB(self, DBName, collection): c = Connection('localhost',27017) dbh = c[DBName] assert dbh.connection == c dbh[collection].drop() c.disconnect() return
def dropCollectionInDB(self, DBName, collection): c = Connection('localhost', 27017) dbh = c[DBName] assert dbh.connection == c dbh[collection].drop() c.disconnect() return
def activate(conf, options, args): '''When PUBLIC_REGISTRATION is set to false, you have to activate registered accounts manually by invoking "activate $email"''' con = Connection(conf['MONGODB_HOST'], conf['MONGODB_PORT']) accounts = con[conf['MONGODB_NAME']].accounts if len(args) == 2: acc = accounts.find_one({'email': args[1]}) if not acc: print '`%s` does not exist' % args[1] sys.exit(1) elif acc['activated_at'] != None: print '`%s` already activated' % args[1] else: act = {'activated_at': strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())} accounts.update({'_id': acc['_id']}, {'$set': act}) print '`%s` activated' % args[1] else: inactivated = [acc for acc in accounts.find() if not acc.get('activated_at', True)] if not inactivated: print 'no pending non-activated accounts' for acc in inactivated: print '%s [%s]' % (acc['email'].ljust(16), acc['created_at']) con.disconnect()
def __init__(self, tweet, host = 'localhost', database = 'twitter_grafico', displayed = False): self.id = tweet.id self.user = tweet.user.id self.message = tweet.text #The date comes in GTM self.created_at = datetime.strptime(tweet.created_at,"%a %b %d %H:%M:%S +0000 %Y") self.url = self.get_url(self.message) User(tweet.user.id, tweet.user.screen_name, tweet.user.followers_count).save() connection = Connection(host = host) result = connection[database].tweets.find_one({"id":self.id}) self.image = None self.keywords = None self.text = None self.massive = False if displayed: self.displayed = True self.check_massive_users(tweet.user) if result: self.image = result.get('image') self.text = result.get('text') else: if self.url: params = self.get_representative_image_keywords(self.url) self.image = params.get('image') self.keywords = params.get('keywords') self.text = params.get('text') self.save(host = host, database = database) connection.disconnect()
def registerUser(self,params): # Conecta con la base de datos conn_ = Connection() db_ = conn_['blogomatic'] err = False try: # Se autentifica para acceder a la base de datos db_.authenticate('blogomatic','bnBN98cv.mongo') inv_ = db_['invitations'] # Nos aseguramos de que tanto el email como la clave cumplen una longitud minima if len(params['email']) < 3 or not('@' in params['email'] ) or len(params['password']) < 6 : err = True else: uDoc = inv_.find_one({'email': params['email']}) if uDoc == None: # Guardamos el usuario y su clave y limitamos su numero maximo de blogs a 20 uDoc = {'email': params['email'],'password': params['password'],'instances': 20} inv_.save(uDoc) else: err = True except TypeError: err = True except: err = True finally: db_.logout() conn_.disconnect() return err
class MongoPersistence(): def __init__(self): pass def setup(self): self.__connection = Connection(db_host, db_port) db = self.__connection.td self.__tdlist_collection = db.tdlist def teardown(self): self.__connection.disconnect() def persist(self, tdlist): #tdlist_doc = {"date" : "2012-FEB-06", # "tds": [ {"status": "DONE", "number": 1, "text" : "elso feladat"}, # {"status": "TODO", "number": 2, "text" : "masodik feladat"}]} self.__tdlist_collection.insert(tdlist_todict(tdlist)) def search(self, date_str): result = self.__tdlist_collection.find_one({"date": date_str}) return result def remove(self, date_str): self.__tdlist_collection.remove({"date": date_str})
def uploadStudy(studyid): #path for study files path = 'Studies/' + studyid +'/' listing = os.listdir(path) connection = Connection(HOST, PORT) db = connection.MINE #for all studies (exclude the file log.txt) for file in listing: if not file == 'log.txt' and not file == 'logx.txt': try: #open the file logger.info("trying to open " + file) f = open(path + file) count = 0 #for each line after the header logger.info("trying to read lines") for line in f: count = count + 1 if count > 1: logger.debug("reading line " + str(count)) try: #insert lines #logging.debug("splitting line") cells = line.split("\t") #logging.debug("stripping newline char") cells[-1] = cells[-1].rstrip() #logging.debug("uploading line") #uploadLine(studyid, cells[0], cells[1:]) line = {"id": cells[0], "canonical_id":cells[0].upper(), "data": cells[1:]} db[studyid].insert(line) except: logging.error("line " + count + " could not be read") else: logger.debug("header line read") gsm_ids = line.rstrip().split("\t") gsm_ids = gsm_ids[1:] n = 1 for id in gsm_ids: more_ids = id.split(";") for x in more_ids: db[studyid+"order"].insert({"no":n,"GSM":x}) n+= 1 #close file logger.info("lines successfully read") logger.info("closing " + file) f.close() #mark request queued markRequestQueued(studyid) db[studyid].ensure_index("canonical_id",1) except: logger.error("Error parsing file " + file + "See last line to see where program failed") Connection.disconnect(connection)
def retrieveAttrData(studyid): #connect connection = Connection(HOST, PORT) db = connection.MINE # get list of data attributes = list(db[studyid+'GSM'].find_one().keys()) wantedAttrs = {} for attribute in attributes: x = db[studyid+'GSM'].find().distinct(attribute) if len(x) > 1 and not attribute == '_id' : wantedAttrs[attribute] = 2 wantedAttrs["_id"] = 0 times = db[studyid].find_one()['data'] retlist=[] n=1 specdict = {} for x in times: next = db[studyid +"order"].find({"no":n}).distinct("GSM")[0] specdict["geo_accession"] = next ret = db[studyid+"GSM"].find(spec=specdict, fields=wantedAttrs) for r in ret: retlist.append({"sample":r}) #disconnect n+=1 Connection.disconnect(connection) return retlist
def searchBlog(self,params): # Conecta con la base de datos conn_ = Connection() db_ = conn_['blogomatic'] ret = {} try: # Se autentifica para acceder a la base de datos db_.authenticate('blogomatic','bnBN98cv.mongo') blogs_ = db_['blogs'] keywords = params['search'].split(' ') for keyword in keywords: # Hace una busqueda por palabra clave ignorando mayusculas o minusculas for blog in blogs_.find({'title': {'$regex': keyword, '$options':'i'}}): ret[blog['url']] = blog['title'] except TypeError: ret = {} except: ret = {} finally: db_.logout() conn_.disconnect() return ret
def activate(conf, options, args): '''When PUBLIC_REGISTRATION is set to false, you have to activate registered accounts manually by invoking "activate $email"''' con = Connection(conf['MONGODB_HOST'], conf['MONGODB_PORT']) accounts = con[conf['MONGODB_NAME']].accounts if len(args) == 2: acc = accounts.find_one({'email': args[1]}) if not acc: print '`%s` does not exist' % args[1] sys.exit(1) elif acc['activated_at'] != None: print '`%s` already activated' % args[1] else: act = {'activated_at': strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())} accounts.update({'_id': acc['_id']}, {'$set': act}) print '`%s` activated' % args[1] else: inactivated = [ acc for acc in accounts.find() if not acc.get('activated_at', True) ] if not inactivated: print 'no pending non-activated accounts' for acc in inactivated: print '%s [%s]' % (acc['email'].ljust(16), acc['created_at']) con.disconnect()
def index(): connection = Connection(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] _id = request.args.get('_id','') selected=None try: selected = collection.find({'_id':ObjectId(_id)}).next() except: pass items = collection.find({'visible':'True'}).sort('name') connection.disconnect() fields=deepcopy(FIELDS) editform=False if selected is not None: editform=True for field in fields: try: fieldname = field['name'] field['default_value']=selected[fieldname] except: field['default_value']='' return render_template("index.html", items=items, selected=selected, fields=fields, editform=editform, )
def retrieveChartData(studyid, gene_x, gene_y): #connect connection = Connection(HOST, PORT) db = connection.MINE # get list of data x = list( db[studyid].find({"canonical_id":gene_x.upper()}) )[0]['data'] y = list( db[studyid].find({"canonical_id":gene_y.upper()}) )[0]['data'] n = 1 ret = [] for t in x: entry = {} next = db[studyid +"order"].find({"no":n}).distinct("GSM")[0] data = db[studyid +"GSM"].find({"geo_accession":next}).distinct("characteristics_ch1") #change list of characteristics to dict sample = {} for item in data: pair = item.split(": ") sample[pair[0]] = getCorrectType(pair[1]) sample['id'] = next entry["y"] = getCorrectType(y[n-1]) entry["x"] = getCorrectType(x[n-1]) entry["sample"] = sample ret.append(entry) n+=1 #disconnect Connection.disconnect(connection) return ret
def writeAttrFile(f, studyid): #connection to the database connection = Connection(HOST, PORT) db = connection.MINE attributes = db[studyid+'GSM'].find_one().keys() samples = db[studyid+'order'].find().distinct('GSM') #write attributes to file in form attribute_(attr) # # # # where #'s are the sample's attributes and attr is the attribute name for attr in attributes: if not attr == '_id' and not attr == 'GSM': f.write('attribute_'+attr) for sample in samples: sample = sample.rstrip() samp = db[studyid+'GSM'].find({"geo_accession":sample}).distinct(attr) if samp[0]: #if integer or float cast to string try: text = str(samp[0]) #else encode into unicode except: text = samp[0].encode("utf8") f.write('\t' + text) f.write('\n') Connection.disconnect(connection)
def test_05_save_entity(self): connection = MongoDatabaseAPI("localhost", DB_NAME) connection.connect() # Given wrong record type to save self.assertRaises(BIValueError, connection.save_entity, MongoDatabaseAPI.ET_RESOURCE, "not a dict value") # Insert new record("_id" exist in incoming dictionary but not in DB) new_record = {"_id" : objectid.ObjectId("5010cb8137adc7191b000000"), "res_type" : "printer", "res_status" : "active", "owner" : "Priocom", "external_system" : "", "description" : "Printer in the Brannigan Team workroom", "additional_parameters" : {"Vendor" : "Canon", "model" : "iR1018"}} id = connection.save_entity(MongoDatabaseAPI.ET_RESOURCE, new_record) conn = Connection() result = conn[DB_NAME][MongoDatabaseAPI.ET_RESOURCE].find_one({'_id': objectid.ObjectId(id)}) self.assertEqual(new_record, result) # Insert another new record(no "_id" key in incoming dictionary) new_record = {"res_type" : "printer", "res_status" : "active", "owner" : "Priocom", "external_system" : "", "description" : "Printer in the Brannigan Team workroom", "additional_parameters" : {"Vendor" : "Canon", "model" : "iR1018"}} id = connection.save_entity(MongoDatabaseAPI.ET_RESOURCE, new_record) conn = Connection() result = conn[DB_NAME][MongoDatabaseAPI.ET_RESOURCE].find_one({'_id': objectid.ObjectId(id)}) self.assertEqual(new_record, result) # Update existing record(change res_status and additional_parameters:model) new_record = {"_id": id, "res_type" : "printer", "res_status" : "down", "owner" : "Priocom", "external_system" : "", "description" : "Printer in the Brannigan Team workroom", "additional_parameters" : {"Vendor" : "Canon", "model" : "iR1022"}} id = connection.save_entity(MongoDatabaseAPI.ET_RESOURCE, new_record) result = conn[DB_NAME][MongoDatabaseAPI.ET_RESOURCE].find_one({'_id': objectid.ObjectId(id)}) self.assertEqual(new_record, result) conn.disconnect() connection.close()
class Database(): def __init__(self): config = Config() self.host = config.parser['database']['host'] self.port = int(config.parser['database']['port']) self.db_name = config.parser['database']['name'] self.conn = Connection(self.host, self.port) self.db = self.conn[self.db_name] def insert(self, collection=None, *args): if collection: return self.db[collection].insert(args[0]) def select(self, collection=None, *args): if collection: return self.db[collection].find(args[0]) def select_one(self, collection=None, *args): if collection: return self.db[collection].find_one(args[0]) def delete(self, collection=None, *args): if collection: return self.db[collection].remove(args[0]) def update(self, collection=None, *args): if collection: return self.db[collection].update(args[0]) def close(self): self.conn.disconnect() #Return database cursor def connect(self): return Connection(self.host, self.port)
def do_host_status(self,mongo): con = Connection(host=mongo['host'], port=mongo['port'], slave_okay=True) db = con['admin'] if mongo['user'] and mongo['password']: db.authenticate(mongo['user'], mongo['password']) server_status = db.command('serverStatus') # operations for k, v in server_status['opcounters'].items(): self.submit('mongo_total_operations', k, v, host=mongo['host'], port=mongo['port']) # memory for t in ['resident', 'virtual', 'mapped']: self.submit('mongo_memory', t, server_status['mem'][t], host=mongo['host'], port=mongo['port']) # connections self.submit('mongo_connections', 'connections', server_status['connections']['current'],host=mongo['host'], port=mongo['port']) # locks if mongo['lockTotalTime'] is not None and mongo['lockTime'] is not None: #collectd.warning( "total %d - %d / %d - %d " % (server_status['globalLock']['totalTime'], self.lockTotalTime, server_status['globalLock']['totalTime'], self.lockTime)) if mongo['lockTime']==server_status['globalLock']['lockTime']: value=0.0 else: value=float (server_status['globalLock']['lockTime'] - mongo['lockTime']) *100.0 / float(server_status['globalLock']['totalTime'] - mongo['lockTotalTime'] ) #collectd.warning( "Submitting value %d " % value) self.submit('mongo_percent', 'lock_ratio', value, host=mongo['host'], port=mongo['port']) mongo['lockTotalTime']=server_status['globalLock']['totalTime'] mongo['lockTime']=server_status['globalLock']['lockTime'] # indexes accesses = server_status['indexCounters']['btree']['accesses'] misses = server_status['indexCounters']['btree']['misses'] if misses: self.submit('mongo_cache_ratio', 'cache_misses', accesses / float(misses),host=mongo['host'], port=mongo['port']) else: self.submit('mongo_cache_ratio', 'cache_misses', 0, host=mongo['host'], port=mongo['port']) for mongo_db in mongo['db']: db = con[mongo_db] if mongo['user'] and mongo['password']: db.authenticate(mongo['user'], mongo['password']) db_stats = db.command('dbstats') # stats counts self.submit('mongo_counter', 'object_count', db_stats['objects'], mongo_db, host=mongo['host'], port=mongo['port']) self.submit('mongo_counter', 'collections', db_stats['collections'], mongo_db, host=mongo['host'], port=mongo['port']) self.submit('mongo_counter', 'num_extents', db_stats['numExtents'], mongo_db, host=mongo['host'], port=mongo['port']) self.submit('mongo_counter', 'indexes', db_stats['indexes'], mongo_db, host=mongo['host'], port=mongo['port']) # stats sizes self.submit('mongo_file_size', 'storage', db_stats['storageSize'], mongo_db, host=mongo['host'], port=mongo['port']) self.submit('mongo_file_size', 'index', db_stats['indexSize'], mongo_db, host=mongo['host'], port=mongo['port']) self.submit('mongo_file_size', 'data', db_stats['dataSize'], mongo_db, host=mongo['host'], port=mongo['port']) con.disconnect()
def save(self, host = 'localhost', database = 'twitter_grafico'): connection = Connection(host = host) result = connection[database].users.find_one({"user_id":self.user_id}) if result: connection[database].users.update({'user_id': self.user_id}, self.__dict__) else: connection[database].users.save(self.__dict__) connection.disconnect()
def getNumberOfColumns(studyid): connection = Connection(HOST, PORT) db = connection.MINE #get data from database x = len(db[studyid].find_one()['data']) Connection.disconnect(connection) return x
def removeByNumber(studyid): #connect connection = Connection(HOST, PORT) db = connection.MINE #remove studyid request db.request.remove({"gse":studyid}) #disconnect Connection.disconnect(connection)
def getDateRequested(studyid): #connect connection = Connection(HOST, PORT) db = connection.MINE ret = db.request.find({"gse":studyid}).distinct("datetime")[0] ret = ret.strftime("%B %d, %Y") Connection.disconnect(connection) return ret
def markRequestQueued(studyid): #connect connection = Connection(HOST, PORT) db = connection.MINE #update request db.request.update({"gse":studyid,}, {"$set": {"queued":True}}) #disconnect Connection.disconnect(connection)
def createBlog(self,params): # Conecta con la base de datos conn_ = Connection() db_ = conn_['blogomatic'] err = False try: # Se autentifica para acceder a la base de datos db_.authenticate('blogomatic','bnBN98cv.mongo') inv_ = db_['invitations'] blogs_ = db_['blogs'] # Nos aseguramos de que ningun campo se encuentra vacio if params['title'] == "" or params['url'] == "" or params['keywords'] == "" or (' ' in params['url']) : err = True else: # Comprobamos el usuario uDoc = inv_.find_one({'email': params['email']}) if uDoc != None and uDoc['instances'] > 0: # Comprobamos la clave password = uDoc['password'] if password == params['password']: # Ahora podemos comprobar que no existe un blog con el mismo titulo y url bDocTitle = blogs_.find_one({'title': params['title']}) bDocUrl = blogs_.find_one({'url': params['url']}) # Si es vacio significa que podemos crear el solicitado if bDocTitle == None and bDocUrl == None: # Llama al script del sistema para crear el blog # Saca el path del script del fichero de configuracion proc = Popen('/opt/blog-o-matic/bin/create_blog.sh "%s" "%s" "%s" >> /opt/blog-o-matic/var/log/create_blog.log 2>&1' % (params['title'], params['url'], params['keywords']), shell=True, stdout=PIPE) bDoc = {'email': params['email'],'url': params['url'],'title': params['title']} blogs_.save(bDoc) inst = uDoc['instances'] uDoc['instances'] = inst - 1 inv_.save(uDoc) else: err = True else: err = True else: err = True except TypeError: err = True except: err = True finally: db_.logout() conn_.disconnect() return err
def markRequestSent(studyid): #connect connection = Connection(HOST, PORT) db = connection.MINE #update request db.request.update({"gse":studyid,}, {"$set": {"queued":True}}) #disconnect Connection.disconnect(connection) logging.info("study " + studyid + "has been successfully queued")
def getWaitingStudyList(): #connect connection = Connection(HOST, PORT) db = connection.MINE # get list of distinct processed GSEids x = db.request.find({"processed":False, "queued":False}).distinct("gse") #disconnect Connection.disconnect(connection) return x
def uploadLine(studyid, varname, floats): #connect connection = Connection(HOST, PORT) db = connection.MINE #create and upload line line = {"id": varname,"canonical_id":varname.upper(), "data": floats} db[studyid].insert(line) #disconnect Connection.disconnect(connection)
def save(self, host = 'localhost', database = 'twitter_grafico'): connection = Connection(host = host) result = connection[database].tweets.find_one({"id":self.id}) if result: self._id = result.get("_id") connection[database].tweets.update({'_id': result.get("_id")}, self.__dict__) else: connection[database].tweets.save(self.__dict__) connection.disconnect()
def uploadProcessedData(studyid, var1, var2, mic, pcc): # connect connection = Connection(HOST, PORT) db = connection.MINE #upload data data = {"var1": var1,"var2":var2, "mic": mic, "pcc": pcc} db[studyid].insert(data) #disconnect Connection.disconnect(connection)
def save(self, host = 'localhost', database = 'twitter_grafico'): connection = Connection(host = host) result = connection[database].accounts.find_one({"username":self.username}) if result: self.__dict__['username'] = result.get("username") connection[database].accounts.update({'username': result.get("username")}, self.__dict__) else: connection[database].accounts.save(self.__dict__) connection.disconnect()
def insKeywordCountinDB(self, indict, DBName, collection): c = Connection('localhost', 27017) dbh = c[DBName] dbh.connection == c for keyword, count in indict[0:40]: dbh[collection].save({"keyword": keyword, "count": count}) c.disconnect() return
def getNumberOfRows(studyid): connection = Connection(HOST, PORT) db = connection.MINE #get data from database ret = db[studyid].find().count() Connection.disconnect(connection) return ret
def donorschoose_projects(): connection = Connection(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] projects = collection.find(fields=FIELDS) json_projects = [] for project in projects: json_projects.append(project) json_projects = json.dumps(json_projects, default=json_util.default) connection.disconnect() return json_projects
def test_projects(): connection = Connection(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] projects = collection.find({"harvested_at" : {"$gte": start}},fields=FIELDS).limit(100000) json_projects = [] for project in projects: json_projects.append(project) json_projects = json.dumps(json_projects, default=json_util.default) connection.disconnect() return json_projects
def execute(self): """ return list of latest data. specified group_name (and specified command_name) """ mongos = { 'mongo_host': self.config.get('server', 'mongo_host'), 'mongo_port': self.config.get('server', 'mongo_port'), 'mongo_dbs': self.config.get('server', 'mongo_dbs'), 'mongo_collection': self.config.get('server', 'mongo_collection'), } connection = Connection( mongos['mongo_host'], int(mongos['mongo_port']), ) collection = connection[mongos['mongo_dbs']][ mongos['mongo_collection']] base_find_condition = { 'group_name': self.group_name, 'visible': 'True', } if self.command_name is not None: base_find_condition.update({ 'command_name': self.command_name, }) # find hosts in group condition = deepcopy(base_find_condition) host_names = collection.find(condition).distinct('host_name') host_names.sort() # find latest result on each host results = [] for host_name in host_names: condition = deepcopy(base_find_condition) condition.update({'host_name': host_name}) latests = collection.find(condition).sort('execute_at', DESCENDING).limit(1) latest_execute_at = '' for latest in latests: latest_execute_at = latest['execute_at'] break condition = deepcopy(base_find_condition) condition.update({'host_name': host_name}) condition.update({'execute_at': latest_execute_at}) documents = collection.find(condition).sort( 'command_name', ASCENDING) for document in documents: results.append(document) # finalize connection.disconnect() return results
def retKeywordCountFromDB(self, DBName, collection): c = Connection('localhost', 27017) dbh = c[DBName] assert dbh.connection == c outdict = dict() cursor = dbh[collection].find() for doc in cursor: outdict[doc["keyword"]] = doc["count"] c.disconnect() return outdict
def getMetroEvents(metro_id): conn = Connection() db = conn.ts_db events = db.events one_week = str(date.today() + relativedelta(days=+9)) print "THIS IS ONE WEEK: " + one_week all_events = [] for e in events.find({"date": {"$lt": one_week}}, {'_id': 0}): all_events.append(e) conn.disconnect() return json.dumps(all_events)
def entityType(entity): # Conexion a Mongo conn = Connection() db = conn.grupo10_taller4 colTypeQueryCache = db.typeQueryCache colDescribeQueryCache = db.describeQueryCache # Revisar si la consulta existe en el cache typeCache = colTypeQueryCache.find({"term":entity}) if (typeCache.count() > 0): print "Type cache hit for %s" % entity for cacheResult in typeCache: return cacheResult["mainType"] sparql = SPARQLWrapper("http://dbpedia.org/sparql") sparql.setQuery(""" PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> SELECT DISTINCT ?s ?o ?class WHERE { ?someobj ?p ?s . ?s rdfs:label '%s' @en . ?s rdfs:label ?o . ?s rdfs:comment ?comment . ?s rdf:type ?class . FILTER (!regex(str(?s), '^http://dbpedia.org/resource/Category:')). FILTER (!regex(str(?s), '^http://dbpedia.org/resource/List')). FILTER (!regex(str(?s), '^http://sw.opencyc.org/')). FILTER (lang(?o) = 'en'). FILTER (!isLiteral(?someobj)). } Limit 20 """ % entity) sparql.setReturnFormat(JSON) results = sparql.query().convert() for result in results["results"]["bindings"]: currentValue = result["class"]["value"] if "Person" in currentValue: mainType = "Person" break elif "Place" in currentValue: mainType = "Place" break elif "Organization" in currentValue: mainType = "Organization" break if mainType == None: mainType = "Thing" queryRes = [{"term":entity,"mainType":mainType,"results":results}] colTypeQueryCache.insert(queryRes) conn.disconnect() return mainType
def testExistInDB(self, DBName, article): c = Connection('localhost', 27017) dbh = c[DBName] assert dbh.connection == c cursor = dbh[article].find() cnt = cursor.count() if cnt == 0: print article, "does not exist!!!" c.disconnect() return 0 else: print article, "exist" c.disconnect() return 1
class MongoHandler(logging.Handler): def __init__(self, level=logging.NOTSET, **kwargs): super(MongoHandler, self).__init__(level) self.host = kwargs.get('host', settings.database.host) self.port = kwargs.get('port', settings.database.port) self.database_name = kwargs.get('database', settings.database.name) self.collection_name = kwargs.get('collection', 'logs') self.fail_silently = kwargs.get('fail_silently', False) self.formatter = kwargs.get('formatter', MongoFormatter()) self.connection = None self.database = None self.collection = None self.connect() def connect(self): """ Connect to the Mongo database. """ try: self.connection = Connection(host=self.host, port=self.port) except PyMongoError: if self.fail_silently: return else: raise self.database = self.connection[self.database_name] self.collection = self.database[self.collection_name] def close(self): """ Close the connection to the Mongo database. """ if self.connection is not None: self.connection.disconnect() self.connection = None def emit(self, record): """ Insert log record into Mongo database """ if self.collection is not None: try: self.collection.insert(self.format(record)) except Exception: if not self.fail_silently: self.handleError(record)
def api_values(group_name, field_name): connection = Connection(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] filter_condition = {'visible': 'True'} if group_name != '*': filter_condition.update({'group_name': group_name}) items = collection.find(filter_condition).sort(field_name).distinct( field_name) jsondata = list() for item in items: if item == '': continue jsondata.append(item) jsondata.sort() connection.disconnect() return jsonify({'values': jsondata})
def load_config(self): from pymongo import Connection self._config = {} name_by_id = dict( (sconf['id'], name) for name, sconf in self._topology.items()) for master in self._coll.find(): if master['_id'] != 'master_checkpoint': self._config[name_by_id[master['_id']]] = master master_uri = self._topology[name_by_id[master['_id']]]['uri'] conn = Connection(master_uri) coll = conn.local[MMM_DB_NAME] for cp in coll.find(dict(_id='master_checkpoint')): self._master_checkpoint = cp['ts'] conn.disconnect()
def info(conf): '''A basic, incomplete short info. Displays overall file size and account counts.''' con = Connection(conf['MONGODB_HOST'], conf['MONGODB_PORT']) db = con[conf['MONGODB_NAME']] fs = GridFS(db) overall_file_size = sum([f['length'] for f in fs._GridFS__files.find()]) inactivated = [ acc for acc in db.accounts.find() if not acc.get('activated_at', True) ] print fs._GridFS__files.count(), 'files [%s]' % ppsize(overall_file_size) print db.accounts.count() - 1, 'accounts total,', len( inactivated), 'not activated' con.disconnect()
def cprofile_main(): from pymongo import Connection connection = Connection() connection.drop_database('timeit_test') connection.disconnect() from mongoengine import Document, DictField, connect connect("timeit_test") class Noddy(Document): fields = DictField() for i in xrange(1): noddy = Noddy() for j in range(20): noddy.fields["key" + str(j)] = "value " + str(j) noddy.save()
def account(conf, options, args): '''View details or summary of all accounts.''' con = Connection(conf['MONGODB_HOST'], conf['MONGODB_PORT']) db = con[conf['MONGODB_NAME']] fs = GridFS(db) if options.all: query = None elif len(args) == 2: query = { '_id': int(args[1]) } if args[1].isdigit() else { 'email': args[1] } else: log.error('account <email or _id> requires a valid email or _id') sys.exit(1) for acc in db.accounts.find(query): if str(acc['_id']).startswith('_'): continue print '%s [id:%s]' % (acc['email'], acc['id']) for key in acc: if key in ['email', '_id', 'id']: continue if key == 'items': try: size = sum([fs.get(_id).length for _id in acc['items']]) except NoFile: log.warn('Account `%s` has some files missing:', _id) # fail safe counting size = 0 missing = [] for i in acc['items']: if not fs.exists(i): missing.append(i) else: size += fs.get(i).length print ' size: %s' % ppsize(size) print ' %s: %s' % (key, acc[key]) if options.all: print db.accounts.count() - 1, 'accounts total' # -1 for _autoinc con.disconnect()
def get_opcounters(self): """ doctest convenience function to retrieve ``opcounters`` from ``serverStatus`` for each configured node, aggregate the results and compute some custom counters on them. The purpose is to prove that the ``write`` operations are dispatched to the ``PRIMARY``, while the ``read`` operations are dispatched to the ``SECONDARIES``. """ from pymongo import Connection stats = { 'custom': {}, 'hosts': {}, } for port in self.storage_ports: host = '{0}:{1}'.format(self.hostname, port) mongo_conn = Connection(host, safe=True) status = mongo_conn.admin.command('serverStatus') mongo_conn.disconnect() # get all opcounters info = status.get('opcounters') # whether the current host is a PRIMARY or a SECONDARY ismaster = status.get('repl', {}).get('ismaster', False) # record all opcounter data by host stats['hosts'][host] = info # record custom counters: # - how many inserts are done on the PRIMARY # - how many queries are routed to all SECONDARIES stats['custom'].setdefault('primary.insert', 0) stats['custom'].setdefault('secondary.query', 0) if ismaster: stats['custom']['primary.insert'] += info['insert'] else: stats['custom']['secondary.query'] += info['query'] return stats
def api_results(group_name, field_name): connection = Connection(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] filter_condition = {'visible': 'True'} if group_name == '*': return jsonify({'values': ''}, 500) filter_condition.update({'group_name': group_name}) if field_name != '*' and re.match(r'^command_', field_name): filter_condition.update({'command_name': field_name}) host_names = collection.find(filter_condition).distinct('host_name') host_names.sort() items = [] for host_name in host_names: host_filter = {'visible': 'True'} host_filter.update({'group_name': group_name}) host_filter.update({'command_name': field_name}) host_filter.update({'host_name': host_name}) latest = collection.find(host_filter).sort('time', DESCENDING).limit(1).next() items.append(latest) jsondata = list() for item in items: if item == '': continue else: ## output item['output'] = escape(item['output']) ## _id item['_id'] = str(item['_id']) ## time from datetime import datetime from calendar import timegm timetuple = item['time'].timetuple() item['time'] = timegm(timetuple) jsondata.append(item) connection.disconnect() return jsonify({'results': jsondata})
def persistSimulation(self): # TODO include the result from getEnvironmentInformation simulationTotals = { "SimulationId": self.simulationId.encode('utf-8'), "CO2": self.total_CO2_mg, "CO": self.total_CO_mg, "HC": self.total_HC_mg, "PMx": self.total_PMx_mg, "NOx": self.total_NOx_mg, "Fuel": self.total_Fuel_ml, "Noise": self.total_Noise_dBA, "Halting": self.total_Halting_vehicles, "MeanSpeed": self.total_MeanSpeed_m_per_s, "SimulationSteps": self.simulation_steps, "NetworkConfiguration": self.visible_network_agents } self.verbose_display("Simulation Totals: %s", simulationTotals, 1) # Persist to MongoDB cn = Connection('localhost') db = cn.macts metrics = db.metrics metrics.insert(simulationTotals) cn.disconnect()
class Corpus(object): def __init__(self, address = 'localhost', database = '', collection = ''): if len(database) == 0 or len(collection) == 0: print('Need specified name database or collection.') return self.connection = Connection('localhost', 27017) self.db = self.connection[database] self.collection = self.db[collection] def __del__(self): self.connection.disconnect() def append(self, record): """ ディクショナリ形式でデータを追加, . """ try: if self.collection.find(record).count() > 0: pass else: self.collection.insert(record) except bson.errors.InvalidStringData as e: print(e.message) def find(self, queries = None): findresult = {} for i, r in enumerate(self.collection.find(queries)): findresult.setdefault(i, r) return tuple([record[1] for record in findresult.items()]) def update(self, record, new_record): try: self.collection.update(record, new_record) except TypeError as e: print('%s #%s\n\tMessage: %s' % (str(self.__class__.__name__), traceback.extract_stack()[-1][2], e.message)) def exists(self, queries): find_count = 0 for attr, value in queries.items(): result = self.find({ attr: value }) find_count += len(result) return find_count > 0 def remove(self, item): self.collection.remove(item)
def do_server_status(self): con = Connection(host=self.mongo_host, port=self.mongo_port, slave_okay=True) db = con[self.mongo_db[0]] if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) server_status = db.command('serverStatus') version = server_status['version'] at_least_2_4 = V(version) >= V('2.4.0') # operations for k, v in server_status['opcounters'].items(): self.submit('total_operations', k, v) # memory for t in ['resident', 'virtual', 'mapped']: self.submit('memory', t, server_status['mem'][t]) # connections self.submit('connections', 'connections', server_status['connections']['current']) # locks if self.lockTotalTime is not None and self.lockTime is not None: if self.lockTime == server_status['globalLock']['lockTime']: value = 0.0 else: value = float(server_status['globalLock']['lockTime'] - self.lockTime) * 100.0 / float( server_status['globalLock']['totalTime'] - self.lockTotalTime) self.submit('percent', 'lock_ratio', value) self.lockTotalTime = server_status['globalLock']['totalTime'] self.lockTime = server_status['globalLock']['lockTime'] # indexes accesses = None misses = None index_counters = server_status[ 'indexCounters'] if at_least_2_4 else server_status[ 'indexCounters']['btree'] if self.accesses is not None: accesses = index_counters['accesses'] - self.accesses if accesses < 0: accesses = None misses = (index_counters['misses'] or 0) - (self.misses or 0) if misses < 0: misses = None if accesses and misses is not None: self.submit('cache_ratio', 'cache_misses', int(misses * 100 / float(accesses))) else: self.submit('cache_ratio', 'cache_misses', 0) self.accesses = index_counters['accesses'] self.misses = index_counters['misses'] for mongo_db in self.mongo_db: db = con[mongo_db] if self.mongo_user and self.mongo_password: con[self.mongo_db[0]].authenticate(self.mongo_user, self.mongo_password) db_stats = db.command('dbstats') # stats counts self.submit('counter', 'object_count', db_stats['objects'], mongo_db) self.submit('counter', 'collections', db_stats['collections'], mongo_db) self.submit('counter', 'num_extents', db_stats['numExtents'], mongo_db) self.submit('counter', 'indexes', db_stats['indexes'], mongo_db) # stats sizes self.submit('file_size', 'storage', db_stats['storageSize'], mongo_db) self.submit('file_size', 'index', db_stats['indexSize'], mongo_db) self.submit('file_size', 'data', db_stats['dataSize'], mongo_db) con.disconnect()
class MonitorCronJob(object): def __init__(self): self.mongo_host = MONGO_HOST self.mongo_port = MONGO_PORT self.redis_host = REDIS_HOST self.redis_port = REDIS_PORT self.rmap = REDIS_MAP self.emap = EXTRA_MAP self.ulist = '%s:start_urls' % SPIDER self.dupefilter_key = '%s:dupefilter' % SPIDER self._connectDB() self._connectRedis() self._cleanRedis() def __del__(self): self._disconnectDB() def _connectDB(self): self.c = Connection(self.mongo_host, self.mongo_port) self.db = self.c[TASK_DB] self.collection = self.db[TASK_COLLECTION] print "[log] Connect to MongoDB %s:%s" % (self.mongo_host, self.mongo_port) def _disconnectDB(self): self.c.disconnect() del self.c, self.db, self.collection print "[log] Disconnect from MongoDB %s:%s" % (self.mongo_host, self.mongo_port) def _alive(self): return True if self.c and self.c.alive() else False def _connectRedis(self): self.r = redis.Redis(host=self.redis_host, port=self.redis_port) print "[log] Connect to Redis %s:%s" % (self.redis_host, self.redis_port) def _cleanRedis(self): self.r.delete(self.rmap) print "[log] Clean url hash map of %s" % self.rmap self.r.delete(self.emap) print "[log] Clean extra url hash map of %s" % self.emap self.r.delete(self.dupefilter_key) print "[log] Clean dupefilter_key of %s" % self.dupefilter_key def read_task(self, rto_type='cursor'): """ Get tasks from MongoDB, then return a list """ if self._alive(): cursor = self.collection.find({'state': 1}) if rto_type == 'cursor': print "[log] Read %d tasks from DB" % cursor.count() return cursor elif rto_type == 'list': tasks = [] for i in cursor: tasks.append(i) print "[log] Read %d tasks from DB" % len(tasks) return tasks else: pass def map_tasks(self): """ Map tasks' url and SKU ---------------------- Task's structure { 'sku' : string, product sku of feifei, 'urls' : list, list of this task's urls } """ def to_redis(url): self.r.lpush(self.ulist, url) tasks = self.read_task('cursor') for i in tasks: sku = i.get('sku', None) urls = i.get('urls', None) extras = i.get('extras', None) if sku: if urls: for url in urls: self.r.hset(self.rmap, url, sku) # hset(hash, key, value) to_redis(url) print "[log] List %s to %s" % (url, self.ulist) if extras: for extra in extras: self.r.hset(self.emap, extra, sku) to_redis(extra) print "[log] List %s to %s" % (extra, self.ulist) def test(self): with open('test.txt', 'ab') as f: f.write('1\n') def __getattribute__(self, name): try: rt = object.__getattribute__(self, name) except: rt = None return rt
start = datetime.date.today() - datetime.timedelta(1) end = datetime.date(2013, 1, 1) if False: get_prec_block() sys.exit() for pair in db.weather_place.find(): prec = pair['prec_id'] block = pair['block_id'] date = start while date != end: if block < 10: url = u'{0}hourly_a1.php?prec_no={1}&block_no=000{2}&year={3}&month={4}&day={5}&view='.format( base_url, prec, block, date.year, date.month, date.day) elif block < 100: url = u'{0}hourly_a1.php?prec_no={1}&block_no=00{2}&year={3}&month={4}&day={5}&view='.format( base_url, prec, block, date.year, date.month, date.day) elif block < 1000: url = u'{0}hourly_a1.php?prec_no={1}&block_no=0{2}&year={3}&month={4}&day={5}&view='.format( base_url, prec, block, date.year, date.month, date.day) elif block > 40000: url = u'{0}hourly_s1.php?prec_no={1}&block_no={2}&year={3}&month={4}&day={5}&view='.format( base_url, prec, block, date.year, date.month, date.day) else: url = u'{0}hourly_a1.php?prec_no={1}&block_no={2}&year={3}&month={4}&day={5}&view='.format( base_url, prec, block, date.year, date.month, date.day) source = common.get_source_content(url) if len(source) < 6500: continue if Weather.get_weather(url, prec, block, date, source): break date = date - datetime.timedelta(1) connect.disconnect()
class HttpApiTest(TestCase): """ You must Django client cheat sheet: Client methods: get(path, data={}, follow=False) post(path, data={}, follow=False) head(path, data={}, follow=False) options(path, data={}, follow=False) put(path, data={}, follow=False) delete(path, data={}, follow=False) login(options) logout() Reponse attributes: client content context request status_code template """ fixtures = ['features_test.json'] def __init__(self, *args, **kwargs): # settings up HTTP authentification credentials try: username = settings.HTTP_TEST_USERNAME password = settings.HTTP_TEST_PASSWORD except AttributeError: raise Exception('You must define settings.HTTP_TEST_USERNAME '\ 'and settings.HTTP_TEST_USERNAME to be able to '\ 'test HTTP authentification') if not authenticate(username=username, password=password): raise Exception('settings.HTTP_TEST_USERNAME and '\ 'settings.HTTP_TEST_PASSWORD are not valid '\ 'credentials. Could not login.') auth = 'Basic %s' % base64.encodestring('%s:%s' % (username, password)) self.auth = {'HTTP_AUTHORIZATION': auth.strip()} TestCase.__init__(self, *args, **kwargs) def assertJsonMatchDict(self, url, d, code): response = self.client.get(url, **self.auth) self.assertEqual(d, json.loads(response.content)) self.failUnlessEqual(str(response.status_code), str(code)) def assertPageMatch(self, url, content, code='200'): response = self.client.get(url, **self.auth) self.assertTrue(content in response.content) self.assertEqual(str(response.status_code), str(code)) def create_test_feature(self, name='test'): url = '/api/v1/createfeature/' coord = "[%s, %s]" % (random.random() * 10, random.random() * 10) response = self.client.post( url, { "subdivision_code": "FR", "country_code": "FR", "feature_type": "school", "geometry_type": "Point", "geometry_coordinates": coord, "name": name }, **self.auth) return response def setUp(self): self.client = Client() # set up mongo self.con = Connection(settings.MONGO_HOST, settings.MONGO_PORT) self.db = self.con[settings.MONGO_DB_NAME] self.transactions = self.db[settings.MONGO_DB_NAME] self.history = self.db[settings.MONGO_HISTORYDB_NAME] # set up a test user self.user, created = User.objects.get_or_create( username=settings.HTTP_TEST_USERNAME) self.user.set_password(settings.HTTP_TEST_PASSWORD) self.user.save() def tearDown(self): self.con.drop_database(settings.MONGO_DB_NAME) self.con.disconnect() def test_view_unknown_page(self): response = self.client.get('/nothing/', {}, **self.auth) self.failUnlessEqual(response.status_code, 404) def test_auth(self): response = self.client.get( '/api/v1/feature/111111111111111111111.json') self.assertEqual( json.loads(response.content), { u'message': u'Unauthorized - Your account credentials were invalid.', u'code': u'401' }) self.failUnlessEqual(str(response.status_code), '401') def test_view_unknown_feature(self): self.assertJsonMatchDict('/api/v1/feature/111111111111111111111.json', { 'status': 404, 'total': 0, 'type': "FeatureCollection", 'features': [] }, 404) def test_home_page(self): self.assertPageMatch('/', 'Welcome to georegistry') def test_create_feature_form(self): url = '/api/v1/createfeature/' self.assertPageMatch(url, 'Upload Feature Form') response = self.create_test_feature() self.assertEqual(str(response.status_code), "200") self.assertTrue("test" in response.content) def test_get_feature(self): response = self.create_test_feature() data = json.loads(response.content) feature_id = data['features'][0]['properties']['id'] epoch = data['features'][0]['properties']['epoch'] response = self.client.get('/api/v1/feature/%s.json' % feature_id, **self.auth) self.assertTrue("test" in response.content) self.assertTrue(feature_id in response.content) response = self.client.get( '/api/v1/feature/%s@%s.json' % (feature_id, epoch), **self.auth) self.assertTrue("test" in response.content) self.assertTrue(feature_id in response.content) self.assertTrue(epoch in response.content) def test_search_feature(self): response = self.create_test_feature() data = json.loads(response.content) feature_id = data['features'][0]['properties']['id'] epoch = data['features'][0]['properties']['epoch'] response = self.client.get('/api/v1/features/search', {'feature_type': 'school'}, **self.auth) self.assertTrue("test" in response.content) self.assertTrue("school" in response.content) response = self.client.get('/api/v1/search', {'feature_type': 'church'}, **self.auth) self.assertEqual(str(response.status_code), "404") def test_update_feature(self): pass
def do_server_status(self): host = self.mongo_host port = self.mongo_port user = self.mongo_user passwd = self.mongo_password perf_data = False con = Connection(host=self.mongo_host, port=self.mongo_port, slave_okay=True) if not self.mongo_db: self.mongo_db = con.database_names() db = con[self.mongo_db[0]] if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) server_status = db.command('serverStatus') version = server_status['version'] at_least_2_4 = V(version) >= V('2.4.0') # operations for k, v in server_status['opcounters'].items(): self.submit('total_operations', k, v) # memory for t in ['resident', 'virtual', 'mapped']: self.submit('memory', t, server_status['mem'][t]) # connections self.submit('connections', 'connections', server_status['connections']['current']) # locks if self.lockTotalTime is not None and self.lockTime is not None: if self.lockTime == server_status['globalLock']['lockTime']: value = 0.0 else: value = float(server_status['globalLock']['lockTime'] - self.lockTime) * 100.0 / float( server_status['globalLock']['totalTime'] - self.lockTotalTime) self.submit('percent', 'lock_ratio', value) self.lockTotalTime = server_status['globalLock']['totalTime'] self.lockTime = server_status['globalLock']['lockTime'] # indexes accesses = None misses = None index_counters = server_status[ 'indexCounters'] if at_least_2_4 else server_status[ 'indexCounters']['btree'] if self.accesses is not None: accesses = index_counters['accesses'] - self.accesses if accesses < 0: accesses = None misses = (index_counters['misses'] or 0) - (self.misses or 0) if misses < 0: misses = None if accesses and misses is not None: self.submit('cache_ratio', 'cache_misses', int(misses * 100 / float(accesses))) else: self.submit('cache_ratio', 'cache_misses', 0) self.accesses = index_counters['accesses'] self.misses = index_counters['misses'] for mongo_db in self.mongo_db: db = con[mongo_db] if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) db_stats = db.command('dbstats') # stats counts self.submit('counter', 'object_count', db_stats['objects'], mongo_db) self.submit('counter', 'collections', db_stats['collections'], mongo_db) self.submit('counter', 'num_extents', db_stats['numExtents'], mongo_db) self.submit('counter', 'indexes', db_stats['indexes'], mongo_db) # stats sizes self.submit('file_size', 'storage', db_stats['storageSize'], mongo_db) self.submit('file_size', 'index', db_stats['indexSize'], mongo_db) self.submit('file_size', 'data', db_stats['dataSize'], mongo_db) # Replica check rs_status = {} slaveDelays = {} try: # Get replica set status try: rs_status = con.admin.command("replSetGetStatus") except pymongo.errors.OperationFailure, e: if e.code == None and str(e).find( 'failed: not running with --replSet"'): print "OK - Not running with replSet" con.disconnect() return 0 rs_conf = con.local.system.replset.find_one() for member in rs_conf['members']: if member.get('slaveDelay') is not None: slaveDelays[member['host']] = member.get('slaveDelay') else: slaveDelays[member['host']] = 0 # Find the primary and/or the current node primary_node = None host_node = None for member in rs_status["members"]: if member["stateStr"] == "PRIMARY": primary_node = member if member["name"].split(':')[0] == host and int( member["name"].split(':')[1]) == port: host_node = member # Check if we're in the middle of an election and don't have a primary if primary_node is None: print "WARNING - No primary defined. In an election?" con.disconnect() return 1 # Check if we failed to find the current host # below should never happen if host_node is None: print "CRITICAL - Unable to find host '" + host + "' in replica set." con.disconnect() return 2 # Is the specified host the primary? if host_node["stateStr"] == "PRIMARY": if max_lag == False: print "OK - This is the primary." con.disconnect() return 0 else: #get the maximal replication lag data = "" maximal_lag = 0 for member in rs_status['members']: if not member['stateStr'] == "ARBITER": lastSlaveOpTime = member['optimeDate'] replicationLag = abs( primary_node["optimeDate"] - lastSlaveOpTime ).seconds - slaveDelays[member['name']] data = data + member[ 'name'] + " lag=%d;" % replicationLag maximal_lag = max(maximal_lag, replicationLag) # send message with maximal lag message = "Maximal lag is " + str(maximal_lag) + " seconds" print message self.submit('replication', 'maximal-lag-seconds', str(maximal_lag)) # send message with maximal lag in percentage err, con = mongo_connect( primary_node['name'].split(':')[0], int(primary_node['name'].split(':')[1]), False, user, passwd) if err != 0: con.disconnect() return err primary_timediff = replication_get_time_diff(con) maximal_lag = int( float(maximal_lag) / float(primary_timediff) * 100) message = "Maximal lag is " + str( maximal_lag) + " percents" print message self.submit('replication', 'maximal-lag-percentage', str(maximal_lag)) con.disconnect() return str(maximal_lag) elif host_node["stateStr"] == "ARBITER": print "OK - This is an arbiter" con.disconnect() return 0 # Find the difference in optime between current node and PRIMARY optime_lag = abs(primary_node["optimeDate"] - host_node["optimeDate"]) if host_node['name'] in slaveDelays: slave_delay = slaveDelays[host_node['name']] elif host_node['name'].endswith(':27017') and host_node[ 'name'][:-len(":27017")] in slaveDelays: slave_delay = slaveDelays[host_node['name'][:-len(":27017")]] else: raise Exception( "Unable to determine slave delay for {0}".format( host_node['name'])) try: # work starting from python2.7 lag = optime_lag.total_seconds() except: lag = float(optime_lag.seconds + optime_lag.days * 24 * 3600) # send message with lag message = "Lag is " + str(lag) + " seconds" print message self.submit('replication', 'lag-seconds', str(lag)) # send message with lag in percentage err, con = mongo_connect(primary_node['name'].split(':')[0], int(primary_node['name'].split(':')[1]), False, user, passwd) if err != 0: con.disconnect() return err primary_timediff = replication_get_time_diff(con) if primary_timediff != 0: lag = int(float(lag) / float(primary_timediff) * 100) else: lag = 0 message = "Lag is " + str(lag) + " percents" print message self.submit('replication', 'lag-percentage', str(lag)) con.disconnect() return str(lag)
#!/usr/bin/env python from pymongo import Connection conn = Connection("localhost:27017", slave_okay=True) config = { '_id': 'foo', 'members': [{ '_id': 0, 'host': 'localhost:27017' }, { '_id': 1, 'host': 'localhost:27018' }, { '_id': 2, 'host': 'localhost:27019' }] } resp = conn.admin.command("replSetInitiate", config) print resp conn.disconnect()