def mongo_conn(target, port=27017, mass=False): """ Establishes Connection with MongoDB and Determines whether alive or not and returns the connection object """ try: # Moved to MongoClient since Connection is Deprecated conn = MongoClient(str(target), port) # db = conn['test'] if mass and conn.database_names(): # Checks when mass scan of a IP List is provided print(colored("[+] MongoDB port open on " + target + ":27017!\n", 'green')) elif conn.database_names(): print(colored("[+] MongoDB port open on " + target + ":27017!\n", 'green')) return conn except pymongo.errors.OperationFailure: if mass: print(colored("[+] MongoDB port open " + target + ":27017 \n", 'green')) else: print(colored("[+] MongoDB port open " + target + ":27017 \n", 'green')) return conn # sys.exit() except pymongo.errors.ConnectionFailure: if mass: print(colored("[+] MongoDB port closed " + target + ":27017 \n", 'red')) else: print(colored("[-] MongoDB port closed. \n", 'red'))
class MyMongoClient(object): def __init__(self, hostname, port): self._client = MongoClient(hostname, port) self._project_db_name = 'project_%s' self._cn_last_update = 'last_update' def get_project_db(self, project): name = self._project_db_name % project return self._client[name] def get_project_names(self): project_names = [dn.split('_', 1)[1] for dn in self._client.database_names() if dn.startswith('project_') and 'manual' not in dn] project_names.sort() return project_names def get_latest_sprint_name(self, project): project_db = self.get_project_db(project) last_updates = list(project_db[self._cn_last_update].find()) if last_updates: last_updates.sort(key=lambda r: r['timestamp']) return last_updates[-1]['sprint_name'] def get_m_projects(self): project_names = [dn.split('_', 2)[-1] for dn in self._client.database_names() if dn.startswith('project_manual_')] project_names.sort() return project_names
def test_copy_db(self): authed_client = auth_context.client if is_mongos(authed_client): raise SkipTest("SERVER-6427") c = MongoClient(host, port) authed_client.admin.add_user("admin", "password") c.admin.authenticate("admin", "password") c.drop_database("pymongo_test") c.drop_database("pymongo_test1") c.pymongo_test.test.insert({"foo": "bar"}) try: c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) finally: # Cleanup remove_all_users(c.pymongo_test) c.admin.remove_user("admin") c.disconnect()
def drop_database(): client = MongoClient(DB_LOCAL_MONGO_IP, DB_PORT) print client.database_names() client.drop_database(DB_NAME) client.drop_database(DB_NAME + "Debug") print client.database_names() client.close()
def test_gssapi_simple(self): client = MongoClient(GSSAPI_HOST, GSSAPI_PORT) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) # Just test that we can run a simple command. self.assertTrue(client.database_names()) uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) self.assertTrue(client.database_names()) set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoReplicaSetClient(GSSAPI_HOST, port=GSSAPI_PORT, replicaSet=set_name) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) self.assertTrue(client.database_names()) uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoReplicaSetClient(uri) self.assertTrue(client.database_names())
def init(workDBName, dropWorkDB=False, matchDBName = None, dropMatchDB=False, indexes=False): #Indexes only used from standalone, long-running programs #If called from UI.py it interferes with database deletion cmn = {} #cmn['RGDid'] = RGDadm.seq cmn['contributor'] = RGDadm.contributor cmn['contribution'] = RGDadm.contribution client = MongoClient() if dropWorkDB: client.drop_database(workDBName) for mdb in client.database_names(): for coll in client[mdb].collection_names(): if coll.endswith(workDBName): print 'Dropping', mdb, coll client[mdb][coll].drop() db = client[workDBName] cmn['workDB'] = workDBName cmn['persons'] = db.persons cmn['families'] = db.families cmn['relations'] = db.relations cmn['originalData'] = db.originalData if indexes: cmn['relations'].ensure_index('relTyp') cmn['relations'].ensure_index('famId') cmn['relations'].ensure_index('persId') cmn['originalData'].ensure_index('recordId') cmn['originalData'].ensure_index('type') if matchDBName: cmn['matchDB'] = matchDBName cmn['matches'] = db['matches_' + matchDBName] cmn['fam_matches'] = db['fam_matches_' + matchDBName] cmn['flags'] = db['flags_' + matchDBName] matchClient = MongoClient() if dropMatchDB: matchClient.drop_database(matchDBName) for mdb in matchClient.database_names(): for coll in matchClient[mdb].collection_names(): if coll.endswith(matchDBName): print 'Dropping', mdb, coll matchClient[mdb][coll].drop() matchDB = matchClient[matchDBName] cmn['match_persons'] = matchDB.persons cmn['match_families'] = matchDB.families cmn['match_relations'] = matchDB.relations cmn['match_originalData'] = matchDB.originalData if indexes: cmn['matches'].ensure_index('workid') cmn['matches'].ensure_index('matchid') cmn['matches'].ensure_index('status') cmn['fam_matches'].ensure_index('workid') cmn['fam_matches'].ensure_index('matchid') cmn['fam_matches'].ensure_index('status') cmn['match_relations'].ensure_index('relTyp') cmn['match_relations'].ensure_index('famId') cmn['match_relations'].ensure_index('persId') cmn['match_originalData'].ensure_index('recordId') cmn['match_originalData'].ensure_index('type') return cmn
def setup_mongodb(): #####################Task t2a: your code ####################### #Connect to mongodb #connection=pymongo.Connection() # print(u'\2019') mongodb_client = MongoClient('localhost',27017) mongodb_db = mongodb_client['uta-edu-corpus'] print mongodb_client.database_names()
class MongoConnection(object): def __init__(self, database=None): self.client = MongoClient() self.db = self.connect(database) if database else None @property def database_names(self): return self.mc.database_names() @property @check_database_set def collection_names(self): return self.db.collection_names(include_system_collections=False) def connect(self, database): if database not in self.client.database_names(): raise DatabaseError('Database "' + database + '"" does not exist') return self.client[database] def create_database(self, database): if database in self.client.database_names(): raise DatabaseError('Database "' + database + '"" already exists') return self.client[database] @check_database_set def lazy_create_collection(self, collection): self.forceCollectionDoesNotExist(collection) return self.db[collection] @check_database_set def drop_collection(self, collection): self.forceCollectionExists(collection) self.db.drop_collection(collection) @check_database_set def get_collection(self, collection): self.forceCollectionExists(collection) return self.db[collection] @check_database_set def get_all_documents(self, collection): self.forceCollectionExists(collection) return self.db[collection].find() @check_database_set def forceCollectionExists(self, collection): if collection not in self.db.collection_names(include_system_collections=False): raise DatabaseError('Collection "' + collection + '" does not exist') @check_database_set def forceCollectionDoesNotExist(self, collection): if collection in self.db.collection_names(include_system_collections=False): raise DatabaseError('Collection "' + collection + '" already exists')
def establish_connection(): connection = MongoClient("localhost:27017") # MongoDB URI format:MongoClient('mongodb://localhost:27017/') #print all the existing databases: print connection.database_names() db = connection.myFirstMB #accessing the db, similar to use <db> return db
def check_connection(): timeout = TIMEOUT args = dict( connectTimeoutMS=timeout, socketTimeoutMS=timeout, serverSelectionTimeoutMS=timeout ) try: m = MongoClient(MONGODB_URI, **args) m.database_names() return True except ServerSelectionTimeoutError: return False return False
def setUp(): global client, SD, db, mt, baseSD client = MongoClient(DB_INFO['MONGO_HOST'], DB_INFO['MONGO_PORT']) if DB_INFO['DB_NAME'] in client.database_names(): client.drop_database(DB_INFO['DB_NAME']) if DB_INFO['CACHE_DB_NAME'] in client.database_names(): client.drop_database(DB_INFO['CACHE_DB_NAME']) db = client[DB_INFO['DB_NAME']] cdb = client[DB_INFO['CACHE_DB_NAME']] SD = sd('tester') baseSD = sd('baseTester') os.system('rm -rf importDir') shutil.copytree('importDirBak','importDir') SD.connectMongo() baseSD.connectMongo()
class MongoDataStore(APIView): permission_classes = ( IsAuthenticatedOrReadOnly,) renderer_classes = (DataBrowsableAPIRenderer, mongoJSONRenderer, mongoJSONPRenderer, XMLRenderer, YAMLRenderer) title = "Database" parser_classes = (JSONParser,) def __init__(self, connect_uri=config.DATA_STORE_MONGO_URI): self.db = MongoClient(host=connect_uri) def get(self, request, database=None, format=None): urls = [] if database: self.title = "Collection" data = list(self.db[database].collection_names()) #print data data.sort() for col in data: urls.append(reverse('data-detail', kwargs={'database': database, 'collection': col}, request=request)) return Response({'Database': database, 'Available Collections': urls}) else: self.title = "Database" data = list(self.db.database_names()) data.sort() for db in data: urls.append(reverse('data-list', kwargs={'database': db}, request=request)) return Response({ 'Available Databases': urls})
class MongoQuery: def __init__(self): self.client = MongoClient() def _get_db(self): db_list = [] for db in self.client.database_names(): db_list.append(self.client[db]) return db_list def _get_col(self, db): col_list = [] for col in db.collection_names(): col_list.append(db[col]) return col_list def _get_doc(self, col): doc_list = [] for doc in col.find(): doc_list.append(doc) return doc_list def query_all(self): for db_object in self._get_db(): print db_object.name for col_ob in self._get_col(db_object): print '\n' + '+'*6 + col_ob.name for doc in self._get_doc(col_ob): print '\n' + '+'*12 + str(doc)
class MongoDbItemsRepository(): #mongodb://admin:[email protected]:36709/ def __init__(self, connectionString='mongodb://*****:*****@ds036709.mlab.com:36709/'): self.connectionString = connectionString self.dbName = 'collections' self._connect() self.db = self.client[self.dbName] def _connect(self): self.client = MongoClient(self.connectionString + self.dbName) def database_exists(self, databaseName): names = self.client.database_names() return databaseName in names def insert_item(self, collectionName, item): collection = self.db[str(collectionName)] return collection.insert_one(item).inserted_id def get_items(self, collectionName, page=1, pageSize=100): collection = self.db[str(collectionName)] count = 0 res = [] for x in collection.find(): res.append(x) count+=1 if count >= pageSize: break return res def get_item_by_id(self, collectionName, itemUUID): collection = self.db[str(collectionName)] for x in collection.find({"uuid":itemUUID}): return x
def make_mongodump(name,exp_uid_list=[]): tmp_dir = tempfile.mkdtemp() if len(exp_uid_list)==0: subprocess.call(('/usr/bin/mongodump -vvvvv --host {hostname}:{port} ' '--out {path}').format(hostname=constants.MONGODB_HOST, port=constants.MONGODB_PORT, path=tmp_dir), shell=True) else: exp_uid_list_str = '["'+'","'.join(exp_uid_list)+'"]' query_str = '\'{ $or: [ {"exp_uid":{$in:%s}}, {"object_id":{$in:%s}} ] }\'' % (exp_uid_list_str,exp_uid_list_str) # subprocess.call(('/usr/bin/mongodump -vvvvv --host {hostname}:{port} ' # '--out {path} --query {query_str}').format(hostname=constants.MONGODB_HOST, client = MongoClient(constants.MONGODB_HOST, constants.MONGODB_PORT) for db in client.database_names(): for col in client[db].collection_names(): subprocess.call(('/usr/bin/mongodump -vvvvv --host {hostname}:{port} ' '--out {path} -d '+str(db)+' -c '+str(col)+' --query {query_str}').format(hostname=constants.MONGODB_HOST, port=constants.MONGODB_PORT, path=tmp_dir, query_str=query_str), shell=True) subprocess.call('mkdir -p /dump',shell=True) subprocess.call(('tar czf /dump/{name} ' '-C {path} .').format(name=name,path=tmp_dir), shell=True) shutil.rmtree(tmp_dir) return '/dump/{}'.format(name)
def estimate_backup_size(self): """Estimate the size (in bytes) of the backup this plugin would produce, if run. :returns: int. size in bytes """ ret = 0 uri = self.config["mongodump"].get("uri") if uri and uri != ['']: uri = ','.join(uri) else: uri = "mongodb://" username = self.config["mongodump"].get("username") if username: uri += urllib.parse.quote_plus(username) password = self.config["mongodump"].get("password") if password: uri += ":" + urllib.parse.quote_plus(password) uri += "@" uri += self.config["mongodump"].get("host") client = MongoClient(uri) dbs = client.database_names() for database in dbs: c_db = client[database] tup = c_db.command("dbstats") ret += int(tup["storageSize"]) # Give an upper estimate to make sure that we have enough disk space return ret * 2
def getTableList(uri, internalTables=False, **kwargs): """ Get a list of known databases, each of which has a list of known collections from the database. This is of the form [{'database': (database 1), 'tables': [{'table': (collection 1)}, {'table': (collection 2)}, ...]}, {'database': (database 2), 'tables': [...]}, ...] :param uri: uri to connect to the database. :param internaltables: True to return tables about the database itself. Ignored for Mongo. :returns: A list of known collections. """ conn = MongoClient(uri) databaseName = base.databaseFromUri(uri) if databaseName is None: databaseNames = conn.database_names() else: databaseNames = [databaseName] results = [] for name in databaseNames: database = conn[name] results.append({ 'database': name, 'tables': [{'table': collection, 'name': collection} for collection in database.collection_names(False)] }) return results
class Database: ''' Database类连接MongoDB数据库进行操作 :param str host: 数据库主机,默认是'localhost' :param int port: 数据库端口,默认是27017 :return: 无返回值 ''' def __init__(self,host='localhost',port=27017): self.client = MongoClient(host,port) def connect(self,database_name,collection_name): '''连接MongoDB数据中的集合 :param str database_name: 数据库名称 :param str collection_name: 集合名称 :return: 数据集合的连接 :rtype: pymongo.collection.Collection对象 ''' if database_name in self.client.database_names(): self.db = self.client[database_name] else: print('No such database: ',database_name) raise NameError if collection_name in self.db.collection_names(): self.collection = self.db[collection_name] else: print('No such collection: ',collection_name) raise NameError return self.collection
def verify(self,head='',context='',ip='',port='27017',productname={},keywords='',hackinfo=''): result = {} result['result']=False r=None try: r = MongoClient(ip, 27017, connectTimeoutMS=1000, socketTimeoutMS=1000, waitQueueTimeoutMS=1000) serverInfo = r.server_info() dbList = r.database_names() result['result']=True result['VerifyInfo'] = {} result['VerifyInfo']['type']='MongoClient unauth' result['VerifyInfo']['URL'] =ip+':'+port result['VerifyInfo']['payload']='None' result['VerifyInfo']['result'] ='MongoClient unauth' except Exception,e: print e.text
def main(): parser = argparse.ArgumentParser(description='Run experiments') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--db', help="name of mongo database") group.add_argument('--list', const=True, action='store_const', help="list available databases") parser.add_argument('-c', '--col') args = parser.parse_args() mongo_client = MongoClient() if args.list: return print(mongo_client.database_names()) db = mongo_client[args.db] headers = [] #for evset in "basic partial full".split(): for colName in db.collection_names(): # for t in [1, 3, 10, 30, 90, 180, 300]: benches = defaultdict(lambda: defaultdict(list)) #colName = "stab_%s_%ss" % (evset, t) print("===%s==="%colName) headers += [colName] col = db[colName] for r in col.find(): try: n = r['ann'] if 'ann' in r else r['name'] except Exception as err: print(err) for k,v in r.items(): benches[n][k] += [v] for k,r in benches.items(): print( r['ann'] if 'ann' in r else r['name']) for k,v in r.items():#[('instructions', r['instructions'])]: #r.items(): print(k, ["{:.3f}".format(v) for v in v if isinstance(v, (int,float))])
def validateDatabase(): client = MongoClient() databases = client.database_names() if any(dbName in s for s in databases): uploadMongoCollectionsToS3() else: print "Please Provide a Valid Database Name"
def test_unix_socket(self): authed_client = auth_context.client if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") if (sys.platform == 'darwin' and not version.at_least(authed_client, (2, 7, 1))): raise SkipTest("SERVER-8492") mongodb_socket = '/tmp/mongodb-27017.sock' if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessable") self.assertTrue(MongoClient("mongodb://%s" % mongodb_socket)) authed_client.admin.add_user('admin', 'pass') try: client = MongoClient("mongodb://%s" % mongodb_socket) client.admin.authenticate('admin', 'pass') client.pymongo_test.test.save({"dummy": "object"}) # Confirm we can read via the socket dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket self.assertRaises(ConnectionFailure, MongoClient, "mongodb:///tmp/none-existent.sock") finally: authed_client.admin.remove_user('admin')
def loop(ip): try: #print "[-] Scan %s ......"%ip client = MongoClient(ip,27017,connectTimeoutMS=1000,socketTimeoutMS=1000,waitQueueTimeoutMS=1000) serverInfo = client.server_info() try: serverInfo = client.server_info() print ip + "\n" for keys in serverInfo: print "[-]"+(keys)+":"+str(serverInfo[keys]) #print "\n" #print colored("[-] %s Auth Succeeded"%ip,'red') #print "[-] Get %s Server Information Succeeded!"%ip try: dbList = client.database_names() print ip + "\n" for i in dbList: print "[-] "+i #print "\n" #print "[-] Find: %s Auth Succeeded!"%ip #print result.append(ip) except: pass except: pass client.close() #print str(ip)+" Congratulations, NO auth needed!" except KeyboardInterrupt: print "[-] Interrupted by user. Exiting..." exit() except: #print str(ip)+" Connection Failed :<" pass
def validateDatabase(): client = MongoClient() databases = client.database_names() if any(dbName in s for s in databases): print "*************" else: print "Please Provide a Valid Database Name"
def test(): # Please change only the add_field and update_db functions! # Changes done to this function will not be taken into account # when doing a Test Run or Submit, they are just for your own reference # and as an example for running this code locally! data = add_field(DATAFILE, FIELDS) from pymongo import MongoClient client = MongoClient("mongodb://localhost:27017") print "client" print client.database_names() db = client.examples print db.collection_names() update_db(data, db) updated = db.arachnid.find_one({'label': 'Opisthoncana'}) assert updated['classification']['binomialAuthority'] == 'Embrik Strand' pprint.pprint(data)
def connectionMongo(self,ip='127.0.0.1',port=27017):#Todo fix ip addres client=MongoClient('192.168.1.235',27017,serverSelectionTimeoutMS=10) try: return client.database_names()[:-1] logging.info('Connection established') except errors.ServerSelectionTimeoutError: logging.error('Connection problem') return ''
def _drop_database(self, database_name): c = MongoClient() try: if database_name in c.database_names(): self.log("Dropping database: %s" % database_name) c.drop_database(database_name) finally: c.close()
def test_gssapi_simple(self): # Call authenticate() without authMechanismProperties. client = MongoClient(GSSAPI_HOST, GSSAPI_PORT) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) client.test.collection.find_one() # Log in using URI, without authMechanismProperties. uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.test.collection.find_one() # Call authenticate() with authMechanismProperties. self.assertTrue(client.test.authenticate( PRINCIPAL, mechanism='GSSAPI', authMechanismProperties='SERVICE_NAME:mongodb')) client.test.collection.find_one() # Log in using URI, with authMechanismProperties. uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI;authMechanismProperties' '=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.test.collection.find_one() set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(GSSAPI_HOST, port=GSSAPI_PORT, replicaSet=set_name) # Without authMechanismProperties self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoClient(uri) client.database_names() # With authMechanismProperties self.assertTrue(client.test.authenticate( PRINCIPAL, mechanism='GSSAPI', authMechanismProperties='SERVICE_NAME:mongodb')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s;authMechanismProperties' '=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoClient(uri) client.database_names()
def insert(file_name): from pymongo import MongoClient client = MongoClient('mongodb://localhost:27017/') print(client.database_names()) db = client.city.cities with open(file_name, 'r') as f: data = json.loads(f.read()) db.insert(data) print(db.find_one())
def validateDatabase(database, getInternalIP): client = MongoClient(getInternalIP) databases = client.database_names() if database in databases: return True else: logging.exception("Provided Database "+ database+ " Not exists...") print "Please Provide a Valid Database Name" return False
def custom_init(self, pinit=False, pdestroy=False, **kwargs): """ Note: we ignore args here """ # recover instance with the parent method db = super().custom_init() if pdestroy: # massive destruction client = db.connection.database from pymongo import MongoClient client = MongoClient(self.variables.get('host'), int(self.variables.get('port'))) system_dbs = ['admin', 'local', 'config'] for db in client.database_names(): if db not in system_dbs: client.drop_database(db) log.critical("Dropped db '%s'", db) # if pinit: # pass return db
def get_corsor(self): host = eval(self.hosts) client = MongoClient(host) query_line = eval(self.query) returns_line = eval(self.returns) options_line = self.options db = client.get_database( self.database, read_preference=ReadPreference.SECONDARY_PREFERRED) dblist = client.database_names() ##连接MongoDB if self.database in dblist: print(self.database, "database exists in", dblist) collist = db.collection_names() if self.collection in collist: print(self.collection, "collection exists in", collist) col = db[self.collection] print('Connection to mongodb', self.hosts, self.database, self.collection, 'success!') else: print(self.collection, "collection not exists in", collist) else: print(self.database, "database not exists in", dblist) ##连接成功后,确认查询类别,支持aggregate,distinct及find查询 if self.query_type == "aggregate": cursor = col.aggregate(pipeline=eval(query), allowDiskUse=True) elif self.query_type == "distinct": cursor = col.distinct(query, options) else: print('Start query now') #cursor = col.find(query_line,returns_line,options_line) cursor = col.find({}, { 'searchId': 1, '_id': 0 }, no_cursor_timeout=True) return cursor
class MongoWrapper: ressource_static = None def __init__(self, db_name="test_db"): """ Init Connection to MongoDB """ if not MongoWrapper.ressource_static: self.ressource = MongoClient() MongoWrapper.ressource_static = self.ressource else: self.ressource = MongoWrapper.ressource_static self.db_name = db_name def useDB(self, db_name): self.db_name = db_name def getCollectionNames(self): return self.ressource[self.db_name].collection_names() def getDbs(self): return self.ressource.database_names() def getCollection(self, col_name): return self.ressource[self.db_name][col_name]
def refresh_server(self,db_name): self.dbroot_node.takeChildren() mongo=MongoClient(['localhost:27017']) dbnames=mongo.database_names() for dbname in dbnames: dbtree_node = QTreeWidgetItem(MainWindow.c_db_node) dbtree_node.setText(0,QString(dbname)) dbtree_node.setIcon(0,QIcon('../icon/db.svg')) self.dbroot_node.addChild(dbtree_node) db = mongo.get_database(dbname) collections_node = \ QTreeWidgetItem(MainWindow.c_coll_folder_node) collections_node.setText(0,self.tr('collections')) collections_node.setIcon(0,QIcon('../icon/table.svg')) dbtree_node.addChild(collections_node) cols= db.collection_names() for col in cols: col_node = QTreeWidgetItem(MainWindow.c_coll_node) col_node.setText(0,QString(col)) col_node.setIcon(0,QIcon('../icon/table.svg')) collections_node.addChild(col_node) functions_node = QTreeWidgetItem(MainWindow.c_func_folder_node) functions_node.setText(0,self.tr('functions')) functions_node.setIcon(0,QIcon('../icon/function.svg')) dbtree_node.addChild(functions_node) users_node = QTreeWidgetItem(MainWindow.c_user_folder_node) users_node.setText(0,self.tr('users')) users_node.setIcon(0,QIcon('../icon/user.svg')) dbtree_node.addChild(users_node)
""" 远程操作mongodb http://api.mongodb.com/python/current/installation.html http://api.mongodb.com/python/current/tutorial.html """ from pymongo import MongoClient client = MongoClient('mongodb://118.31.19.120:27017/') print(client.database_names()) db = client.get_database('node_club_dev') collections = db.collection_names() print(collections) users = db.get_collection('users') print(users.find()) users.find_one_and_update({"loginname": "testuser"}, {'$set': { "active": True }})
def init(workDBName, dropWorkDB=False, matchDBName=None, dropMatchDB=False, indexes=False): #Indexes only used from standalone, long-running programs #If called from UI.py it interferes with database deletion cmn = {} #cmn['RGDid'] = RGDadm.seq cmn['contributor'] = RGDadm.contributor cmn['contribution'] = RGDadm.contribution client = MongoClient() if dropWorkDB: client.drop_database(workDBName) for mdb in client.database_names(): for coll in client[mdb].collection_names(): if coll.endswith(workDBName): print 'Dropping', mdb, coll client[mdb][coll].drop() db = client[workDBName] cmn['workDB'] = workDBName cmn['persons'] = db.persons cmn['families'] = db.families cmn['relations'] = db.relations cmn['originalData'] = db.originalData if indexes: cmn['relations'].ensure_index('relTyp') cmn['relations'].ensure_index('famId') cmn['relations'].ensure_index('persId') cmn['originalData'].ensure_index('recordId') cmn['originalData'].ensure_index('type') if matchDBName: cmn['matchDB'] = matchDBName cmn['matches'] = db['matches_' + matchDBName] cmn['fam_matches'] = db['fam_matches_' + matchDBName] cmn['flags'] = db['flags_' + matchDBName] matchClient = MongoClient() if dropMatchDB: matchClient.drop_database(matchDBName) for mdb in matchClient.database_names(): for coll in matchClient[mdb].collection_names(): if coll.endswith(matchDBName): print 'Dropping', mdb, coll matchClient[mdb][coll].drop() matchDB = matchClient[matchDBName] cmn['match_persons'] = matchDB.persons cmn['match_families'] = matchDB.families cmn['match_relations'] = matchDB.relations cmn['match_originalData'] = matchDB.originalData if indexes: cmn['matches'].ensure_index('workid') cmn['matches'].ensure_index('matchid') cmn['matches'].ensure_index('status') cmn['fam_matches'].ensure_index('workid') cmn['fam_matches'].ensure_index('matchid') cmn['fam_matches'].ensure_index('status') cmn['match_relations'].ensure_index('relTyp') cmn['match_relations'].ensure_index('famId') cmn['match_relations'].ensure_index('persId') cmn['match_originalData'].ensure_index('recordId') cmn['match_originalData'].ensure_index('type') return cmn
class MongoDB(object): _instance = None def __init__(self, host=None, port=None, database_name=None, user=None, password=None, collection_name=None, drop_n_create=False): try: # self._connection = MongoClient(host=host, port=port, maxPoolSize=200) # address = 'mongodb://*****:*****@101"@{}:{}/{}'.format(host, database_name) self._connection = MongoClient(host=host, port=port) except Exception as error: raise Exception(error) if drop_n_create: self.drop_db(database_name) self._database = None self._collection = None if database_name: self._database = self._connection[database_name] self._database.authenticate(user, password) if collection_name: self._collection = self._database[collection_name] @classmethod def get_instance(cls): if not cls._instance: cls._instance = MongoDB() return cls._instance @staticmethod def check_state(obj): if not obj: return False else: return True def check_db(self): if not self.check_state(self._database): # validate the database name raise ValueError('Database is empty/not created') def check_collection(self): # validate the collection name if not self.check_state(self._collection): raise ValueError('Collection is empty/not created') def get_overall_details(self): # get overall connection information client = self._connection details = dict( (db, [collection for collection in client[db].collection_names()]) for db in client.database_names()) return details def get_current_status(self): # get current connection information return { 'connection': self._connection, 'database': self._database, 'collection': self._collection } def create_db(self, database_name=None): # create the database name self._database = self._connection[database_name] def create_collection(self, collection_name=None): # create the collection name self.check_db() self._collection = self._database[collection_name] def get_database_names(self): # get the database name you are currently connected too return self._connection.database_names() def get_collection_names(self): # get the collection name you are currently connected too self.check_collection() return self._database.collection_names( include_system_collections=False) def drop_db(self, database_name): # drop/delete whole database self._database = None self._collection = None return self._connection.drop_database(str(database_name)) def drop_collection(self): # drop/delete a collection self._collection.drop() self._collection = None def insert(self, post): # add/append/new single record self.check_collection() post_id = self._collection.insert_one(post).inserted_id return post_id def insert_many(self, posts): # add/append/new multiple records self.check_collection() result = self._collection.insert_many(posts) return result.inserted_ids def find_one(self, *args): # search/find only one matching record return self._collection.find_one(*args) def find(self, count=False, *args): # search/find many matching records self.check_collection() if not count: return self._collection.find(*args) # return only count return self._collection.find(*args).count() def findMany(self, *args): return self._collection.find(*args) def find_max(self, count_key, *args): # search/find many matching records self.check_collection() return self._collection.find(*args).sort([(count_key, -1)]).limit(1) def find_min(self, count_key, *args): # search/find many matching records self.check_collection() return self._collection.find(*args).sort([(count_key, 1)]).limit(1) def count(self): # get the records count in collection self.check_collection() return self._collection.count() def remove(self, *args): # remove/delete records return self._collection.remove(*args) def update(self, object_id, post): # updating/modifying the records self.check_collection() return self._collection.update({"ObjectId": "{}".format(object_id)}, {"$push": post}) # pass def updateByParams(self, *args): self.check_collection() return self._collection.update_one(*args) def create_index(self): # create a single index pass def create_indexes(self): # create multiple indexes pass def close(self): return self._connection.close() def get_collection(self, collection_name=None): # get collection name self.check_db() try: self._collection = self._database.get_collection(collection_name) return self except Exception as e: return None
class MongoKnowledgeBase: """ Use Mongo Database to store and manage knowledge data """ # Constants FACTS_COLLECTION_NAME = "facts" RULES_COLLECTION_NAME = "rules" def __init__(self): self.__client = None self.__data_base = None self.__facts = None self.__rules = None self.root_facts = list() pass # TODO: error handling def connect(self, db_name, mongo_url='localhost', mongo_port=27017): self.__client = MongoClient(mongo_url, mongo_port) # TODO: temporary solution. Always from scratch if db_name in self.__client.database_names(): self.__client.drop_database(db_name) pass self.__data_base = self.__client[db_name] self.__facts = self.__data_base.facts self.__rules = self.__data_base.rules pass def __clean(self): if self.__data_base is not None: self.__data_base.drop_collection(self.FACTS_COLLECTION_NAME) self.__data_base.drop_collection(self.RULES_COLLECTION_NAME) self.root_facts = list() pass def facts_count(self): return self.__facts.count() def rules_count(self): return self.__rules.count() def set_decision_graph(self, decision_graph_data): self.__clean() graph = json_graph.node_link_graph(decision_graph_data) consequent_list = list() for node in graph: if graph.in_degree(node) == 0: # root self.root_facts.append(node) if graph.node[node]['type'] == FactType.Consequent: # consequent consequent_list.append(node) self.__facts.insert_one(self.transform_node_to_fact(node, graph)) pass print("Consequents count:", len(consequent_list)) for rule in self.transform_to_rules(consequent_list, graph): self.__rules.insert_one(rule) pass pass # TODO: handle intermediate consequents # TODO: add types and classes for facts (ant, con, incon) def find_antecedents(self): return self.__facts.distinct( ID_STRING, { 'type': { "$in": [FactType.Antecedent, FactType.IntermediateConsequent] } }) def get_fact_by_id(self, fact_id): return self.__facts.find_one({ID_STRING: fact_id}, {"_id": 0}) def get_rule_by_id(self, rule_id): return ProductionRule( self.__rules.find_one({ID_STRING: rule_id}, {"_id": 0})) def get_text_description(self, fact_id): return self.get_fact_by_id(fact_id)['text'] def get_type(self, fact_id): return self.get_fact_by_id(fact_id)['type'] def get_rules_with_predecessor(self, fact_id): return [ ProductionRule(x) for x in self.__rules.find( {PREDECESSORS_STRING: { '$elemMatch': { ID_STRING: fact_id } }}, {"_id": 0}) ] def transform_node_to_fact(self, node, graph): attributes = graph.node[node] fact = { "id": node, "type": attributes['type'], "text": attributes['text'] } if attributes['type'] == FactType.Consequent: if graph.out_degree(node) != 0: fact['type'] = FactType.IntermediateConsequent fact["text"] = re.sub('\(\d(\.\d*)?\)', '', attributes['text']) fact['coefficient'] = float(attributes['coefficient']) return fact def fix_path_from_intermediate_consequent(self, path): last_intermediate_consequent = next( (i for i in reversed(path[:-1]) if self.get_type(i) == FactType.IntermediateConsequent), None) if last_intermediate_consequent is not None: path = path[path.index(last_intermediate_consequent):] return path def convert_path_to_rule(self, path, graph): path = self.fix_path_from_intermediate_consequent(path) consequent_id = path[-1] rule = { PREDECESSORS_STRING: [], SUCCESSOR_STRING: { ID_STRING: consequent_id, COEFFICIENT_STRING: float(graph.node[consequent_id][COEFFICIENT_STRING]) } } for i in range(len(path) - 1): rule[PREDECESSORS_STRING].append({ ID_STRING: path[i], COEFFICIENT_STRING: float(graph.get_edge_data(path[i], path[i + 1])['weight']) }) return rule def transform_to_rules(self, consequent_list, graph): if len(self.root_facts) == 0: print("Can not add rules: No root elements") return rule_id = 0 for root in self.root_facts: for consequent in consequent_list: paths = algorithms.all_simple_paths(graph, root, consequent) for path in paths: rule = self.convert_path_to_rule(path, graph) rule[ID_STRING] = rule_id rule_id += 1 yield rule pass pass pass
def show_databases(Host, Port): conn = MongoClient(host=Host, port=Port) print conn.database_names()
def threadScan(self): while not self.queue.empty(): ip = self.queue.get() try: self.lock.acquire() print 'try to connect ' + ip + ':' + str(self.port) self.lock.release() # 原mongo连接,用MongoClient,空密码连接 client = MongoClient(ip, self.port, connectTimeoutMS=self.conn_timeout * 1000, socketTimeoutMS=self.conn_timeout * 1000, waitQueueTimeoutMS=self.conn_timeout * 1000) client.server_info() client.database_names() client.close() self.lock.acquire() print '########## ' + ip + ':' + str( self.port) + ' Connect Success!' file = open(self.fileout, 'a') file.write(ip + ':' + str(self.port) + '\n') file.close() self.success_count += 1 self.lock.release() except Exception, e: self.lock.acquire() print e self.lock.release() # TODO 读取用户名密码登陆 try: success_flag = False user_file = open(users, 'r') for user in user_file: user = user.strip() pass_file = open(passwds, 'r') for passwd in pass_file: passwd = passwd.strip() try: self.lock.acquire() print 'try to connect ' + ip + ':' + str( self.port) + ' by %s:%s' % (user, passwd) self.lock.release() client = MongoClient( "mongodb://%s:%s@%s:27017/" % (user, passwd, ip)) client.server_info() client.database_names() client.close() self.lock.acquire() success_flag = True print '########## ' + ip + ':' + str( self.port ) + ' Connect with %s:%s Success!' % (user, passwd) file = open(self.fileout, 'a') file.write(ip + ':' + str(self.port) + '\t' + user + ':' + passwd + '\n') file.close() self.success_count += 1 self.lock.release() break # 停止遍历 except Exception, e: self.lock.acquire() print e self.lock.release() self.lock.acquire() if success_flag: success_flag = False self.lock.release() break else: self.lock.release() except Exception, e: print e exit()
class DatabaseInstance: """MongoDB Database wrapper.""" def __init__(self, config: DatabaseConfig) -> None: """Initialize wrapper.""" self.config = config self.client = MongoClient(config.connection_string) self.database = self.client.get_database(config.database_name) def __enter__(self): if self.config.restore_from_dump == RestoreMode.ALWAYS or ( self.config.restore_from_dump == RestoreMode.ONLY_NEW and self.config.database_name not in self.client.database_names()): self.restore() return self def __exit__(self, exc_type, exc_value, traceback): if self.config.dump_on_exit: self.enable_cascades(False) self.dump() def drop(self): """Drop the database.""" self.client.drop_database(self.config.database_name) def restore(self): """Restore the database from the 'self.dump_directory'.""" subprocess.run([ 'mongorestore', '--nsInclude', f'{self.config.database_name}.*', '--drop' ], shell=True, check=True, cwd=self.config.dump_path) def dump(self): """Dump the database into 'self.dump_directory'.""" subprocess.run(['mongodump', '--db', self.config.database_name], cwd=self.config.dump_path, check=True) def enable_sbe(self, state: bool) -> None: """Enable new query execution engine. Throw pymongo.errors.OperationFailure in case of failure.""" self.client.admin.command({ 'setParameter': 1, 'internalQueryFrameworkControl': 'trySbeEngine' if state else 'forceClassicEngine' }) def enable_cascades(self, state: bool) -> None: """Enable new query optimizer. Requires featureFlagCommonQueryFramework set to True.""" self.client.admin.command({ 'configureFailPoint': 'enableExplainInBonsai', 'mode': 'alwaysOn' }) self.client.admin.command({ 'setParameter': 1, 'internalQueryFrameworkControl': 'tryBonsai' if state else 'trySbeEngine' }) def explain(self, collection_name: str, pipeline: Pipeline) -> dict[str, any]: """Return explain for the given pipeline.""" return self.database.command('explain', { 'aggregate': collection_name, 'pipeline': pipeline, 'cursor': {} }, verbosity='executionStats') def hide_index(self, collection_name: str, index_name: str) -> None: """Hide the given index from the query optimizer.""" self.database.command({ 'collMod': collection_name, 'index': { 'name': index_name, 'hidden': True } }) def unhide_index(self, collection_name: str, index_name: str) -> None: """Make the given index visible for the query optimizer.""" self.database.command({ 'collMod': collection_name, 'index': { 'name': index_name, 'hidden': False } }) def hide_all_indexes(self, collection_name: str) -> None: """Hide all indexes of the given collection from the query optimizer.""" for index in self.database[collection_name].list_indexes(): if index['name'] != '_id_': self.hide_index(collection_name, index['name']) def unhide_all_indexes(self, collection_name: str) -> None: """Make all indexes of the given collection visible fpr the query optimizer.""" for index in self.database[collection_name].list_indexes(): if index['name'] != '_id_': self.unhide_index(collection_name, index['name']) def drop_collection(self, collection_name: str) -> None: """Drop collection.""" self.database[collection_name].drop() def insert_many(self, collection_name: str, docs: Sequence[Mapping[str, any]]) -> None: """Insert documents into the collection with the given name.""" requests = [InsertOne(doc) for doc in docs] self.database[collection_name].bulk_write(requests, ordered=False) def get_all_documents(self, collection_name: str): """Get all documents from the collection with the given name.""" return self.database[collection_name].find({}) def get_stats(self, collection_name: str): """Get collection statistics.""" return self.database.command('collstats', collection_name) def get_average_document_size(self, collection_name: str) -> float: """Get average document size for the given collection.""" stats = self.get_stats(collection_name) avg_size = stats.get('avgObjSize') return avg_size if avg_size is not None else 0
class DataLayer: client = None db = None def __init__(self, db_conn_str, db_database): self.db_connect(db_conn_str, db_database) def db_connect(self, db_conn_str, db_database): # # db connect # self.client = MongoClient(db_conn_str) if db_database not in self.client.database_names(): print(("database list: " + str(self.client.database_names()))) raise ValueError("requested database not found") self.db = self.client[db_database] def find_db_data(self, db_collection, file_path, find_where, find_select, db_conn_str=None, db_database=None): if db_conn_str is not None and db_database is not None: self.db_connect(db_conn_str, db_database) if db_collection not in self.db.collection_names(): print(("collection list: " + str(self.db.collection_names()))) raise ValueError("requested collection not found") collection = self.db[db_collection] # # data query # db_documents = [] if find_select is not None and len(find_select) > 0: db_documents = list(collection.find(filter=find_where, projection=find_select)) else: db_documents = list(collection.find(filter=find_where)) print(("Found {0} {1} documents".format(len(db_documents), db_collection))) # # save result data # pickle.dump(db_documents, open(file_path, 'wb')) print(("Saved documents to {0}".format(file_path))) def update_db_data(self, db_collection, index_field, update_field, update_data, db_conn_str=None, db_database=None): if db_conn_str is not None and db_database is not None: self.db_connect(db_conn_str, db_database) if db_collection not in self.db.collection_names(): print(("collection list: " + str(self.db.collection_names()))) raise ValueError("requested collection not found") collection = self.db[db_collection] match_count = 0 update_count = 0 for update in update_data: result = collection.update_one({index_field:update[index_field]},{'$set':{update_field:update[update_field]}}) match_count += result.matched_count update_count += result.modified_count print(("{0}.{1} updated/matched {2}/{3}".format(db_collection, update_field, update_count, match_count)))
class Mongo(Entity): def __init__(self, bmss, config): """Setup the MongoDB component.""" self.__state = STATE_UNKNOWN self.bmss = bmss self.__config = config[DOMAIN] self.__host = util.convert(self.__config.get(CONF_HOST), str, DEFAULT_HOST) self.__port = util.convert(self.__config.get(CONF_PORT), int, DEFAULT_PORT) self.__tz_aware = util.convert(self.__config.get(CONF_TZ_AWARE), bool, DEFAULT_TZ_AWARE) self.__socket_timeout_ms = util.convert( self.__config.get(CONF_SOCKET_TIMEOUT_MS), int, DEFAULT_SOCKET_TIMEOUT_MS) self.__ssl = util.convert(self.__config.get(CONF_SSL), bool, DEFAULT_SSL) self.__max_pool_size = util.convert( self.__config.get(CONF_MAX_POOL_SIZE), int, DEFAULT_MAX_POOL_SIZE) self.__socket_keep_alive = util.convert( self.__config.get(CONF_SOCKET_KEEP_ALIVE), int, DEFAULT_SOCKET_KEEP_ALIVE) from pymongo import MongoClient from pymongo.monitoring import CommandListener class MongoCommandEvent(CommandListener): """ https://api.mongodb.com/python/current/api/pymongo/monitoring.html#module-pymongo.monitoring """ def started(self, event): _LOGGER.debug("Command {0.command_name} with request id " "{0.request_id} started on server " "{0.connection_id}".format(event)) def succeeded(self, event): _LOGGER.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "succeeded in {0.duration_micros} " "microseconds".format(event)) def failed(self, event): _LOGGER.warn("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "failed in {0.duration_micros} " "microseconds".format(event)) self.__client = MongoClient(host=self.__host, port=self.__port, tz_aware=self.__tz_aware, maxPoolSize=self.__max_pool_size, socketTimeoutMS=self.__socket_timeout_ms, ssl=self.__ssl, socketKeepAlive=self.__socket_keep_alive, document_class=dict, connect=True, event_listeners=[MongoCommandEvent()]) # Will fail here if connection is not able to be established assert (self.__client is not None) self.__state = STATE_IDLE bmss.bus.listen_once(EVENT_BLUMATE_STOP, self.disconnect) bmss.bus.listen_once(EVENT_BLUMATE_START, self.discover_databases) bmss.services.register(DOMAIN, SERVICE_DISCOVER_DATABASES, self.discover_databases) bmss.services.register(DOMAIN, SERVICE_UNLOCK, self.unlock) bmss.services.register(DOMAIN, SERVICE_DISCONNECT, self.disconnect) def discover_databases(self, event): """Discover available databases.""" self.__state = STATE_ACTIVE database_list = self.__client.database_names() self.__state = STATE_IDLE _LOGGER.info("Available Databases: %s", database_list) def unlock(self, event): """Enables writes to the server.""" _LOGGER.debug("Unlocking server...") self.__client.unlock() if self.__client.is_locked: _LOGGER.warn("Server is still locked. Maybe a permissions issue?") else: _LOGGER.info("Server is unlocked.") def disconnect(self, event): """Disconnect from the MongoDB Server.""" _LOGGER.debug("Disconnecting from MongoDB Server...") self.__client.close() _LOGGER.info("Disconnected from MongoDB Server.")
result = {'users': 'ok'} return result # -------------------------------------------------------------------------------------------------------------------- def drop_test_database(): cprint('WARNING: DATABASE {} WILL BE DELETED'.format(TEST_DATABASE), color='red', attrs=['bold', 'blink']) # sleep(60) client.drop_database(TEST_DATABASE) def load_pagelist(): collection =db.pages for type in facebook_page_lists: for sub_type in facebook_page_lists[type]: for page_id in facebook_page_lists[type][sub_type]: page_doc={} page_doc['type']=type page_doc['sub_type']=sub_type page_doc['id']=page_id page_doc['name']=fb_get_page(page_id)['name'] #{name:..., id:...} collection.update_one({'id':page_id}, {'$set':page_doc}, upsert=True) if __name__ == '__main__': print '_' * 120 print 'Databases: ', client.database_names() print 'Collections: ', db.collection_names() print setup_databases() load_pagelist()
chunk = "" for segment in segments: chunk += segment #try: #yield json.loads(chunk) yaml.load(chunk) #get rid of random backslashes! #yield json.loads(chunk.replace('\r\n', '\\r\\n')) chunk = "" #Problem with json file! ''' except ValueError: print("error!") pass ''' connection = MongoClient('localhost', 27017) connection.database_names() db = connection.outlook count = 0 with open('outlook_contacts.json') as f: #data = json.load(f) for parsed_json in load_json_multiple(f): count += 1 if count % 1000 == 0: print(count) #print(parsed_json) data = parsed_json contacts = db.cleancontacts contact_id = contacts.insert_one(data).inserted_id
from pymongo import MongoClient from pprint import pprint from pymongo.errors import ConnectionFailure from ConfigFilesLoader import db_config_data, logger_name import Logger import logging logger = logging.getLogger(logger_name) db_name = db_config_data['database_info']['name'] collection_names = db_config_data['database_info']['collections'] client = MongoClient(host=db_config_data['host'], port=db_config_data['port']) ## Logic needs to be written in case some connection failure happens if db_config_data['database_info']['name'] not in client.database_names(): database = client[db_config_data['database_info']['name']] logger.info("Database created succesfully") else: logger.info("Database already exists")
""" """ import gridfs from pymongo import MongoClient #client = MongoClient('localhost', 27017) client = MongoClient('psanaphi105', 27017) #, username=uname, password=pwd) db = client['calib-cxi12345'] fs = gridfs.GridFS(db) col = db['cxi12345'] docs = col.find() doc = docs[0] print('client.database_names(): ', client.database_names()) print('db.collection_names() : ', db.collection_names(False)) print('==== fs :\n', fs) print('==== col :\n', col) print('==== docs :\n', docs) print('==== doc :\n', doc) print('==== vars(col) :\n', vars(col))
def test_gssapi_simple(self): client = MongoClient(GSSAPI_HOST, GSSAPI_PORT) # Without gssapiServiceName self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.database_names() # With gssapiServiceName self.assertTrue( client.test.authenticate(PRINCIPAL, mechanism='GSSAPI', gssapiServiceName='mongodb')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI;gssapiServiceName=mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI;authMechanismProperties=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.database_names() set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoReplicaSetClient(GSSAPI_HOST, port=GSSAPI_PORT, replicaSet=set_name) # Without gssapiServiceName self.assertTrue( client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoReplicaSetClient(uri) client.database_names() # With gssapiServiceName self.assertTrue( client.test.authenticate(PRINCIPAL, mechanism='GSSAPI', gssapiServiceName='mongodb')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s;gssapiServiceName=mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoReplicaSetClient(uri) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet=%s;' 'authMechanismProperties=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoReplicaSetClient(uri) client.database_names()
print("addRiskNumbers.py <mongoURL> <mongoPort> <userName> <password>") sys.exit(0) mongoURL = sys.argv[1] mongoPort = sys.argv[2] userName = sys.argv[3] password = sys.argv[4] connString = "mongodb://" + userName + ":" + password + "@" + mongoURL + ":" + mongoPort + "/" ##### Enable dry run to not commit to the database ##### dryRun = True ##### Connect to the Database ##### db = MongoClient(connString) for database in db.database_names(): if database != "admin" and database != "local": db = MongoClient(connString)[database] print("--database:" + database) ##### Get a model ID and find entries ##### for setting in db.settings.find(no_cursor_timeout=True): modelId = setting.get("_id") print("\t--model: " + modelId) riskNumber = 1 db[modelId + ".risks"].ensure_index([("created", pymongo.ASCENDING) ]) for risk in db[modelId + ".risks"].find( no_cursor_timeout=True).sort("created", pymongo.ASCENDING): riskId = risk.get("_id") print("\t\t--risk (" + str(riskNumber) + "): " + str(riskId))
#install pymongo import pip pip.main(['install','pymongo']) from pymongo import MongoClient # connect # myip = '10.1.2.31' # localhost # con = MongoClient(myip,27017) con = MongoClient('localhost', 27017) # List databases con.database_names() # delete test databases if they already exist try: con.drop_database("pythondb") except: pass try: con.drop_database("pythondb2") except: pass con.database_names() # Create two databases two different ways db = con.pythondb db2 = con['pythondb2'] # Lazy creation con.database_names()
class MongoDB(object): def __init__(self): self.mongo_host = "127.0.0.1" self.mongo_port = 27017 self.mongo_db = [ "admin", ] self.mongo_user = None self.mongo_password = None self.__conn = None self.__dbnames = None self.__metrics = [] # Connect to MongoDB def connect(self): if self.__conn is None: try: self.__conn = MongoClient(host=self.mongo_host, port=self.mongo_port) except Exception as e: print 'Error in MongoDB connection: %s' % str(e) # Add each mectrics to the metrics list def addMetrics(self, k, v): dict = {} dict['key'] = k dict['value'] = v self.__metrics.append(dict) # Print out all metrics def printMetrics(self): metrics = self.__metrics for m in metrics: zabbix_item_key = str(m['key']) zabbix_item_value = str(m['value']) print '- ' + zabbix_item_key + ' ' + zabbix_item_value # Get a list of DB names def getDBNames(self): if self.__conn is None: self.connect() db = self.__conn[self.mongo_db[0]] if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) master = db.command('isMaster')['ismaster'] dict = {} dict['key'] = 'mongodb.ismaster' DBList = [] if master: dict['value'] = 1 DBNames = self.__conn.database_names() self.__dbnames = DBNames else: dict['value'] = 0 self.__metrics.append(dict) # Print DB list in json format, to be used for mongo db discovery in zabbix def getMongoDBLLD(self): if self.__dbnames is None: DBNames = self.getDBNames() else: DBNames = self.__dbnames DBList = [] if DBNames is not None: for db in DBNames: dict = {} dict['{#MONGODBNAME}'] = db DBList.append(dict) return {"data": DBList} def getOplog(self): db = MongoClient(self.mongo_host, self.mongo_port) if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) dbl = db.local coll = dbl['oplog.rs'] op_first = (coll.find().sort('$natural', 1).limit(1)) while op_first.alive: op_fst = (op_first.next())['ts'].time op_last = (coll.find().sort('$natural', -1).limit(1)) while op_last.alive: op_last_st = op_last[0]['ts'] op_lst = (op_last.next())['ts'].time status = round(float(op_lst - op_fst), 1) self.addMetrics('mongodb.oplog', status) currentTime = timegm(gmtime()) oplog = int(((str(op_last_st).split('('))[1].split(','))[0]) self.addMetrics('mongodb.oplog-sync', (currentTime - oplog)) def getMaintenance(self): db = MongoClient(self.mongo_host, self.mongo_port) if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) host_name = socket.gethostname() fsync_locked = int(db.is_locked) config = db.admin.command("replSetGetConfig", 1) for i in range(0, len(config['config']['members'])): if host_name in config['config']['members'][i]['host']: priority = config['config']['members'][i]['priority'] hidden = int(config['config']['members'][i]['hidden']) self.addMetrics('mongodb.fsync-locked', fsync_locked) self.addMetrics('mongodb.priority', priority) self.addMetrics('mongodb.hidden', hidden) # Get Server Status def getServerStatusMetrics(self): if self.__conn is None: self.connect() db = self.__conn[self.mongo_db[0]] if self.mongo_user and self.mongo_password: db.authenticate(self.mongo_user, self.mongo_password) ss = db.command('serverStatus') #print ss # db info self.addMetrics('mongodb.version', ss['version']) self.addMetrics('mongodb.storageEngine', ss['storageEngine']['name']) self.addMetrics('mongodb.uptime', int(ss['uptime'])) self.addMetrics('mongodb.okstatus', int(ss['ok'])) # asserts for k, v in ss['asserts'].items(): self.addMetrics('mongodb.asserts.' + k, v) # operations for k, v in ss['opcounters'].items(): self.addMetrics('mongodb.operation.' + k, v) # memory for k in ['resident', 'virtual', 'mapped', 'mappedWithJournal']: self.addMetrics('mongodb.memory.' + k, ss['mem'][k]) # connections for k, v in ss['connections'].items(): self.addMetrics('mongodb.connection.' + k, v) # network for k, v in ss['network'].items(): self.addMetrics('mongodb.network.' + k, v) # extra info self.addMetrics('mongodb.page.faults', ss['extra_info']['page_faults']) #wired tiger self.addMetrics( 'mongodb.used-cache', ss['wiredTiger']['cache']["bytes currently in the cache"]) self.addMetrics('mongodb.total-cache', ss['wiredTiger']['cache']["maximum bytes configured"]) self.addMetrics( 'mongodb.dirty-cache', ss['wiredTiger']['cache']["tracked dirty bytes in the cache"]) # global lock lockTotalTime = ss['globalLock']['totalTime'] self.addMetrics('mongodb.globalLock.totalTime', lockTotalTime) for k, v in ss['globalLock']['currentQueue'].items(): self.addMetrics('mongodb.globalLock.currentQueue.' + k, v) for k, v in ss['globalLock']['activeClients'].items(): self.addMetrics('mongodb.globalLock.activeClients.' + k, v) # Get DB stats for each DB def getDBStatsMetrics(self): if self.__conn is None: self.connect() if self.__dbnames is None: self.getDBNames() if self.__dbnames is not None: for mongo_db in self.__dbnames: db = self.__conn[mongo_db] if self.mongo_user and self.mongo_password: self.__conn[self.mongo_db[0]].authenticate( self.mongo_user, self.mongo_password) dbs = db.command('dbstats') for k, v in dbs.items(): if k in [ 'storageSize', 'ok', 'avgObjSize', 'indexes', 'objects', 'collections', 'fileSize', 'numExtents', 'dataSize', 'indexSize', 'nsSizeMB' ]: self.addMetrics( 'mongodb.stats.' + k + '[' + mongo_db + ']', int(v)) # Close connection def close(self): if self.__conn is not None: self.__conn.close()
class MongoDB(object): """main script class""" # pylint: disable=too-many-instance-attributes def __init__(self): self.mongo_host = "localhost" self.mongo_port = 27017 self.mongo_db = [ "admin", ] self.mongo_user = None self.mongo_password = None self.__conn = None self.__dbnames = None self.__metrics = [] def connect(self): """Connect to MongoDB""" if self.__conn is None: if self.mongo_user is None: try: self.__conn = MongoClient( 'mongodb://%s:%s' % (self.mongo_host, self.mongo_port)) except errors.PyMongoError as py_mongo_error: print('Error in MongoDB connection: %s' % str(py_mongo_error)) else: try: self.__conn = MongoClient( 'mongodb://%s:%s@%s:%s' % (self.mongo_user, self.mongo_password, self.mongo_host, self.mongo_port)) except errors.PyMongoError as py_mongo_error: print('Error in MongoDB connection: %s' % str(py_mongo_error)) def add_metrics(self, k, v): """add each metric to the metrics list""" dict_metrics = {} dict_metrics['key'] = k dict_metrics['value'] = v self.__metrics.append(dict_metrics) def print_metrics(self): """print out all metrics""" metrics = self.__metrics for metric in metrics: zabbix_item_key = str(metric['key']) zabbix_item_value = str(metric['value']) print('- ' + zabbix_item_key + ' ' + zabbix_item_value) def get_db_names(self): """get a list of DB names""" if self.__conn is None: self.connect() db_handler = self.__conn[self.mongo_db[0]] master = db_handler.command('isMaster')['ismaster'] dict_metrics = {} dict_metrics['key'] = 'mongodb.ismaster' if master: dict_metrics['value'] = 1 db_names = self.__conn.database_names() self.__dbnames = db_names else: dict_metrics['value'] = 0 self.__metrics.append(dict_metrics) def get_mongo_db_lld(self): """print DB list in json format, to be used for mongo db discovery in zabbix""" if self.__dbnames is None: db_names = self.get_db_names() else: db_names = self.__dbnames dict_metrics = {} db_list = [] dict_metrics['key'] = 'mongodb.discovery' dict_metrics['value'] = {"data": db_list} if db_names is not None: for db_name in db_names: dict_lld_metric = {} dict_lld_metric['{#MONGODBNAME}'] = db_name db_list.append(dict_lld_metric) dict_metrics['value'] = '{"data": ' + json.dumps(db_list) + '}' self.__metrics.insert(0, dict_metrics) def get_oplog(self): """get replica set oplog information""" if self.__conn is None: self.connect() db_handler = self.__conn['local'] coll = db_handler.oplog.rs op_first = (coll.find().sort('$natural', 1).limit(1)) op_last = (coll.find().sort('$natural', -1).limit(1)) # if host is not a member of replica set, without this check we will # raise StopIteration as guided in # http://api.mongodb.com/python/current/api/pymongo/cursor.html if op_first.count() > 0 and op_last.count() > 0: op_fst = (op_first.next())['ts'].time op_last_st = op_last[0]['ts'] op_lst = (op_last.next())['ts'].time status = round(float(op_lst - op_fst), 1) self.add_metrics('mongodb.oplog', status) current_time = timegm(gmtime()) oplog = int(((str(op_last_st).split('('))[1].split(','))[0]) self.add_metrics('mongodb.oplog-sync', (current_time - oplog)) def get_maintenance(self): """get replica set maintenance info""" if self.__conn is None: self.connect() db_handler = self.__conn fsync_locked = int(db_handler.is_locked) self.add_metrics('mongodb.fsync-locked', fsync_locked) def get_server_status_metrics(self): """get server status""" if self.__conn is None: self.connect() db_handler = self.__conn[self.mongo_db[0]] ss = db_handler.command('serverStatus') # db info self.add_metrics('mongodb.okstatus', int(ss['ok'])) # extra info self.add_metrics('mongodb.page.faults', ss['extra_info']['page_faults']) # network self.add_metrics('mongodb.network.bytesIn', long(ss['network']['bytesIn'])) self.add_metrics('mongodb.network.bytesOut', long(ss['network']['bytesOut'])) self.add_metrics('mongodb.network.physicalBytesIn', long(ss['network']['physicalBytesIn'])) self.add_metrics('mongodb.network.physicalBytesOut', long(ss['network']['physicalBytesOut'])) self.add_metrics('mongodb.network.numRequests', long(ss['network']['numRequests'])) self.add_metrics( 'mongodb.network.serviceExecutorTaskStats.threadsRunning', int(ss['network']['serviceExecutorTaskStats']['threadsRunning'])) # memory for k in ['resident', 'virtual', 'bits']: self.add_metrics('mongodb.memory.' + k, ss['mem'][k]) #wired tiger if ss['storageEngine']['name'] == 'wiredTiger': self.add_metrics( 'mongodb.used-cache', ss['wiredTiger']['cache']["bytes currently in the cache"]) self.add_metrics( 'mongodb.total-cache', ss['wiredTiger']['cache']["maximum bytes configured"]) self.add_metrics( 'mongodb.dirty-cache', ss['wiredTiger']['cache']["tracked dirty bytes in the cache"]) # global lock for k, v in ss['globalLock']['currentQueue'].items(): self.add_metrics('mongodb.globalLock.currentQueue.' + k, v) for k, v in ss['globalLock']['activeClients'].items(): self.add_metrics('mongodb.globalLock.activeClients.' + k, v) def get_db_stats_metrics(self): """get DB stats for each DB""" if self.__conn is None: self.connect() if self.__dbnames is None: self.get_db_names() if self.__dbnames is not None: for mongo_db in self.__dbnames: isAdminDb = mongo_db == 'admin' metricsList = [ 'storageSize', 'ok', 'avgObjSize', 'dataSize', 'indexSize' ] if isAdminDb: metricsList = [ 'storageSize', 'ok', 'avgObjSize', 'dataSize', 'indexSize', 'fsUsedSize', 'fsTotalSize' ] db_handler = self.__conn[mongo_db] dbs = db_handler.command('dbstats') for k, v in dbs.items(): if k in metricsList: self.add_metrics( 'mongodb.stats.' + k + '[' + mongo_db + ']', int(v)) def close(self): """close connection to mongo""" if self.__conn is not None: self.__conn.close()
#------------PROBLEMS------------# #The program is only reading in one entry from the json #------------PROBLEMS------------# #http://www.vizgr.org/historical-events/search.php?format=json&begin_date=-3000000&end_date=20151231&lang=en from pymongo import MongoClient from json import load connection = MongoClient('homer.stuy.edu') #if the database hasnt been made, then make it and add it if not 'stan_smith_xus' in connection.database_names(): print "i tried" collection = connection.stan_smith_xus #the database db = collection.events with open('events.json') as f: event = load(f) print event db.insert(event) else: #for debugging, the db is removed if you run the program after making a db on the server print "removing..." connection.drop_database('stan_smith_xus')
from pymongo import MongoClient, errors try: '''Connect with local mongdb''' # MongoClient = MongoClient('localhost',27017) '''Connect with mongdb cluster''' MongoClient = MongoClient( "mongodb://<Username>:<Password>@cluster0-shard-00-00-fnmen.mongodb.net:27017,cluster0-shard-00-01-fnmen.mongodb.net:27017,cluster0-shard-00-02-fnmen.mongodb.net:27017/test?ssl=true&replicaSet=Cluster0-shard-0&authSource=admin" ) print("Connect successfully!") except errors.ConnectionFailure: print("Could not connect to MongoDB") print("Show database", MongoClient.database_names()) '''Change database''' database = MongoClient.test '''Get collection name''' collection = database.collection_names(include_system_collections=False) for collect in collection: print(collect) '''Get collection restaurants''' restaurants = database.restaurants '''Print one sample''' print(restaurants.find_one()) '''Print one sample with contrain''' print(restaurants.find_one({"name": "Wendy'S"}))
update1 = db.rssInfo.update_one( {'_id': i['_id']}, {'$set': { rssName: i[rssName] }}) update2 = db.currentInfo.update_one( {'_id': current_j['_id']}, {'$set': { rssName: current_j[rssName] }}) return client = MongoClient('localhost', port=27017) #printing all databases print(client.database_names()) if ("SourceDB" in client.database_names()): print("Database present\n") #connecting to database db = client.SourceDB target = db.targetInfo current = db.currentInfo cursor_target = target.find() cursor_current = current.find() rssList = [ 'Food Packets', 'Sanitary Napkins', 'Water Bottles', 'Medkits', 'Blankets',
# DEFINE COLLECTION (TABLE) WHERE YOU'LL INSERT THE TWEETS tweets = db['tweets'] # CREATE UNIQUE INDEX FOR TABLE (TO AVOID DUPLICATES) #db.tweets.create_index([('id_str', pymongo.ASCENDING)], unique=True) #SHOW INDEX ON TWEETS COLLECTION list(db.tweets.index_information()) #SHOW NUMBER OF TWEETS IN TABLE tweets.count() #TO SEE LIST OF CURRENT MONGODB DATABASES client.database_names() #TO SEE LIST OF COLLECTIONS IN THE *TWITTER* DATABASE db.collection_names() #testing the importing of df #df = pd.read_csv('sample_draft.csv') #df = pd.read_csv('sample1_LEFT.csv', header=None) #REPEAT FOR RIGHT #df = pd.read_csv('sample1_RIGHT.csv', header=None) #df.rename(columns={0: '_id'}, inplace=True) #df.columns #records = json.loads(df.T.to_json()).values() #records #len(records)
# coding:utf-8 # mongodb未授权检测脚本 # author:ske # usage: python3 mongodb_unauth.py ip port # 默认端口28017和27017 from pymongo import MongoClient import sys ip = sys.argv[1] port = int(sys.argv[2]) try: conn = MongoClient(ip, port, socketTimeoutMS=5000) # 连接MongoDB,延时5秒 dbs = conn.database_names() print('[ok] -> {}:{} database_names : {}'.format(ip, port, dbs)) conn.close() except Exception as e: error = e.args print('[-] -> {}:{} error : {}'.format(ip, port, error))
#!/usr/bin/python import pymongo import datetime from pymongo import MongoClient #connect to mongodb server client = MongoClient() print client.database_names() #get database, db is defined take the test data base from mongodb #db = client.local db = client.test #dbs = db.show() #print db.name() #get collection #collection = db.test #document post post = { "author": "Mike", "text": "My first blog post!", "tags": ["mongodb", "python", "pymongo"], "date": datetime.datetime.utcnow() } #new collection called posts posts = db.posts post_id = posts.insert(post) print post_id
from pymongo import MongoClient import datetime client = MongoClient('localhost',27017) # client = MongoClient() <--localhost print(client.database_names()) #데이터베이스 이름 출력 db = client.lab2 print(db.collection_names()) #데이터베이스 내의 컬랙션 이름 출력 collection = db.apples post = {"author":"Mike","text":"My first blog post!", "tags":["MongoDB","Python","PyMongo"],"date":datetime.datetime.utcnow()} db.lab2.inset_one(post)
class Database(object): '''Database creation''' def __init__(self, database_name): self.client = MongoClient('mongodb://localhost,localhost:27017') self.db_name = database_name self.db = self.client[self.db_name] #self.jobs = self.client[self.db_name].jobs #self.results = self.db['results'] #self.queue = self.db['queue'] #self.log = self.db['log'] #self.sources = self.db['sources'] #self.jobs = self.db['jobs'] #self.db.x = self.db[x] # def __repr__(self, database_name): # print "Using database: %s" %self.client[database_name] # return self.db def use_db(self, database_name): return self.client[str(database_name)] def use_coll(self, coll_name): return self.db[coll_name] def show_dbs(self): return self.client.database_names() def create_coll(self, coll_name): setattr(self, str(coll_name), self.db[str(coll_name)]) #print "coll : %s has been created in db:%s " %(self.__dict__[str(coll_name)], self.db_name) return self.__dict__[str(coll_name)] def create_colls(self, coll_names=["results", "sources", "log", "queue"]): for n in coll_names: setattr(self, n, self.db[str(n)]) # self.queue = self.db['queue'] # self.log = self.db['log'] # self.sources = self.db['sources'] # #print "Creating coll", [n for n in self.db.collection_names()] return [n for n in self.db.collection_names()] def show_coll(self): try: print "using collection %s in DB : %s" % (self.coll_name, self.db_name) return self.coll_name except AttributeError: return False #return self.db.collection_names() def show_coll_items(self, coll_name): return [n for n in self.db[str(coll_name)].find()] # def count(self, coll_name): # self.db_coll = self.db[str(coll_name)] # return self.db_coll.count() def drop(self, type, name): if type == "collection": return self.db[str(name)].drop() elif type == "database": return self.client.drop_database(str(name)) else: print "Unknown Type" return False def drop_all_dbs(self): '''remove EVERY SINGLE MONGO DATABASE''' for n in self.show_dbs(): #if n not in ["projects", "tasks"]: self.use_db(n) self.drop("database", n) def stats(self): '''Output the current stats of database in Terminal''' title = "===STATS===\n" name = "Stored results in Mongo Database: %s \n" % (self.db_name) res = "\t-Nombre de resultats dans la base: %d\n" % ( self.db.results.count()) sources = "\t-Nombre de sources: %d\n" % len( self.db.sources.distinct('url')) url = "\t-urls en cours de traitement: %d\n" % (self.db.queue.count()) url2 = "\t-urls traitees: %d\n" % (self.db.results.count() + self.db.log.count()) url3 = "\t-urls erronées: %d\n" % (self.db.log.count()) size = "\t-Size of the database %s: %d MB\n" % ( self.db_name, (self.db.command('dbStats', 1024)['storageSize']) / 1024 / 1024.) result = [title, name, res, sources, url, url2, size] return "".join(result) def report(self): ''' Output the currents of database for Email Report''' res = "<li>Nombre de resultats dans la base: %d</li>" % ( self.db.results.count()) sources = "<li>Nombre de sources: %d</li>" % len( self.db.sources.distinct('url')) url = "<li>urls en cours de traitement: %d\n</li>" % ( self.db.queue.count()) url2 = "<li>urls traitees: %d</li>" % (self.db.results.count() + self.db.log.count()) size = "<li>Size of the database %s: %d MB</li>" % ( self.db_name, (self.db.command('dbStats', 1024)['storageSize']) / 1024 / 1024.) result = [res, sources, url, url2, size] return "".join(result)