def mongo_conn(target, port=27017, mass=False):
"""
Establishes Connection with MongoDB and Determines whether alive or not and returns the connection object
"""
try:
# Moved to MongoClient since Connection is Deprecated
conn = MongoClient(str(target), port)
# db = conn['test']
if mass and conn.database_names(): # Checks when mass scan of a IP List is provided
print(colored("[+] MongoDB port open on " +
target + ":27017!\n", 'green'))
elif conn.database_names():
print(colored("[+] MongoDB port open on " +
target + ":27017!\n", 'green'))
return conn
except pymongo.errors.OperationFailure:
if mass:
print(colored("[+] MongoDB port open " +
target + ":27017 \n", 'green'))
else:
print(colored("[+] MongoDB port open " +
target + ":27017 \n", 'green'))
return conn
# sys.exit()
except pymongo.errors.ConnectionFailure:
if mass:
print(colored("[+] MongoDB port closed " +
target + ":27017 \n", 'red'))
else:
print(colored("[-] MongoDB port closed. \n", 'red'))
class MyMongoClient(object):
def __init__(self, hostname, port):
self._client = MongoClient(hostname, port)
self._project_db_name = 'project_%s'
self._cn_last_update = 'last_update'
def get_project_db(self, project):
name = self._project_db_name % project
return self._client[name]
def get_project_names(self):
project_names = [dn.split('_', 1)[1] for dn in self._client.database_names()
if dn.startswith('project_') and 'manual' not in dn]
project_names.sort()
return project_names
def get_latest_sprint_name(self, project):
project_db = self.get_project_db(project)
last_updates = list(project_db[self._cn_last_update].find())
if last_updates:
last_updates.sort(key=lambda r: r['timestamp'])
return last_updates[-1]['sprint_name']
def get_m_projects(self):
project_names = [dn.split('_', 2)[-1] for dn in self._client.database_names()
if dn.startswith('project_manual_')]
project_names.sort()
return project_names
def test_copy_db(self):
authed_client = auth_context.client
if is_mongos(authed_client):
raise SkipTest("SERVER-6427")
c = MongoClient(host, port)
authed_client.admin.add_user("admin", "password")
c.admin.authenticate("admin", "password")
c.drop_database("pymongo_test")
c.drop_database("pymongo_test1")
c.pymongo_test.test.insert({"foo": "bar"})
try:
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
finally:
# Cleanup
remove_all_users(c.pymongo_test)
c.admin.remove_user("admin")
c.disconnect()
def drop_database():
client = MongoClient(DB_LOCAL_MONGO_IP, DB_PORT)
print client.database_names()
client.drop_database(DB_NAME)
client.drop_database(DB_NAME + "Debug")
print client.database_names()
client.close()
def test_gssapi_simple(self):
client = MongoClient(GSSAPI_HOST, GSSAPI_PORT)
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
# Just test that we can run a simple command.
self.assertTrue(client.database_names())
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
self.assertTrue(client.database_names())
set_name = client.admin.command('ismaster').get('setName')
if set_name:
client = MongoReplicaSetClient(GSSAPI_HOST,
port=GSSAPI_PORT,
replicaSet=set_name)
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
self.assertTrue(client.database_names())
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
'=%s' % (quote_plus(PRINCIPAL),
GSSAPI_HOST, GSSAPI_PORT, str(set_name)))
client = MongoReplicaSetClient(uri)
self.assertTrue(client.database_names())
def init(workDBName, dropWorkDB=False, matchDBName = None, dropMatchDB=False,
indexes=False):
#Indexes only used from standalone, long-running programs
#If called from UI.py it interferes with database deletion
cmn = {}
#cmn['RGDid'] = RGDadm.seq
cmn['contributor'] = RGDadm.contributor
cmn['contribution'] = RGDadm.contribution
client = MongoClient()
if dropWorkDB:
client.drop_database(workDBName)
for mdb in client.database_names():
for coll in client[mdb].collection_names():
if coll.endswith(workDBName):
print 'Dropping', mdb, coll
client[mdb][coll].drop()
db = client[workDBName]
cmn['workDB'] = workDBName
cmn['persons'] = db.persons
cmn['families'] = db.families
cmn['relations'] = db.relations
cmn['originalData'] = db.originalData
if indexes:
cmn['relations'].ensure_index('relTyp')
cmn['relations'].ensure_index('famId')
cmn['relations'].ensure_index('persId')
cmn['originalData'].ensure_index('recordId')
cmn['originalData'].ensure_index('type')
if matchDBName:
cmn['matchDB'] = matchDBName
cmn['matches'] = db['matches_' + matchDBName]
cmn['fam_matches'] = db['fam_matches_' + matchDBName]
cmn['flags'] = db['flags_' + matchDBName]
matchClient = MongoClient()
if dropMatchDB:
matchClient.drop_database(matchDBName)
for mdb in matchClient.database_names():
for coll in matchClient[mdb].collection_names():
if coll.endswith(matchDBName):
print 'Dropping', mdb, coll
matchClient[mdb][coll].drop()
matchDB = matchClient[matchDBName]
cmn['match_persons'] = matchDB.persons
cmn['match_families'] = matchDB.families
cmn['match_relations'] = matchDB.relations
cmn['match_originalData'] = matchDB.originalData
if indexes:
cmn['matches'].ensure_index('workid')
cmn['matches'].ensure_index('matchid')
cmn['matches'].ensure_index('status')
cmn['fam_matches'].ensure_index('workid')
cmn['fam_matches'].ensure_index('matchid')
cmn['fam_matches'].ensure_index('status')
cmn['match_relations'].ensure_index('relTyp')
cmn['match_relations'].ensure_index('famId')
cmn['match_relations'].ensure_index('persId')
cmn['match_originalData'].ensure_index('recordId')
cmn['match_originalData'].ensure_index('type')
return cmn
def setup_mongodb():
#####################Task t2a: your code #######################
#Connect to mongodb
#connection=pymongo.Connection()
# print(u'\2019')
mongodb_client = MongoClient('localhost',27017)
mongodb_db = mongodb_client['uta-edu-corpus']
print mongodb_client.database_names()
class MongoConnection(object):
def __init__(self, database=None):
self.client = MongoClient()
self.db = self.connect(database) if database else None
@property
def database_names(self):
return self.mc.database_names()
@property
@check_database_set
def collection_names(self):
return self.db.collection_names(include_system_collections=False)
def connect(self, database):
if database not in self.client.database_names():
raise DatabaseError('Database "' + database + '"" does not exist')
return self.client[database]
def create_database(self, database):
if database in self.client.database_names():
raise DatabaseError('Database "' + database + '"" already exists')
return self.client[database]
@check_database_set
def lazy_create_collection(self, collection):
self.forceCollectionDoesNotExist(collection)
return self.db[collection]
@check_database_set
def drop_collection(self, collection):
self.forceCollectionExists(collection)
self.db.drop_collection(collection)
@check_database_set
def get_collection(self, collection):
self.forceCollectionExists(collection)
return self.db[collection]
@check_database_set
def get_all_documents(self, collection):
self.forceCollectionExists(collection)
return self.db[collection].find()
@check_database_set
def forceCollectionExists(self, collection):
if collection not in self.db.collection_names(include_system_collections=False):
raise DatabaseError('Collection "' + collection + '" does not exist')
@check_database_set
def forceCollectionDoesNotExist(self, collection):
if collection in self.db.collection_names(include_system_collections=False):
raise DatabaseError('Collection "' + collection + '" already exists')
def establish_connection():
connection = MongoClient("localhost:27017")
# MongoDB URI format:MongoClient('mongodb://localhost:27017/')
#print all the existing databases:
print connection.database_names()
db = connection.myFirstMB #accessing the db, similar to use <db>
return db
def check_connection():
timeout = TIMEOUT
args = dict(
connectTimeoutMS=timeout,
socketTimeoutMS=timeout,
serverSelectionTimeoutMS=timeout
)
try:
m = MongoClient(MONGODB_URI, **args)
m.database_names()
return True
except ServerSelectionTimeoutError:
return False
return False
def setUp():
global client, SD, db, mt, baseSD
client = MongoClient(DB_INFO['MONGO_HOST'], DB_INFO['MONGO_PORT'])
if DB_INFO['DB_NAME'] in client.database_names():
client.drop_database(DB_INFO['DB_NAME'])
if DB_INFO['CACHE_DB_NAME'] in client.database_names():
client.drop_database(DB_INFO['CACHE_DB_NAME'])
db = client[DB_INFO['DB_NAME']]
cdb = client[DB_INFO['CACHE_DB_NAME']]
SD = sd('tester')
baseSD = sd('baseTester')
os.system('rm -rf importDir')
shutil.copytree('importDirBak','importDir')
SD.connectMongo()
baseSD.connectMongo()
class MongoDataStore(APIView):
permission_classes = ( IsAuthenticatedOrReadOnly,)
renderer_classes = (DataBrowsableAPIRenderer, mongoJSONRenderer, mongoJSONPRenderer, XMLRenderer, YAMLRenderer)
title = "Database"
parser_classes = (JSONParser,)
def __init__(self, connect_uri=config.DATA_STORE_MONGO_URI):
self.db = MongoClient(host=connect_uri)
def get(self, request, database=None, format=None):
urls = []
if database:
self.title = "Collection"
data = list(self.db[database].collection_names())
#print data
data.sort()
for col in data:
urls.append(reverse('data-detail', kwargs={'database': database, 'collection': col}, request=request))
return Response({'Database': database, 'Available Collections': urls})
else:
self.title = "Database"
data = list(self.db.database_names())
data.sort()
for db in data:
urls.append(reverse('data-list', kwargs={'database': db}, request=request))
return Response({
'Available Databases': urls})
class MongoQuery: def __init__(self): self.client = MongoClient() def _get_db(self): db_list = [] for db in self.client.database_names(): db_list.append(self.client[db]) return db_list def _get_col(self, db): col_list = [] for col in db.collection_names(): col_list.append(db[col]) return col_list def _get_doc(self, col): doc_list = [] for doc in col.find(): doc_list.append(doc) return doc_list def query_all(self): for db_object in self._get_db(): print db_object.name for col_ob in self._get_col(db_object): print '\n' + '+'*6 + col_ob.name for doc in self._get_doc(col_ob): print '\n' + '+'*12 + str(doc)
class MongoDbItemsRepository(): #mongodb://admin:[email protected]:36709/ def __init__(self, connectionString='mongodb://*****:*****@ds036709.mlab.com:36709/'): self.connectionString = connectionString self.dbName = 'collections' self._connect() self.db = self.client[self.dbName] def _connect(self): self.client = MongoClient(self.connectionString + self.dbName) def database_exists(self, databaseName): names = self.client.database_names() return databaseName in names def insert_item(self, collectionName, item): collection = self.db[str(collectionName)] return collection.insert_one(item).inserted_id def get_items(self, collectionName, page=1, pageSize=100): collection = self.db[str(collectionName)] count = 0 res = [] for x in collection.find(): res.append(x) count+=1 if count >= pageSize: break return res def get_item_by_id(self, collectionName, itemUUID): collection = self.db[str(collectionName)] for x in collection.find({"uuid":itemUUID}): return x
def make_mongodump(name,exp_uid_list=[]):
tmp_dir = tempfile.mkdtemp()
if len(exp_uid_list)==0:
subprocess.call(('/usr/bin/mongodump -vvvvv --host {hostname}:{port} '
'--out {path}').format(hostname=constants.MONGODB_HOST,
port=constants.MONGODB_PORT,
path=tmp_dir),
shell=True)
else:
exp_uid_list_str = '["'+'","'.join(exp_uid_list)+'"]'
query_str = '\'{ $or: [ {"exp_uid":{$in:%s}}, {"object_id":{$in:%s}} ] }\'' % (exp_uid_list_str,exp_uid_list_str)
# subprocess.call(('/usr/bin/mongodump -vvvvv --host {hostname}:{port} '
# '--out {path} --query {query_str}').format(hostname=constants.MONGODB_HOST,
client = MongoClient(constants.MONGODB_HOST, constants.MONGODB_PORT)
for db in client.database_names():
for col in client[db].collection_names():
subprocess.call(('/usr/bin/mongodump -vvvvv --host {hostname}:{port} '
'--out {path} -d '+str(db)+' -c '+str(col)+' --query {query_str}').format(hostname=constants.MONGODB_HOST,
port=constants.MONGODB_PORT,
path=tmp_dir,
query_str=query_str),
shell=True)
subprocess.call('mkdir -p /dump',shell=True)
subprocess.call(('tar czf /dump/{name} '
'-C {path} .').format(name=name,path=tmp_dir),
shell=True)
shutil.rmtree(tmp_dir)
return '/dump/{}'.format(name)
def estimate_backup_size(self):
"""Estimate the size (in bytes) of the backup this plugin would
produce, if run.
:returns: int. size in bytes
"""
ret = 0
uri = self.config["mongodump"].get("uri")
if uri and uri != ['']:
uri = ','.join(uri)
else:
uri = "mongodb://"
username = self.config["mongodump"].get("username")
if username:
uri += urllib.parse.quote_plus(username)
password = self.config["mongodump"].get("password")
if password:
uri += ":" + urllib.parse.quote_plus(password)
uri += "@"
uri += self.config["mongodump"].get("host")
client = MongoClient(uri)
dbs = client.database_names()
for database in dbs:
c_db = client[database]
tup = c_db.command("dbstats")
ret += int(tup["storageSize"])
# Give an upper estimate to make sure that we have enough disk space
return ret * 2
def getTableList(uri, internalTables=False, **kwargs):
"""
Get a list of known databases, each of which has a list of known
collections from the database. This is of the form [{'database':
(database 1), 'tables': [{'table': (collection 1)}, {'table':
(collection 2)}, ...]}, {'database': (database 2), 'tables': [...]},
...]
:param uri: uri to connect to the database.
:param internaltables: True to return tables about the database itself.
Ignored for Mongo.
:returns: A list of known collections.
"""
conn = MongoClient(uri)
databaseName = base.databaseFromUri(uri)
if databaseName is None:
databaseNames = conn.database_names()
else:
databaseNames = [databaseName]
results = []
for name in databaseNames:
database = conn[name]
results.append({
'database': name,
'tables': [{'table': collection, 'name': collection}
for collection in database.collection_names(False)]
})
return results
class Database:
''' Database类连接MongoDB数据库进行操作
:param str host: 数据库主机，默认是'localhost'
:param int port: 数据库端口，默认是27017
:return: 无返回值
'''
def __init__(self,host='localhost',port=27017):
self.client = MongoClient(host,port)
def connect(self,database_name,collection_name):
'''连接MongoDB数据中的集合
:param str database_name: 数据库名称
:param str collection_name: 集合名称
:return: 数据集合的连接
:rtype: pymongo.collection.Collection对象
'''
if database_name in self.client.database_names():
self.db = self.client[database_name]
else:
print('No such database: ',database_name)
raise NameError
if collection_name in self.db.collection_names():
self.collection = self.db[collection_name]
else:
print('No such collection: ',collection_name)
raise NameError
return self.collection
def verify(self,head='',context='',ip='',port='27017',productname={},keywords='',hackinfo=''):
result = {}
result['result']=False
r=None
try:
r = MongoClient(ip, 27017, connectTimeoutMS=1000, socketTimeoutMS=1000, waitQueueTimeoutMS=1000)
serverInfo = r.server_info()
dbList = r.database_names()
result['result']=True
result['VerifyInfo'] = {}
result['VerifyInfo']['type']='MongoClient unauth'
result['VerifyInfo']['URL'] =ip+':'+port
result['VerifyInfo']['payload']='None'
result['VerifyInfo']['result'] ='MongoClient unauth'
except Exception,e:
print e.text
def main():
parser = argparse.ArgumentParser(description='Run experiments')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--db', help="name of mongo database")
group.add_argument('--list', const=True, action='store_const', help="list available databases")
parser.add_argument('-c', '--col')
args = parser.parse_args()
mongo_client = MongoClient()
if args.list:
return print(mongo_client.database_names())
db = mongo_client[args.db]
headers = []
#for evset in "basic partial full".split():
for colName in db.collection_names():
# for t in [1, 3, 10, 30, 90, 180, 300]:
benches = defaultdict(lambda: defaultdict(list))
#colName = "stab_%s_%ss" % (evset, t)
print("===%s==="%colName)
headers += [colName]
col = db[colName]
for r in col.find():
try:
n = r['ann'] if 'ann' in r else r['name']
except Exception as err:
print(err)
for k,v in r.items():
benches[n][k] += [v]
for k,r in benches.items():
print( r['ann'] if 'ann' in r else r['name'])
for k,v in r.items():#[('instructions', r['instructions'])]: #r.items():
print(k, ["{:.3f}".format(v) for v in v if isinstance(v, (int,float))])
def validateDatabase():
client = MongoClient()
databases = client.database_names()
if any(dbName in s for s in databases):
uploadMongoCollectionsToS3()
else:
print "Please Provide a Valid Database Name"
def test_unix_socket(self):
authed_client = auth_context.client
if not hasattr(socket, "AF_UNIX"):
raise SkipTest("UNIX-sockets are not supported on this system")
if (sys.platform == 'darwin' and
not version.at_least(authed_client, (2, 7, 1))):
raise SkipTest("SERVER-8492")
mongodb_socket = '/tmp/mongodb-27017.sock'
if not os.access(mongodb_socket, os.R_OK):
raise SkipTest("Socket file is not accessable")
self.assertTrue(MongoClient("mongodb://%s" % mongodb_socket))
authed_client.admin.add_user('admin', 'pass')
try:
client = MongoClient("mongodb://%s" % mongodb_socket)
client.admin.authenticate('admin', 'pass')
client.pymongo_test.test.save({"dummy": "object"})
# Confirm we can read via the socket
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
# Confirm it fails with a missing socket
self.assertRaises(ConnectionFailure, MongoClient,
"mongodb:///tmp/none-existent.sock")
finally:
authed_client.admin.remove_user('admin')
def loop(ip):
try:
#print "[-] Scan %s ......"%ip
client = MongoClient(ip,27017,connectTimeoutMS=1000,socketTimeoutMS=1000,waitQueueTimeoutMS=1000)
serverInfo = client.server_info()
try:
serverInfo = client.server_info()
print ip + "\n"
for keys in serverInfo:
print "[-]"+(keys)+":"+str(serverInfo[keys])
#print "\n"
#print colored("[-] %s Auth Succeeded"%ip,'red')
#print "[-] Get %s Server Information Succeeded!"%ip
try:
dbList = client.database_names()
print ip + "\n"
for i in dbList:
print "[-] "+i
#print "\n"
#print "[-] Find: %s Auth Succeeded!"%ip
#print
result.append(ip)
except:
pass
except:
pass
client.close()
#print str(ip)+" Congratulations, NO auth needed!"
except KeyboardInterrupt:
print "[-] Interrupted by user. Exiting..."
exit()
except:
#print str(ip)+" Connection Failed :<"
pass
def validateDatabase():
client = MongoClient()
databases = client.database_names()
if any(dbName in s for s in databases):
print "*************"
else:
print "Please Provide a Valid Database Name"
def test():
# Please change only the add_field and update_db functions!
# Changes done to this function will not be taken into account
# when doing a Test Run or Submit, they are just for your own reference
# and as an example for running this code locally!
data = add_field(DATAFILE, FIELDS)
from pymongo import MongoClient
client = MongoClient("mongodb://localhost:27017")
print "client"
print client.database_names()
db = client.examples
print db.collection_names()
update_db(data, db)
updated = db.arachnid.find_one({'label': 'Opisthoncana'})
assert updated['classification']['binomialAuthority'] == 'Embrik Strand'
pprint.pprint(data)
def connectionMongo(self,ip='127.0.0.1',port=27017):#Todo fix ip addres
client=MongoClient('192.168.1.235',27017,serverSelectionTimeoutMS=10)
try:
return client.database_names()[:-1]
logging.info('Connection established')
except errors.ServerSelectionTimeoutError:
logging.error('Connection problem')
return ''
def _drop_database(self, database_name):
c = MongoClient()
try:
if database_name in c.database_names():
self.log("Dropping database: %s" % database_name)
c.drop_database(database_name)
finally:
c.close()
def test_gssapi_simple(self):
# Call authenticate() without authMechanismProperties.
client = MongoClient(GSSAPI_HOST, GSSAPI_PORT)
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
client.test.collection.find_one()
# Log in using URI, without authMechanismProperties.
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
client.test.collection.find_one()
# Call authenticate() with authMechanismProperties.
self.assertTrue(client.test.authenticate(
PRINCIPAL, mechanism='GSSAPI',
authMechanismProperties='SERVICE_NAME:mongodb'))
client.test.collection.find_one()
# Log in using URI, with authMechanismProperties.
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI;authMechanismProperties'
'=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL),
GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
client.test.collection.find_one()
set_name = client.admin.command('ismaster').get('setName')
if set_name:
client = MongoClient(GSSAPI_HOST,
port=GSSAPI_PORT,
replicaSet=set_name)
# Without authMechanismProperties
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
'=%s' % (quote_plus(PRINCIPAL),
GSSAPI_HOST, GSSAPI_PORT, str(set_name)))
client = MongoClient(uri)
client.database_names()
# With authMechanismProperties
self.assertTrue(client.test.authenticate(
PRINCIPAL, mechanism='GSSAPI',
authMechanismProperties='SERVICE_NAME:mongodb'))
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
'=%s;authMechanismProperties'
'=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL),
GSSAPI_HOST,
GSSAPI_PORT,
str(set_name)))
client = MongoClient(uri)
client.database_names()
def insert(file_name):
from pymongo import MongoClient
client = MongoClient('mongodb://localhost:27017/')
print(client.database_names())
db = client.city.cities
with open(file_name, 'r') as f:
data = json.loads(f.read())
db.insert(data)
print(db.find_one())
def validateDatabase(database, getInternalIP):
client = MongoClient(getInternalIP)
databases = client.database_names()
if database in databases:
return True
else:
logging.exception("Provided Database "+ database+ " Not exists...")
print "Please Provide a Valid Database Name"
return False
def custom_init(self, pinit=False, pdestroy=False, **kwargs):
""" Note: we ignore args here """
# recover instance with the parent method
db = super().custom_init()
if pdestroy:
# massive destruction
client = db.connection.database
from pymongo import MongoClient
client = MongoClient(self.variables.get('host'),
int(self.variables.get('port')))
system_dbs = ['admin', 'local', 'config']
for db in client.database_names():
if db not in system_dbs:
client.drop_database(db)
log.critical("Dropped db '%s'", db)
# if pinit:
# pass
return db
def get_corsor(self):
host = eval(self.hosts)
client = MongoClient(host)
query_line = eval(self.query)
returns_line = eval(self.returns)
options_line = self.options
db = client.get_database(
self.database, read_preference=ReadPreference.SECONDARY_PREFERRED)
dblist = client.database_names()
##连接MongoDB
if self.database in dblist:
print(self.database, "database exists in", dblist)
collist = db.collection_names()
if self.collection in collist:
print(self.collection, "collection exists in", collist)
col = db[self.collection]
print('Connection to mongodb', self.hosts, self.database,
self.collection, 'success!')
else:
print(self.collection, "collection not exists in", collist)
else:
print(self.database, "database not exists in", dblist)
##连接成功后，确认查询类别，支持aggregate,distinct及find查询
if self.query_type == "aggregate":
cursor = col.aggregate(pipeline=eval(query), allowDiskUse=True)
elif self.query_type == "distinct":
cursor = col.distinct(query, options)
else:
print('Start query now')
#cursor = col.find(query_line,returns_line,options_line)
cursor = col.find({}, {
'searchId': 1,
'_id': 0
},
no_cursor_timeout=True)
return cursor
class MongoWrapper:
ressource_static = None
def __init__(self, db_name="test_db"):
""" Init Connection to MongoDB """
if not MongoWrapper.ressource_static:
self.ressource = MongoClient()
MongoWrapper.ressource_static = self.ressource
else:
self.ressource = MongoWrapper.ressource_static
self.db_name = db_name
def useDB(self, db_name):
self.db_name = db_name
def getCollectionNames(self):
return self.ressource[self.db_name].collection_names()
def getDbs(self):
return self.ressource.database_names()
def getCollection(self, col_name):
return self.ressource[self.db_name][col_name]
def refresh_server(self,db_name):
self.dbroot_node.takeChildren()
mongo=MongoClient(['localhost:27017'])
dbnames=mongo.database_names()
for dbname in dbnames:
dbtree_node = QTreeWidgetItem(MainWindow.c_db_node)
dbtree_node.setText(0,QString(dbname))
dbtree_node.setIcon(0,QIcon('../icon/db.svg'))
self.dbroot_node.addChild(dbtree_node)
db = mongo.get_database(dbname)
collections_node = \
QTreeWidgetItem(MainWindow.c_coll_folder_node)
collections_node.setText(0,self.tr('collections'))
collections_node.setIcon(0,QIcon('../icon/table.svg'))
dbtree_node.addChild(collections_node)
cols= db.collection_names()
for col in cols:
col_node = QTreeWidgetItem(MainWindow.c_coll_node)
col_node.setText(0,QString(col))
col_node.setIcon(0,QIcon('../icon/table.svg'))
collections_node.addChild(col_node)
functions_node = QTreeWidgetItem(MainWindow.c_func_folder_node)
functions_node.setText(0,self.tr('functions'))
functions_node.setIcon(0,QIcon('../icon/function.svg'))
dbtree_node.addChild(functions_node)
users_node = QTreeWidgetItem(MainWindow.c_user_folder_node)
users_node.setText(0,self.tr('users'))
users_node.setIcon(0,QIcon('../icon/user.svg'))
dbtree_node.addChild(users_node)
"""
远程操作mongodb
http://api.mongodb.com/python/current/installation.html
http://api.mongodb.com/python/current/tutorial.html
"""
from pymongo import MongoClient
client = MongoClient('mongodb://118.31.19.120:27017/')
print(client.database_names())
db = client.get_database('node_club_dev')
collections = db.collection_names()
print(collections)
users = db.get_collection('users')
print(users.find())
users.find_one_and_update({"loginname": "testuser"},
{'$set': {
"active": True
}})
def init(workDBName,
dropWorkDB=False,
matchDBName=None,
dropMatchDB=False,
indexes=False):
#Indexes only used from standalone, long-running programs
#If called from UI.py it interferes with database deletion
cmn = {}
#cmn['RGDid'] = RGDadm.seq
cmn['contributor'] = RGDadm.contributor
cmn['contribution'] = RGDadm.contribution
client = MongoClient()
if dropWorkDB:
client.drop_database(workDBName)
for mdb in client.database_names():
for coll in client[mdb].collection_names():
if coll.endswith(workDBName):
print 'Dropping', mdb, coll
client[mdb][coll].drop()
db = client[workDBName]
cmn['workDB'] = workDBName
cmn['persons'] = db.persons
cmn['families'] = db.families
cmn['relations'] = db.relations
cmn['originalData'] = db.originalData
if indexes:
cmn['relations'].ensure_index('relTyp')
cmn['relations'].ensure_index('famId')
cmn['relations'].ensure_index('persId')
cmn['originalData'].ensure_index('recordId')
cmn['originalData'].ensure_index('type')
if matchDBName:
cmn['matchDB'] = matchDBName
cmn['matches'] = db['matches_' + matchDBName]
cmn['fam_matches'] = db['fam_matches_' + matchDBName]
cmn['flags'] = db['flags_' + matchDBName]
matchClient = MongoClient()
if dropMatchDB:
matchClient.drop_database(matchDBName)
for mdb in matchClient.database_names():
for coll in matchClient[mdb].collection_names():
if coll.endswith(matchDBName):
print 'Dropping', mdb, coll
matchClient[mdb][coll].drop()
matchDB = matchClient[matchDBName]
cmn['match_persons'] = matchDB.persons
cmn['match_families'] = matchDB.families
cmn['match_relations'] = matchDB.relations
cmn['match_originalData'] = matchDB.originalData
if indexes:
cmn['matches'].ensure_index('workid')
cmn['matches'].ensure_index('matchid')
cmn['matches'].ensure_index('status')
cmn['fam_matches'].ensure_index('workid')
cmn['fam_matches'].ensure_index('matchid')
cmn['fam_matches'].ensure_index('status')
cmn['match_relations'].ensure_index('relTyp')
cmn['match_relations'].ensure_index('famId')
cmn['match_relations'].ensure_index('persId')
cmn['match_originalData'].ensure_index('recordId')
cmn['match_originalData'].ensure_index('type')
return cmn
class MongoDB(object):
_instance = None
def __init__(self,
host=None,
port=None,
database_name=None,
user=None,
password=None,
collection_name=None,
drop_n_create=False):
try:
# self._connection = MongoClient(host=host, port=port, maxPoolSize=200)
# address = 'mongodb://*****:*****@101"@{}:{}/{}'.format(host, database_name)
self._connection = MongoClient(host=host, port=port)
except Exception as error:
raise Exception(error)
if drop_n_create:
self.drop_db(database_name)
self._database = None
self._collection = None
if database_name:
self._database = self._connection[database_name]
self._database.authenticate(user, password)
if collection_name:
self._collection = self._database[collection_name]
@classmethod
def get_instance(cls):
if not cls._instance:
cls._instance = MongoDB()
return cls._instance
@staticmethod
def check_state(obj):
if not obj:
return False
else:
return True
def check_db(self):
if not self.check_state(self._database):
# validate the database name
raise ValueError('Database is empty/not created')
def check_collection(self):
# validate the collection name
if not self.check_state(self._collection):
raise ValueError('Collection is empty/not created')
def get_overall_details(self):
# get overall connection information
client = self._connection
details = dict(
(db, [collection for collection in client[db].collection_names()])
for db in client.database_names())
return details
def get_current_status(self):
# get current connection information
return {
'connection': self._connection,
'database': self._database,
'collection': self._collection
}
def create_db(self, database_name=None):
# create the database name
self._database = self._connection[database_name]
def create_collection(self, collection_name=None):
# create the collection name
self.check_db()
self._collection = self._database[collection_name]
def get_database_names(self):
# get the database name you are currently connected too
return self._connection.database_names()
def get_collection_names(self):
# get the collection name you are currently connected too
self.check_collection()
return self._database.collection_names(
include_system_collections=False)
def drop_db(self, database_name):
# drop/delete whole database
self._database = None
self._collection = None
return self._connection.drop_database(str(database_name))
def drop_collection(self):
# drop/delete a collection
self._collection.drop()
self._collection = None
def insert(self, post):
# add/append/new single record
self.check_collection()
post_id = self._collection.insert_one(post).inserted_id
return post_id
def insert_many(self, posts):
# add/append/new multiple records
self.check_collection()
result = self._collection.insert_many(posts)
return result.inserted_ids
def find_one(self, *args):
# search/find only one matching record
return self._collection.find_one(*args)
def find(self, count=False, *args):
# search/find many matching records
self.check_collection()
if not count:
return self._collection.find(*args)
# return only count
return self._collection.find(*args).count()
def findMany(self, *args):
return self._collection.find(*args)
def find_max(self, count_key, *args):
# search/find many matching records
self.check_collection()
return self._collection.find(*args).sort([(count_key, -1)]).limit(1)
def find_min(self, count_key, *args):
# search/find many matching records
self.check_collection()
return self._collection.find(*args).sort([(count_key, 1)]).limit(1)
def count(self):
# get the records count in collection
self.check_collection()
return self._collection.count()
def remove(self, *args):
# remove/delete records
return self._collection.remove(*args)
def update(self, object_id, post):
# updating/modifying the records
self.check_collection()
return self._collection.update({"ObjectId": "{}".format(object_id)},
{"$push": post})
# pass
def updateByParams(self, *args):
self.check_collection()
return self._collection.update_one(*args)
def create_index(self):
# create a single index
pass
def create_indexes(self):
# create multiple indexes
pass
def close(self):
return self._connection.close()
def get_collection(self, collection_name=None):
# get collection name
self.check_db()
try:
self._collection = self._database.get_collection(collection_name)
return self
except Exception as e:
return None
class MongoKnowledgeBase:
"""
Use Mongo Database to store and manage knowledge data
"""
# Constants
FACTS_COLLECTION_NAME = "facts"
RULES_COLLECTION_NAME = "rules"
def __init__(self):
self.__client = None
self.__data_base = None
self.__facts = None
self.__rules = None
self.root_facts = list()
pass
# TODO: error handling
def connect(self, db_name, mongo_url='localhost', mongo_port=27017):
self.__client = MongoClient(mongo_url, mongo_port)
# TODO: temporary solution. Always from scratch
if db_name in self.__client.database_names():
self.__client.drop_database(db_name)
pass
self.__data_base = self.__client[db_name]
self.__facts = self.__data_base.facts
self.__rules = self.__data_base.rules
pass
def __clean(self):
if self.__data_base is not None:
self.__data_base.drop_collection(self.FACTS_COLLECTION_NAME)
self.__data_base.drop_collection(self.RULES_COLLECTION_NAME)
self.root_facts = list()
pass
def facts_count(self):
return self.__facts.count()
def rules_count(self):
return self.__rules.count()
def set_decision_graph(self, decision_graph_data):
self.__clean()
graph = json_graph.node_link_graph(decision_graph_data)
consequent_list = list()
for node in graph:
if graph.in_degree(node) == 0: # root
self.root_facts.append(node)
if graph.node[node]['type'] == FactType.Consequent: # consequent
consequent_list.append(node)
self.__facts.insert_one(self.transform_node_to_fact(node, graph))
pass
print("Consequents count:", len(consequent_list))
for rule in self.transform_to_rules(consequent_list, graph):
self.__rules.insert_one(rule)
pass
pass
# TODO: handle intermediate consequents
# TODO: add types and classes for facts (ant, con, incon)
def find_antecedents(self):
return self.__facts.distinct(
ID_STRING, {
'type': {
"$in":
[FactType.Antecedent, FactType.IntermediateConsequent]
}
})
def get_fact_by_id(self, fact_id):
return self.__facts.find_one({ID_STRING: fact_id}, {"_id": 0})
def get_rule_by_id(self, rule_id):
return ProductionRule(
self.__rules.find_one({ID_STRING: rule_id}, {"_id": 0}))
def get_text_description(self, fact_id):
return self.get_fact_by_id(fact_id)['text']
def get_type(self, fact_id):
return self.get_fact_by_id(fact_id)['type']
def get_rules_with_predecessor(self, fact_id):
return [
ProductionRule(x) for x in self.__rules.find(
{PREDECESSORS_STRING: {
'$elemMatch': {
ID_STRING: fact_id
}
}}, {"_id": 0})
]
def transform_node_to_fact(self, node, graph):
attributes = graph.node[node]
fact = {
"id": node,
"type": attributes['type'],
"text": attributes['text']
}
if attributes['type'] == FactType.Consequent:
if graph.out_degree(node) != 0:
fact['type'] = FactType.IntermediateConsequent
fact["text"] = re.sub('\(\d(\.\d*)?\)', '', attributes['text'])
fact['coefficient'] = float(attributes['coefficient'])
return fact
def fix_path_from_intermediate_consequent(self, path):
last_intermediate_consequent = next(
(i for i in reversed(path[:-1])
if self.get_type(i) == FactType.IntermediateConsequent), None)
if last_intermediate_consequent is not None:
path = path[path.index(last_intermediate_consequent):]
return path
def convert_path_to_rule(self, path, graph):
path = self.fix_path_from_intermediate_consequent(path)
consequent_id = path[-1]
rule = {
PREDECESSORS_STRING: [],
SUCCESSOR_STRING: {
ID_STRING:
consequent_id,
COEFFICIENT_STRING:
float(graph.node[consequent_id][COEFFICIENT_STRING])
}
}
for i in range(len(path) - 1):
rule[PREDECESSORS_STRING].append({
ID_STRING:
path[i],
COEFFICIENT_STRING:
float(graph.get_edge_data(path[i], path[i + 1])['weight'])
})
return rule
def transform_to_rules(self, consequent_list, graph):
if len(self.root_facts) == 0:
print("Can not add rules: No root elements")
return
rule_id = 0
for root in self.root_facts:
for consequent in consequent_list:
paths = algorithms.all_simple_paths(graph, root, consequent)
for path in paths:
rule = self.convert_path_to_rule(path, graph)
rule[ID_STRING] = rule_id
rule_id += 1
yield rule
pass
pass
pass
def show_databases(Host, Port):
conn = MongoClient(host=Host, port=Port)
print conn.database_names()
def threadScan(self):
while not self.queue.empty():
ip = self.queue.get()
try:
self.lock.acquire()
print 'try to connect ' + ip + ':' + str(self.port)
self.lock.release()
# 原mongo连接，用MongoClient，空密码连接
client = MongoClient(ip,
self.port,
connectTimeoutMS=self.conn_timeout * 1000,
socketTimeoutMS=self.conn_timeout * 1000,
waitQueueTimeoutMS=self.conn_timeout *
1000)
client.server_info()
client.database_names()
client.close()
self.lock.acquire()
print '########## ' + ip + ':' + str(
self.port) + ' Connect Success!'
file = open(self.fileout, 'a')
file.write(ip + ':' + str(self.port) + '\n')
file.close()
self.success_count += 1
self.lock.release()
except Exception, e:
self.lock.acquire()
print e
self.lock.release()
# TODO 读取用户名密码登陆
try:
success_flag = False
user_file = open(users, 'r')
for user in user_file:
user = user.strip()
pass_file = open(passwds, 'r')
for passwd in pass_file:
passwd = passwd.strip()
try:
self.lock.acquire()
print 'try to connect ' + ip + ':' + str(
self.port) + ' by %s:%s' % (user, passwd)
self.lock.release()
client = MongoClient(
"mongodb://%s:%s@%s:27017/" %
(user, passwd, ip))
client.server_info()
client.database_names()
client.close()
self.lock.acquire()
success_flag = True
print '########## ' + ip + ':' + str(
self.port
) + ' Connect with %s:%s Success!' % (user,
passwd)
file = open(self.fileout, 'a')
file.write(ip + ':' + str(self.port) + '\t' +
user + ':' + passwd + '\n')
file.close()
self.success_count += 1
self.lock.release()
break # 停止遍历
except Exception, e:
self.lock.acquire()
print e
self.lock.release()
self.lock.acquire()
if success_flag:
success_flag = False
self.lock.release()
break
else:
self.lock.release()
except Exception, e:
print e
exit()
class DatabaseInstance:
"""MongoDB Database wrapper."""
def __init__(self, config: DatabaseConfig) -> None:
"""Initialize wrapper."""
self.config = config
self.client = MongoClient(config.connection_string)
self.database = self.client.get_database(config.database_name)
def __enter__(self):
if self.config.restore_from_dump == RestoreMode.ALWAYS or (
self.config.restore_from_dump == RestoreMode.ONLY_NEW and
self.config.database_name not in self.client.database_names()):
self.restore()
return self
def __exit__(self, exc_type, exc_value, traceback):
if self.config.dump_on_exit:
self.enable_cascades(False)
self.dump()
def drop(self):
"""Drop the database."""
self.client.drop_database(self.config.database_name)
def restore(self):
"""Restore the database from the 'self.dump_directory'."""
subprocess.run([
'mongorestore', '--nsInclude', f'{self.config.database_name}.*',
'--drop'
],
shell=True,
check=True,
cwd=self.config.dump_path)
def dump(self):
"""Dump the database into 'self.dump_directory'."""
subprocess.run(['mongodump', '--db', self.config.database_name],
cwd=self.config.dump_path,
check=True)
def enable_sbe(self, state: bool) -> None:
"""Enable new query execution engine. Throw pymongo.errors.OperationFailure in case of failure."""
self.client.admin.command({
'setParameter':
1,
'internalQueryFrameworkControl':
'trySbeEngine' if state else 'forceClassicEngine'
})
def enable_cascades(self, state: bool) -> None:
"""Enable new query optimizer. Requires featureFlagCommonQueryFramework set to True."""
self.client.admin.command({
'configureFailPoint': 'enableExplainInBonsai',
'mode': 'alwaysOn'
})
self.client.admin.command({
'setParameter':
1,
'internalQueryFrameworkControl':
'tryBonsai' if state else 'trySbeEngine'
})
def explain(self, collection_name: str,
pipeline: Pipeline) -> dict[str, any]:
"""Return explain for the given pipeline."""
return self.database.command('explain', {
'aggregate': collection_name,
'pipeline': pipeline,
'cursor': {}
},
verbosity='executionStats')
def hide_index(self, collection_name: str, index_name: str) -> None:
"""Hide the given index from the query optimizer."""
self.database.command({
'collMod': collection_name,
'index': {
'name': index_name,
'hidden': True
}
})
def unhide_index(self, collection_name: str, index_name: str) -> None:
"""Make the given index visible for the query optimizer."""
self.database.command({
'collMod': collection_name,
'index': {
'name': index_name,
'hidden': False
}
})
def hide_all_indexes(self, collection_name: str) -> None:
"""Hide all indexes of the given collection from the query optimizer."""
for index in self.database[collection_name].list_indexes():
if index['name'] != '_id_':
self.hide_index(collection_name, index['name'])
def unhide_all_indexes(self, collection_name: str) -> None:
"""Make all indexes of the given collection visible fpr the query optimizer."""
for index in self.database[collection_name].list_indexes():
if index['name'] != '_id_':
self.unhide_index(collection_name, index['name'])
def drop_collection(self, collection_name: str) -> None:
"""Drop collection."""
self.database[collection_name].drop()
def insert_many(self, collection_name: str,
docs: Sequence[Mapping[str, any]]) -> None:
"""Insert documents into the collection with the given name."""
requests = [InsertOne(doc) for doc in docs]
self.database[collection_name].bulk_write(requests, ordered=False)
def get_all_documents(self, collection_name: str):
"""Get all documents from the collection with the given name."""
return self.database[collection_name].find({})
def get_stats(self, collection_name: str):
"""Get collection statistics."""
return self.database.command('collstats', collection_name)
def get_average_document_size(self, collection_name: str) -> float:
"""Get average document size for the given collection."""
stats = self.get_stats(collection_name)
avg_size = stats.get('avgObjSize')
return avg_size if avg_size is not None else 0
class DataLayer:
client = None
db = None
def __init__(self, db_conn_str, db_database):
self.db_connect(db_conn_str, db_database)
def db_connect(self, db_conn_str, db_database):
#
# db connect
#
self.client = MongoClient(db_conn_str)
if db_database not in self.client.database_names():
print(("database list: " + str(self.client.database_names())))
raise ValueError("requested database not found")
self.db = self.client[db_database]
def find_db_data(self, db_collection, file_path, find_where, find_select, db_conn_str=None, db_database=None):
if db_conn_str is not None and db_database is not None:
self.db_connect(db_conn_str, db_database)
if db_collection not in self.db.collection_names():
print(("collection list: " + str(self.db.collection_names())))
raise ValueError("requested collection not found")
collection = self.db[db_collection]
#
# data query
#
db_documents = []
if find_select is not None and len(find_select) > 0:
db_documents = list(collection.find(filter=find_where, projection=find_select))
else:
db_documents = list(collection.find(filter=find_where))
print(("Found {0} {1} documents".format(len(db_documents), db_collection)))
#
# save result data
#
pickle.dump(db_documents, open(file_path, 'wb'))
print(("Saved documents to {0}".format(file_path)))
def update_db_data(self, db_collection, index_field, update_field, update_data, db_conn_str=None, db_database=None):
if db_conn_str is not None and db_database is not None:
self.db_connect(db_conn_str, db_database)
if db_collection not in self.db.collection_names():
print(("collection list: " + str(self.db.collection_names())))
raise ValueError("requested collection not found")
collection = self.db[db_collection]
match_count = 0
update_count = 0
for update in update_data:
result = collection.update_one({index_field:update[index_field]},{'$set':{update_field:update[update_field]}})
match_count += result.matched_count
update_count += result.modified_count
print(("{0}.{1} updated/matched {2}/{3}".format(db_collection, update_field, update_count, match_count)))
class Mongo(Entity):
def __init__(self, bmss, config):
"""Setup the MongoDB component."""
self.__state = STATE_UNKNOWN
self.bmss = bmss
self.__config = config[DOMAIN]
self.__host = util.convert(self.__config.get(CONF_HOST), str,
DEFAULT_HOST)
self.__port = util.convert(self.__config.get(CONF_PORT), int,
DEFAULT_PORT)
self.__tz_aware = util.convert(self.__config.get(CONF_TZ_AWARE), bool,
DEFAULT_TZ_AWARE)
self.__socket_timeout_ms = util.convert(
self.__config.get(CONF_SOCKET_TIMEOUT_MS), int,
DEFAULT_SOCKET_TIMEOUT_MS)
self.__ssl = util.convert(self.__config.get(CONF_SSL), bool,
DEFAULT_SSL)
self.__max_pool_size = util.convert(
self.__config.get(CONF_MAX_POOL_SIZE), int, DEFAULT_MAX_POOL_SIZE)
self.__socket_keep_alive = util.convert(
self.__config.get(CONF_SOCKET_KEEP_ALIVE), int,
DEFAULT_SOCKET_KEEP_ALIVE)
from pymongo import MongoClient
from pymongo.monitoring import CommandListener
class MongoCommandEvent(CommandListener):
"""
https://api.mongodb.com/python/current/api/pymongo/monitoring.html#module-pymongo.monitoring
"""
def started(self, event):
_LOGGER.debug("Command {0.command_name} with request id "
"{0.request_id} started on server "
"{0.connection_id}".format(event))
def succeeded(self, event):
_LOGGER.info("Command {0.command_name} with request id "
"{0.request_id} on server {0.connection_id} "
"succeeded in {0.duration_micros} "
"microseconds".format(event))
def failed(self, event):
_LOGGER.warn("Command {0.command_name} with request id "
"{0.request_id} on server {0.connection_id} "
"failed in {0.duration_micros} "
"microseconds".format(event))
self.__client = MongoClient(host=self.__host,
port=self.__port,
tz_aware=self.__tz_aware,
maxPoolSize=self.__max_pool_size,
socketTimeoutMS=self.__socket_timeout_ms,
ssl=self.__ssl,
socketKeepAlive=self.__socket_keep_alive,
document_class=dict,
connect=True,
event_listeners=[MongoCommandEvent()])
# Will fail here if connection is not able to be established
assert (self.__client is not None)
self.__state = STATE_IDLE
bmss.bus.listen_once(EVENT_BLUMATE_STOP, self.disconnect)
bmss.bus.listen_once(EVENT_BLUMATE_START, self.discover_databases)
bmss.services.register(DOMAIN, SERVICE_DISCOVER_DATABASES,
self.discover_databases)
bmss.services.register(DOMAIN, SERVICE_UNLOCK, self.unlock)
bmss.services.register(DOMAIN, SERVICE_DISCONNECT, self.disconnect)
def discover_databases(self, event):
"""Discover available databases."""
self.__state = STATE_ACTIVE
database_list = self.__client.database_names()
self.__state = STATE_IDLE
_LOGGER.info("Available Databases: %s", database_list)
def unlock(self, event):
"""Enables writes to the server."""
_LOGGER.debug("Unlocking server...")
self.__client.unlock()
if self.__client.is_locked:
_LOGGER.warn("Server is still locked. Maybe a permissions issue?")
else:
_LOGGER.info("Server is unlocked.")
def disconnect(self, event):
"""Disconnect from the MongoDB Server."""
_LOGGER.debug("Disconnecting from MongoDB Server...")
self.__client.close()
_LOGGER.info("Disconnected from MongoDB Server.")
result = {'users': 'ok'}
return result
# --------------------------------------------------------------------------------------------------------------------
def drop_test_database():
cprint('WARNING: DATABASE {} WILL BE DELETED'.format(TEST_DATABASE), color='red', attrs=['bold', 'blink'])
# sleep(60)
client.drop_database(TEST_DATABASE)
def load_pagelist():
collection =db.pages
for type in facebook_page_lists:
for sub_type in facebook_page_lists[type]:
for page_id in facebook_page_lists[type][sub_type]:
page_doc={}
page_doc['type']=type
page_doc['sub_type']=sub_type
page_doc['id']=page_id
page_doc['name']=fb_get_page(page_id)['name'] #{name:..., id:...}
collection.update_one({'id':page_id}, {'$set':page_doc}, upsert=True)
if __name__ == '__main__':
print '_' * 120
print 'Databases: ', client.database_names()
print 'Collections: ', db.collection_names()
print setup_databases()
load_pagelist()
chunk = ""
for segment in segments:
chunk += segment
#try:
#yield json.loads(chunk)
yaml.load(chunk) #get rid of random backslashes!
#yield json.loads(chunk.replace('\r\n', '\\r\\n'))
chunk = ""
#Problem with json file!
'''
except ValueError:
print("error!")
pass
'''
connection = MongoClient('localhost', 27017)
connection.database_names()
db = connection.outlook
count = 0
with open('outlook_contacts.json') as f:
#data = json.load(f)
for parsed_json in load_json_multiple(f):
count += 1
if count % 1000 == 0:
print(count)
#print(parsed_json)
data = parsed_json
contacts = db.cleancontacts
contact_id = contacts.insert_one(data).inserted_id
from pymongo import MongoClient
from pprint import pprint
from pymongo.errors import ConnectionFailure
from ConfigFilesLoader import db_config_data, logger_name
import Logger
import logging
logger = logging.getLogger(logger_name)
db_name = db_config_data['database_info']['name']
collection_names = db_config_data['database_info']['collections']
client = MongoClient(host=db_config_data['host'], port=db_config_data['port'])
## Logic needs to be written in case some connection failure happens
if db_config_data['database_info']['name'] not in client.database_names():
database = client[db_config_data['database_info']['name']]
logger.info("Database created succesfully")
else:
logger.info("Database already exists")
"""
"""
import gridfs
from pymongo import MongoClient
#client = MongoClient('localhost', 27017)
client = MongoClient('psanaphi105', 27017) #, username=uname, password=pwd)
db = client['calib-cxi12345']
fs = gridfs.GridFS(db)
col = db['cxi12345']
docs = col.find()
doc = docs[0]
print('client.database_names(): ', client.database_names())
print('db.collection_names() : ', db.collection_names(False))
print('==== fs :\n', fs)
print('==== col :\n', col)
print('==== docs :\n', docs)
print('==== doc :\n', doc)
print('==== vars(col) :\n', vars(col))
def test_gssapi_simple(self):
client = MongoClient(GSSAPI_HOST, GSSAPI_PORT)
# Without gssapiServiceName
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
client.database_names()
# With gssapiServiceName
self.assertTrue(
client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI',
gssapiServiceName='mongodb'))
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI;gssapiServiceName=mongodb' %
(quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI;authMechanismProperties=SERVICE_NAME:mongodb' %
(quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
client.database_names()
set_name = client.admin.command('ismaster').get('setName')
if set_name:
client = MongoReplicaSetClient(GSSAPI_HOST,
port=GSSAPI_PORT,
replicaSet=set_name)
# Without gssapiServiceName
self.assertTrue(
client.test.authenticate(PRINCIPAL, mechanism='GSSAPI'))
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
'=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT,
str(set_name)))
client = MongoReplicaSetClient(uri)
client.database_names()
# With gssapiServiceName
self.assertTrue(
client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI',
gssapiServiceName='mongodb'))
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
'=%s;gssapiServiceName=mongodb' %
(quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT,
str(set_name)))
client = MongoReplicaSetClient(uri)
client.database_names()
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet=%s;'
'authMechanismProperties=SERVICE_NAME:mongodb' %
(quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT,
str(set_name)))
client = MongoReplicaSetClient(uri)
client.database_names()
print("addRiskNumbers.py <mongoURL> <mongoPort> <userName> <password>")
sys.exit(0)
mongoURL = sys.argv[1]
mongoPort = sys.argv[2]
userName = sys.argv[3]
password = sys.argv[4]
connString = "mongodb://" + userName + ":" + password + "@" + mongoURL + ":" + mongoPort + "/"
##### Enable dry run to not commit to the database #####
dryRun = True
##### Connect to the Database #####
db = MongoClient(connString)
for database in db.database_names():
if database != "admin" and database != "local":
db = MongoClient(connString)[database]
print("--database:" + database)
##### Get a model ID and find entries #####
for setting in db.settings.find(no_cursor_timeout=True):
modelId = setting.get("_id")
print("\t--model: " + modelId)
riskNumber = 1
db[modelId + ".risks"].ensure_index([("created", pymongo.ASCENDING)
])
for risk in db[modelId + ".risks"].find(
no_cursor_timeout=True).sort("created", pymongo.ASCENDING):
riskId = risk.get("_id")
print("\t\t--risk (" + str(riskNumber) + "): " + str(riskId))
#install pymongo
import pip
pip.main(['install','pymongo'])
from pymongo import MongoClient
# connect
# myip = '10.1.2.31' # localhost
# con = MongoClient(myip,27017)
con = MongoClient('localhost', 27017)
# List databases
con.database_names()
# delete test databases if they already exist
try:
con.drop_database("pythondb")
except:
pass
try:
con.drop_database("pythondb2")
except:
pass
con.database_names()
# Create two databases two different ways
db = con.pythondb
db2 = con['pythondb2']
# Lazy creation
con.database_names()
class MongoDB(object):
def __init__(self):
self.mongo_host = "127.0.0.1"
self.mongo_port = 27017
self.mongo_db = [
"admin",
]
self.mongo_user = None
self.mongo_password = None
self.__conn = None
self.__dbnames = None
self.__metrics = []
# Connect to MongoDB
def connect(self):
if self.__conn is None:
try:
self.__conn = MongoClient(host=self.mongo_host,
port=self.mongo_port)
except Exception as e:
print 'Error in MongoDB connection: %s' % str(e)
# Add each mectrics to the metrics list
def addMetrics(self, k, v):
dict = {}
dict['key'] = k
dict['value'] = v
self.__metrics.append(dict)
# Print out all metrics
def printMetrics(self):
metrics = self.__metrics
for m in metrics:
zabbix_item_key = str(m['key'])
zabbix_item_value = str(m['value'])
print '- ' + zabbix_item_key + ' ' + zabbix_item_value
# Get a list of DB names
def getDBNames(self):
if self.__conn is None:
self.connect()
db = self.__conn[self.mongo_db[0]]
if self.mongo_user and self.mongo_password:
db.authenticate(self.mongo_user, self.mongo_password)
master = db.command('isMaster')['ismaster']
dict = {}
dict['key'] = 'mongodb.ismaster'
DBList = []
if master:
dict['value'] = 1
DBNames = self.__conn.database_names()
self.__dbnames = DBNames
else:
dict['value'] = 0
self.__metrics.append(dict)
# Print DB list in json format, to be used for mongo db discovery in zabbix
def getMongoDBLLD(self):
if self.__dbnames is None:
DBNames = self.getDBNames()
else:
DBNames = self.__dbnames
DBList = []
if DBNames is not None:
for db in DBNames:
dict = {}
dict['{#MONGODBNAME}'] = db
DBList.append(dict)
return {"data": DBList}
def getOplog(self):
db = MongoClient(self.mongo_host, self.mongo_port)
if self.mongo_user and self.mongo_password:
db.authenticate(self.mongo_user, self.mongo_password)
dbl = db.local
coll = dbl['oplog.rs']
op_first = (coll.find().sort('$natural', 1).limit(1))
while op_first.alive:
op_fst = (op_first.next())['ts'].time
op_last = (coll.find().sort('$natural', -1).limit(1))
while op_last.alive:
op_last_st = op_last[0]['ts']
op_lst = (op_last.next())['ts'].time
status = round(float(op_lst - op_fst), 1)
self.addMetrics('mongodb.oplog', status)
currentTime = timegm(gmtime())
oplog = int(((str(op_last_st).split('('))[1].split(','))[0])
self.addMetrics('mongodb.oplog-sync', (currentTime - oplog))
def getMaintenance(self):
db = MongoClient(self.mongo_host, self.mongo_port)
if self.mongo_user and self.mongo_password:
db.authenticate(self.mongo_user, self.mongo_password)
host_name = socket.gethostname()
fsync_locked = int(db.is_locked)
config = db.admin.command("replSetGetConfig", 1)
for i in range(0, len(config['config']['members'])):
if host_name in config['config']['members'][i]['host']:
priority = config['config']['members'][i]['priority']
hidden = int(config['config']['members'][i]['hidden'])
self.addMetrics('mongodb.fsync-locked', fsync_locked)
self.addMetrics('mongodb.priority', priority)
self.addMetrics('mongodb.hidden', hidden)
# Get Server Status
def getServerStatusMetrics(self):
if self.__conn is None:
self.connect()
db = self.__conn[self.mongo_db[0]]
if self.mongo_user and self.mongo_password:
db.authenticate(self.mongo_user, self.mongo_password)
ss = db.command('serverStatus')
#print ss
# db info
self.addMetrics('mongodb.version', ss['version'])
self.addMetrics('mongodb.storageEngine', ss['storageEngine']['name'])
self.addMetrics('mongodb.uptime', int(ss['uptime']))
self.addMetrics('mongodb.okstatus', int(ss['ok']))
# asserts
for k, v in ss['asserts'].items():
self.addMetrics('mongodb.asserts.' + k, v)
# operations
for k, v in ss['opcounters'].items():
self.addMetrics('mongodb.operation.' + k, v)
# memory
for k in ['resident', 'virtual', 'mapped', 'mappedWithJournal']:
self.addMetrics('mongodb.memory.' + k, ss['mem'][k])
# connections
for k, v in ss['connections'].items():
self.addMetrics('mongodb.connection.' + k, v)
# network
for k, v in ss['network'].items():
self.addMetrics('mongodb.network.' + k, v)
# extra info
self.addMetrics('mongodb.page.faults', ss['extra_info']['page_faults'])
#wired tiger
self.addMetrics(
'mongodb.used-cache',
ss['wiredTiger']['cache']["bytes currently in the cache"])
self.addMetrics('mongodb.total-cache',
ss['wiredTiger']['cache']["maximum bytes configured"])
self.addMetrics(
'mongodb.dirty-cache',
ss['wiredTiger']['cache']["tracked dirty bytes in the cache"])
# global lock
lockTotalTime = ss['globalLock']['totalTime']
self.addMetrics('mongodb.globalLock.totalTime', lockTotalTime)
for k, v in ss['globalLock']['currentQueue'].items():
self.addMetrics('mongodb.globalLock.currentQueue.' + k, v)
for k, v in ss['globalLock']['activeClients'].items():
self.addMetrics('mongodb.globalLock.activeClients.' + k, v)
# Get DB stats for each DB
def getDBStatsMetrics(self):
if self.__conn is None:
self.connect()
if self.__dbnames is None:
self.getDBNames()
if self.__dbnames is not None:
for mongo_db in self.__dbnames:
db = self.__conn[mongo_db]
if self.mongo_user and self.mongo_password:
self.__conn[self.mongo_db[0]].authenticate(
self.mongo_user, self.mongo_password)
dbs = db.command('dbstats')
for k, v in dbs.items():
if k in [
'storageSize', 'ok', 'avgObjSize', 'indexes',
'objects', 'collections', 'fileSize', 'numExtents',
'dataSize', 'indexSize', 'nsSizeMB'
]:
self.addMetrics(
'mongodb.stats.' + k + '[' + mongo_db + ']',
int(v))
# Close connection
def close(self):
if self.__conn is not None:
self.__conn.close()
class MongoDB(object):
"""main script class"""
# pylint: disable=too-many-instance-attributes
def __init__(self):
self.mongo_host = "localhost"
self.mongo_port = 27017
self.mongo_db = [
"admin",
]
self.mongo_user = None
self.mongo_password = None
self.__conn = None
self.__dbnames = None
self.__metrics = []
def connect(self):
"""Connect to MongoDB"""
if self.__conn is None:
if self.mongo_user is None:
try:
self.__conn = MongoClient(
'mongodb://%s:%s' % (self.mongo_host, self.mongo_port))
except errors.PyMongoError as py_mongo_error:
print('Error in MongoDB connection: %s' %
str(py_mongo_error))
else:
try:
self.__conn = MongoClient(
'mongodb://%s:%s@%s:%s' %
(self.mongo_user, self.mongo_password, self.mongo_host,
self.mongo_port))
except errors.PyMongoError as py_mongo_error:
print('Error in MongoDB connection: %s' %
str(py_mongo_error))
def add_metrics(self, k, v):
"""add each metric to the metrics list"""
dict_metrics = {}
dict_metrics['key'] = k
dict_metrics['value'] = v
self.__metrics.append(dict_metrics)
def print_metrics(self):
"""print out all metrics"""
metrics = self.__metrics
for metric in metrics:
zabbix_item_key = str(metric['key'])
zabbix_item_value = str(metric['value'])
print('- ' + zabbix_item_key + ' ' + zabbix_item_value)
def get_db_names(self):
"""get a list of DB names"""
if self.__conn is None:
self.connect()
db_handler = self.__conn[self.mongo_db[0]]
master = db_handler.command('isMaster')['ismaster']
dict_metrics = {}
dict_metrics['key'] = 'mongodb.ismaster'
if master:
dict_metrics['value'] = 1
db_names = self.__conn.database_names()
self.__dbnames = db_names
else:
dict_metrics['value'] = 0
self.__metrics.append(dict_metrics)
def get_mongo_db_lld(self):
"""print DB list in json format, to be used for mongo db discovery in zabbix"""
if self.__dbnames is None:
db_names = self.get_db_names()
else:
db_names = self.__dbnames
dict_metrics = {}
db_list = []
dict_metrics['key'] = 'mongodb.discovery'
dict_metrics['value'] = {"data": db_list}
if db_names is not None:
for db_name in db_names:
dict_lld_metric = {}
dict_lld_metric['{#MONGODBNAME}'] = db_name
db_list.append(dict_lld_metric)
dict_metrics['value'] = '{"data": ' + json.dumps(db_list) + '}'
self.__metrics.insert(0, dict_metrics)
def get_oplog(self):
"""get replica set oplog information"""
if self.__conn is None:
self.connect()
db_handler = self.__conn['local']
coll = db_handler.oplog.rs
op_first = (coll.find().sort('$natural', 1).limit(1))
op_last = (coll.find().sort('$natural', -1).limit(1))
# if host is not a member of replica set, without this check we will
# raise StopIteration as guided in
# http://api.mongodb.com/python/current/api/pymongo/cursor.html
if op_first.count() > 0 and op_last.count() > 0:
op_fst = (op_first.next())['ts'].time
op_last_st = op_last[0]['ts']
op_lst = (op_last.next())['ts'].time
status = round(float(op_lst - op_fst), 1)
self.add_metrics('mongodb.oplog', status)
current_time = timegm(gmtime())
oplog = int(((str(op_last_st).split('('))[1].split(','))[0])
self.add_metrics('mongodb.oplog-sync', (current_time - oplog))
def get_maintenance(self):
"""get replica set maintenance info"""
if self.__conn is None:
self.connect()
db_handler = self.__conn
fsync_locked = int(db_handler.is_locked)
self.add_metrics('mongodb.fsync-locked', fsync_locked)
def get_server_status_metrics(self):
"""get server status"""
if self.__conn is None:
self.connect()
db_handler = self.__conn[self.mongo_db[0]]
ss = db_handler.command('serverStatus')
# db info
self.add_metrics('mongodb.okstatus', int(ss['ok']))
# extra info
self.add_metrics('mongodb.page.faults',
ss['extra_info']['page_faults'])
# network
self.add_metrics('mongodb.network.bytesIn',
long(ss['network']['bytesIn']))
self.add_metrics('mongodb.network.bytesOut',
long(ss['network']['bytesOut']))
self.add_metrics('mongodb.network.physicalBytesIn',
long(ss['network']['physicalBytesIn']))
self.add_metrics('mongodb.network.physicalBytesOut',
long(ss['network']['physicalBytesOut']))
self.add_metrics('mongodb.network.numRequests',
long(ss['network']['numRequests']))
self.add_metrics(
'mongodb.network.serviceExecutorTaskStats.threadsRunning',
int(ss['network']['serviceExecutorTaskStats']['threadsRunning']))
# memory
for k in ['resident', 'virtual', 'bits']:
self.add_metrics('mongodb.memory.' + k, ss['mem'][k])
#wired tiger
if ss['storageEngine']['name'] == 'wiredTiger':
self.add_metrics(
'mongodb.used-cache',
ss['wiredTiger']['cache']["bytes currently in the cache"])
self.add_metrics(
'mongodb.total-cache',
ss['wiredTiger']['cache']["maximum bytes configured"])
self.add_metrics(
'mongodb.dirty-cache',
ss['wiredTiger']['cache']["tracked dirty bytes in the cache"])
# global lock
for k, v in ss['globalLock']['currentQueue'].items():
self.add_metrics('mongodb.globalLock.currentQueue.' + k, v)
for k, v in ss['globalLock']['activeClients'].items():
self.add_metrics('mongodb.globalLock.activeClients.' + k, v)
def get_db_stats_metrics(self):
"""get DB stats for each DB"""
if self.__conn is None:
self.connect()
if self.__dbnames is None:
self.get_db_names()
if self.__dbnames is not None:
for mongo_db in self.__dbnames:
isAdminDb = mongo_db == 'admin'
metricsList = [
'storageSize', 'ok', 'avgObjSize', 'dataSize', 'indexSize'
]
if isAdminDb:
metricsList = [
'storageSize', 'ok', 'avgObjSize', 'dataSize',
'indexSize', 'fsUsedSize', 'fsTotalSize'
]
db_handler = self.__conn[mongo_db]
dbs = db_handler.command('dbstats')
for k, v in dbs.items():
if k in metricsList:
self.add_metrics(
'mongodb.stats.' + k + '[' + mongo_db + ']',
int(v))
def close(self):
"""close connection to mongo"""
if self.__conn is not None:
self.__conn.close()
#------------PROBLEMS------------#
#The program is only reading in one entry from the json
#------------PROBLEMS------------#
#http://www.vizgr.org/historical-events/search.php?format=json&begin_date=-3000000&end_date=20151231&lang=en
from pymongo import MongoClient
from json import load
connection = MongoClient('homer.stuy.edu')
#if the database hasnt been made, then make it and add it
if not 'stan_smith_xus' in connection.database_names():
print "i tried"
collection = connection.stan_smith_xus #the database
db = collection.events
with open('events.json') as f:
event = load(f)
print event
db.insert(event)
else: #for debugging, the db is removed if you run the program after making a db on the server
print "removing..."
connection.drop_database('stan_smith_xus')
from pymongo import MongoClient, errors
try:
'''Connect with local mongdb'''
# MongoClient = MongoClient('localhost',27017)
'''Connect with mongdb cluster'''
MongoClient = MongoClient(
"mongodb://<Username>:<Password>@cluster0-shard-00-00-fnmen.mongodb.net:27017,cluster0-shard-00-01-fnmen.mongodb.net:27017,cluster0-shard-00-02-fnmen.mongodb.net:27017/test?ssl=true&replicaSet=Cluster0-shard-0&authSource=admin"
)
print("Connect successfully!")
except errors.ConnectionFailure:
print("Could not connect to MongoDB")
print("Show database", MongoClient.database_names())
'''Change database'''
database = MongoClient.test
'''Get collection name'''
collection = database.collection_names(include_system_collections=False)
for collect in collection:
print(collect)
'''Get collection restaurants'''
restaurants = database.restaurants
'''Print one sample'''
print(restaurants.find_one())
'''Print one sample with contrain'''
print(restaurants.find_one({"name": "Wendy'S"}))
update1 = db.rssInfo.update_one(
{'_id': i['_id']}, {'$set': {
rssName: i[rssName]
}})
update2 = db.currentInfo.update_one(
{'_id': current_j['_id']},
{'$set': {
rssName: current_j[rssName]
}})
return
client = MongoClient('localhost', port=27017)
#printing all databases
print(client.database_names())
if ("SourceDB" in client.database_names()):
print("Database present\n")
#connecting to database
db = client.SourceDB
target = db.targetInfo
current = db.currentInfo
cursor_target = target.find()
cursor_current = current.find()
rssList = [
'Food Packets', 'Sanitary Napkins', 'Water Bottles', 'Medkits', 'Blankets',
# DEFINE COLLECTION (TABLE) WHERE YOU'LL INSERT THE TWEETS
tweets = db['tweets']
# CREATE UNIQUE INDEX FOR TABLE (TO AVOID DUPLICATES)
#db.tweets.create_index([('id_str', pymongo.ASCENDING)], unique=True)
#SHOW INDEX ON TWEETS COLLECTION
list(db.tweets.index_information())
#SHOW NUMBER OF TWEETS IN TABLE
tweets.count()
#TO SEE LIST OF CURRENT MONGODB DATABASES
client.database_names()
#TO SEE LIST OF COLLECTIONS IN THE *TWITTER* DATABASE
db.collection_names()
#testing the importing of df
#df = pd.read_csv('sample_draft.csv')
#df = pd.read_csv('sample1_LEFT.csv', header=None) #REPEAT FOR RIGHT
#df = pd.read_csv('sample1_RIGHT.csv', header=None)
#df.rename(columns={0: '_id'}, inplace=True)
#df.columns
#records = json.loads(df.T.to_json()).values()
#records
#len(records)
# coding:utf-8
# mongodb未授权检测脚本
# author：ske
# usage: python3 mongodb_unauth.py ip port
# 默认端口28017和27017
from pymongo import MongoClient
import sys
ip = sys.argv[1]
port = int(sys.argv[2])
try:
conn = MongoClient(ip, port, socketTimeoutMS=5000) # 连接MongoDB,延时5秒
dbs = conn.database_names()
print('[ok] -> {}:{} database_names : {}'.format(ip, port, dbs))
conn.close()
except Exception as e:
error = e.args
print('[-] -> {}:{} error : {}'.format(ip, port, error))
#!/usr/bin/python
import pymongo
import datetime
from pymongo import MongoClient
#connect to mongodb server
client = MongoClient()
print client.database_names()
#get database, db is defined take the test data base from mongodb
#db = client.local
db = client.test
#dbs = db.show()
#print db.name()
#get collection
#collection = db.test
#document post
post = {
"author": "Mike",
"text": "My first blog post!",
"tags": ["mongodb", "python", "pymongo"],
"date": datetime.datetime.utcnow()
}
#new collection called posts
posts = db.posts
post_id = posts.insert(post)
print post_id
from pymongo import MongoClient
import datetime
client = MongoClient('localhost',27017)
# client = MongoClient() <--localhost
print(client.database_names()) #데이터베이스 이름 출력
db = client.lab2
print(db.collection_names()) #데이터베이스 내의 컬랙션 이름 출력
collection = db.apples
post = {"author":"Mike","text":"My first blog post!",
"tags":["MongoDB","Python","PyMongo"],"date":datetime.datetime.utcnow()}
db.lab2.inset_one(post)
class Database(object):
'''Database creation'''
def __init__(self, database_name):
self.client = MongoClient('mongodb://localhost,localhost:27017')
self.db_name = database_name
self.db = self.client[self.db_name]
#self.jobs = self.client[self.db_name].jobs
#self.results = self.db['results']
#self.queue = self.db['queue']
#self.log = self.db['log']
#self.sources = self.db['sources']
#self.jobs = self.db['jobs']
#self.db.x = self.db[x]
# def __repr__(self, database_name):
# print "Using database: %s" %self.client[database_name]
# return self.db
def use_db(self, database_name):
return self.client[str(database_name)]
def use_coll(self, coll_name):
return self.db[coll_name]
def show_dbs(self):
return self.client.database_names()
def create_coll(self, coll_name):
setattr(self, str(coll_name), self.db[str(coll_name)])
#print "coll : %s has been created in db:%s " %(self.__dict__[str(coll_name)], self.db_name)
return self.__dict__[str(coll_name)]
def create_colls(self, coll_names=["results", "sources", "log", "queue"]):
for n in coll_names:
setattr(self, n, self.db[str(n)])
# self.queue = self.db['queue']
# self.log = self.db['log']
# self.sources = self.db['sources']
# #print "Creating coll", [n for n in self.db.collection_names()]
return [n for n in self.db.collection_names()]
def show_coll(self):
try:
print "using collection %s in DB : %s" % (self.coll_name,
self.db_name)
return self.coll_name
except AttributeError:
return False
#return self.db.collection_names()
def show_coll_items(self, coll_name):
return [n for n in self.db[str(coll_name)].find()]
# def count(self, coll_name):
# self.db_coll = self.db[str(coll_name)]
# return self.db_coll.count()
def drop(self, type, name):
if type == "collection":
return self.db[str(name)].drop()
elif type == "database":
return self.client.drop_database(str(name))
else:
print "Unknown Type"
return False
def drop_all_dbs(self):
'''remove EVERY SINGLE MONGO DATABASE'''
for n in self.show_dbs():
#if n not in ["projects", "tasks"]:
self.use_db(n)
self.drop("database", n)
def stats(self):
'''Output the current stats of database in Terminal'''
title = "===STATS===\n"
name = "Stored results in Mongo Database: %s \n" % (self.db_name)
res = "\t-Nombre de resultats dans la base: %d\n" % (
self.db.results.count())
sources = "\t-Nombre de sources: %d\n" % len(
self.db.sources.distinct('url'))
url = "\t-urls en cours de traitement: %d\n" % (self.db.queue.count())
url2 = "\t-urls traitees: %d\n" % (self.db.results.count() +
self.db.log.count())
url3 = "\t-urls erronées: %d\n" % (self.db.log.count())
size = "\t-Size of the database %s: %d MB\n" % (
self.db_name,
(self.db.command('dbStats', 1024)['storageSize']) / 1024 / 1024.)
result = [title, name, res, sources, url, url2, size]
return "".join(result)
def report(self):
''' Output the currents of database for Email Report'''
res = "<li>Nombre de resultats dans la base: %d</li>" % (
self.db.results.count())
sources = "<li>Nombre de sources: %d</li>" % len(
self.db.sources.distinct('url'))
url = "<li>urls en cours de traitement: %d\n</li>" % (
self.db.queue.count())
url2 = "<li>urls traitees: %d</li>" % (self.db.results.count() +
self.db.log.count())
size = "<li>Size of the database %s: %d MB</li>" % (
self.db_name,
(self.db.command('dbStats', 1024)['storageSize']) / 1024 / 1024.)
result = [res, sources, url, url2, size]
return "".join(result)
