def get_database(client: MongoClient, db_name: str) -> Database:
"""
Use MongoClient and db_name to return a db handle.
Raises pymongo.errors.InvalidName if an invalid database name is used.
"""
db: Database = client.__getattr__(db_name)
return db
def getDatabase():
global client
global database
if client is None:
logger.info('Initializing mongo db connection: %s, %s' % (Configuration.MONGO_DB_IP, Configuration.MONGO_DB_PORT))
# Disabling write concern might be a good idea, but could cause problems.. need to experiment.
if Configuration.MONGO_WRITE_CONCERN_ENABLED:
writeConcern = 1
else:
writeConcern = 0
client = MongoClient(Configuration.MONGO_DB_IP, Configuration.MONGO_DB_PORT, w=writeConcern, socketTimeoutMS=Configuration.MONGO_OPERATION_TIMEOUT, connectTimeoutMS=Configuration.MONGO_CONNECTION_TIMEOUT, use_greenlets=True)
if database is None:
database = client.__getattr__(Configuration.MONGO_DB_DATABASE_NAME)
if Configuration.MONGO_DB_DATABASE_AUTHENTICATION_ENABLED:
username = Configuration.MONGO_DB_DATABASE_AUTHENTICATION_USER_NAME
password = Configuration.MONGO_DB_DATABASE_AUTHENTICATION_PASSWORD
database.authenticate(username, password)
if Configuration.ENABLE_MONGO_PROFILING is True:
database.set_profiling_level(pymongo.OFF)
logger.info('Erasing old MongoDB profiling data..')
getDatabase().system.profile.drop()
logger.info('Enabling MongoDB profiling..')
database.set_profiling_level(Configuration.MONGO_PROFILING_LEVEL)
return database
def show_connections(Host, Port, db_name):
conn = MongoClient(host=Host, port=Port)
#my_db = conn.db_name
my_db = conn.__getattr__(db_name)
#print my_db
colls = my_db.collection_names()
print colls
def getDatabase():
global client
global database
if client is None:
logger.info('Initializing mongo db connection: %s, %s' % (Configuration.MONGO_DB_IP, Configuration.MONGO_DB_PORT))
# Disabling write concern might be a good idea, but could cause problems.. need to experiment.
if Configuration.MONGO_WRITE_CONCERN_ENABLED:
writeConcern = 1
else:
writeConcern = 0
client = MongoClient(Configuration.MONGO_DB_IP, Configuration.MONGO_DB_PORT, w=writeConcern, socketTimeoutMS=Configuration.MONGO_OPERATION_TIMEOUT, connectTimeoutMS=Configuration.MONGO_CONNECTION_TIMEOUT)
if database is None:
database = client.__getattr__(Configuration.MONGO_DB_DATABASE_NAME)
if Configuration.MONGO_DB_DATABASE_AUTHENTICATION_ENABLED:
username = Configuration.MONGO_DB_DATABASE_AUTHENTICATION_USER_NAME
password = Configuration.MONGO_DB_DATABASE_AUTHENTICATION_PASSWORD
database.authenticate(username, password)
if Configuration.ENABLE_MONGO_PROFILING is True:
database.set_profiling_level(pymongo.OFF)
logger.info('Erasing old MongoDB profiling data..')
getDatabase().system.profile.drop()
logger.info('Enabling MongoDB profiling..')
database.set_profiling_level(Configuration.MONGO_PROFILING_LEVEL)
return database
def show_connections(Host, Port, db_name, user, pwd):
conn = MongoClient(host=Host, port=Port)
#my_db = conn.db_name
my_db = conn.__getattr__(db_name)
my_db.authenticate(user, pwd)
#print my_db
colls = my_db.collection_names()
print colls
return colls
def connect(dbname='twitter'):
"""
Connect to a local mongo database for tweets.
"""
from pymongo import MongoClient
client = MongoClient()
db = client.__getattr__(self, dbname)
return db
def __init__(self):
self._db_clients = {}
db_configs = None
with open("config/database.json") as fp:
db_configs = json.load(fp)
for name, config in db_configs.items():
connection = MongoClient(config["address"], config["port"])
db = connection.__getattr__(config["database"])
collection = db.__getattr__(config["collection"])
self._db_clients[name] = collection
class detailInfo:
filename = 'detailInfo.csv'
path = '.\\DataCsv\\'
detailInfo = 'detailInfo'
colname = ['SourseFrom', 'SenstiKind', 'Class', 'time']
res = []
max_len = 100000
i = 1
def __init__(self):
self.files = []
self.files.append(self.filename)
self.client = MongoClient(DBCONFIG['HOST'], DBCONFIG['PORT'])
self.db = self.client.__getattr__(DBCONFIG['NAME'])
self.dst_detailInfo = self.db[self.detailInfo]
def __del__(self):
self.client.close()
@staticmethod
def convert_fileaffix(fileaffix):
if len(fileaffix) == 0:
return ''
while fileaffix[0] == '.':
fileaffix = fileaffix[1:]
if '.' in fileaffix:
return None
else:
return fileaffix
def detailInfodata(self):
self.dst_detailInfo.remove({})
self.res.clear()
print('正在处理文件：%s' % (self.path + self.filename))
with open(self.path + self.filename, 'r', encoding='UTF-8') as infile:
reader = csv.reader(infile)
next(reader)
for row in reader:
self.res.append({
'SourseFrom': str(row[0]),
'SenstiKind': str(row[1]),
'Class': str(row[2]),
'time': str(row[3]),
'title': str(row[4]),
'keys': "0"
})
if self.res.__len__() == self.max_len:
self.dst_detailInfo.insert_many(self.res)
self.res.clear()
if self.res.__len__() > 0:
self.dst_detailInfo.insert_many(self.res)
self.res.clear()
def writetoMongo(self,filepath,mongodbName,mongodbCollectionName):
import glob
import json
from DM import Logging
# create mongodb database
from pymongo import MongoClient
client = MongoClient()
db = client.__getattr__(mongodbName).__getattr__(mongodbCollectionName)
# and inside that DB, a collection called "files"
filenames = glob.glob(filepath)
for filename in filenames:
with open(filename) as f:
for line in f:
db.insert_one(json.loads(line))
Logging.log("Filename " + filename + " was added to mongodb collection name is files")
def writetoMongo(self, filepath, mongodbName, mongodbCollectionName):
import glob
import json
from DM import Logging
# create mongodb database
from pymongo import MongoClient
client = MongoClient()
db = client.__getattr__(mongodbName).__getattr__(mongodbCollectionName)
# and inside that DB, a collection called "files"
filenames = glob.glob(filepath)
for filename in filenames:
with open(filename) as f:
for line in f:
db.insert_one(json.loads(line))
Logging.log("Filename " + filename +
" was added to mongodb collection name is files")
class Mongo(object):
def __init__(self, id):
self.profileid = id
self.client = MongoClient('mongodb://qualdb01.nowtv.dev:27017',
serverSelectionTimeoutMS=1500)
self.client.the_database.authenticate('popcorn',
'kernel',
'customer',
mechanism='MONGODB-CR')
def get_accounts(self):
return self.get_details_by(key='_id',
database='customer',
collection='accounts')
def get_entitlements(self):
return self.get_details_by(key='accountId',
database='customer_passes',
collection='entitlements')
def get_atv_subscriptions(self):
return self.get_details_by(key='accountId',
database='customer',
collection='atv_subscriptions')
def get_vodafone_accounts(self):
return self.get_details_by(key='accountId',
database='customer',
collection='vodafone_accounts')
def get_details_by(self, key='accountId', database='', collection=''):
db_collection = self.client.__getattr__(database).__getattr__(
collection)
details = f"\ndb.getCollection('{collection}').find({{'{key}' : '{self.profileid}'}})\n"
for detail in db_collection.find({key: self.profileid}):
details += pformat(detail)
details += '\n'
return details
def _fnoOpenDb(self, mysDbName):
client = MongoClient(self.sMongoSystem, self.nMongoPort)
db = client.__getattr__(mysDbName)
NTRC.ntracef(3, "SRDM", "Connected to db.")
return db
def _connect_db(self):
connection = MongoClient(self.db_config["address"], self.db_config["port"])
if connection:
log.info("Connected Database on " + self.db_config["address"] + ":" + str(self.db_config["port"]))
self._database = connection.__getattr__(self.db_config["name"])
class MongoManager(object):
"""
mongodb操作封装
"""
def __init__(self, host='localhost', port=27017, db_name=None, collection_name=None):
"""
初始化连接 + 认证
:param host: ip
:param port: 端口
:param db_name: 数据库名称 可选
:param collection_name: 集合名称 可选 需要先有数据库名称后 此数据才可使用
:return:
"""
self.host = host
self.port = port
self.client = MongoClient(host, 27017, username='******', password='******',
authMechanism='SCRAM-SHA-1', authSource='admin')
if db_name:
self.db = self.client.__getattr__(db_name)
else:
self.db = None
if self.db and collection_name:
self.collection = self.db.__getattr__(collection_name)
else:
self.collection = None
def close(self):
self.client.close()
def __enter__(self):
return self
def __exit__(self, type_, value, traceback):
self.close()
def select_db(self, db_name):
"""
选择数据库
:param db_name: 数据库名称
:return:
"""
self.db = self.client.__getattr__(db_name)
def select_collection(self, collection_name):
"""
选择集合
:param collection_name: 集合名称
:return:
"""
if self.db:
self.collection = self.db.__getattr__(collection_name)
else:
raise Exception('db not connection')
def insert_document(self, data):
"""
插入数据 多个和单个
:param data:
:return:
"""
if self.collection:
if isinstance(data, list):
self.collection.insert_many(data)
elif isinstance(data, dict):
self.collection.insert_one(data)
else:
raise Exception('data type error')
else:
raise Exception('collection not connection')
def drop_collection(self, collection_name):
"""
删除集合
:param collection_name 集合名称
:return:
"""
if self.db:
self.db.get_collection(collection_name).drop()
else:
raise Exception('db not connection')
def drop_db(self, db_name):
self.client.drop_database(db_name)
def find_document_many(self, filter_=None, show=None, collection_name=None,
db_name=None, return_type=1):
"""
查询数据
如果有集合连接 直接查询
如果有数据库连接和集合名 可以查询
如果有数据库名称和集合名称 可以查询
分页 数量少的时候可以使用 limit skip
数量多的时候建议使用 搜索_id 并且取比次_id大的指定数量 在创建集合时默认会把_id创建唯一索引
db.node.find({'_id': {'$gt': ObjectId('5ed85a242ae5cf13b63b91aa')}}).limit(10)
此处未使用分页
:param filter_: 搜寻条件 {'_id': '123'}
:param show: 显示字段 {'_id': 0, 'data': 1}
:param collection_name: 集合名称 可选 在没有连接节点时需要传
:param db_name: 数据库名称 可选 在没有连接数据库时需要传
:param return_type: 返回形式 生成器还是列表 默认1 列表 0生成器
:return:
"""
if not filter_:
filter_ = {}
if not show:
show = {}
if self.collection:
collection = self.collection
elif self.db and collection_name:
collection = self.select_collection(collection_name)
else:
if db_name and collection_name:
collection = self.select_db(db_name).select_collection(collection_name)
else:
raise Exception('db not collection')
data = collection.find(filter_, show)
if return_type == 1:
return [res for res in data]
else:
return data
def _fnoOpenDb(self, mysDbName):
client = MongoClient(self.sMongoSystem, self.nMongoPort)
db = client.__getattr__(mysDbName)
NTRC.ntracef(3, "SRDM", "Connected to db.")
return db
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymongo import MongoClient
import pymongo
client = MongoClient("localhost", 27017)
test = client.__getattr__("test")
