def get_connection(username, password, db, host, port): print("inside connection method") if username and password: mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db) connection = mongo_client(mongo_uri) else: connection = mongo_client(host, port) return connection
def setup_verify_mongodbadminpower(): client = mongo_client(host='svc-nosql', serverSelectionTimeoutMS=500) try: client.admin.authenticate( name=request.values.get('UL_DB_ROOT_USER'), password=request.values.get('UL_DB_ROOT_PASS')) return Response(json.dumps({ "type": "success", "msg": "rolesInfo command succeed. We assume that user has root priviledge.", "rolesInfo": client.admin.command({ 'rolesInfo': { 'role': 'root', 'db': 'admin' }, 'showPrivileges': True, 'showBuiltinRoles': True }) }), mimetype='application/json'), 200 except OperationFailure: return Response(json.dumps({ "type": "success", "msg": "rolesInfo command failed. We assume that user doesn't have root privilege." }), mimetype='application/json'), 403
def conn(): if Mongo.__conn__ is None: conn = mongo_client(host=config('UL_DB_HOST')) conn[config('UL_DB_NAME')].authenticate( name=config('UL_DB_USER'), password=config('UL_DB_PASS')) Mongo.__conn__ = conn return Mongo.__conn__
def connect(self): global db try: client = mongo_client(self.connection_string) except: print("Failed to connect to the database") db = client["models"]
def validate_mongo_host(): client = mongo_client(host=request.values.get('UL_DB_HOST'), serverSelectionTimeoutMS=5000) try: client.server_info() except ServerSelectionTimeoutError: raise Exception( json.dumps({ "type": "error", "msg": "Unable to connect to mongodb host. Are you sure that the hostname is reachable?" }))
def testPyMongo(): client = mongo_client("localhost", 27017) db = client.test #db.user.insert_one({"name":u'牟泽波大魔王'}) q = db.user.find_one_and_update({"userId": 1234}, {"$push": { "msg": "123" }}) #q=db.user.find_one_and_update({},{"$set":{"msg":[]}}) print(q) for r in db.user.find(): print(r) print(r["name"])
def setup_verify_mongodb(): client = mongo_client(host=request.values.get('UL_DB_HOST'), serverSelectionTimeoutMS=5000) try: return Response(json.dumps({ "type": "success", "msg": "Connection to mongodb host is successful.", "server_info": client.server_info() }), mimetype='application/json'), 200 except ServerSelectionTimeoutError: return Response( '{"type":"error", "msg":"Unable to connect to mongodb host. Are you sure that the hostname is reachable?"}', mimetype='application/json'), 403
def validate_mongo_admin(): client = mongo_client(host=request.values.get('UL_DB_HOST'), username=request.values.get('UL_DB_ROOT_USER'), password=request.values.get('UL_DB_ROOT_PASS'), serverSelectionTimeoutMS=500) try: client.list_database_names() except OperationFailure: raise Exception( json.dumps({ "type": "error", "msg": "MongoDB username and/or password may be invalid. Unable to perform list_database_names operation." }))
def validate_mongo_power(): client = mongo_client(host='svc-nosql', serverSelectionTimeoutMS=500) try: client.admin.authenticate( name=request.values.get('UL_DB_ROOT_USER'), password=request.values.get('UL_DB_ROOT_PASS')) client.admin.command({ 'rolesInfo': { 'role': 'root', 'db': 'admin' }, 'showPrivileges': True, 'showBuiltinRoles': True }) except OperationFailure: raise Exception( json.dumps({ "type": "success", "msg": "rolesInfo command failed. We assume that user doesn't have root privilege." }))
def setup_verify_mongodbadminuser(): client = mongo_client(host=request.values.get('UL_DB_HOST'), username=request.values.get('UL_DB_ROOT_USER'), password=request.values.get('UL_DB_ROOT_PASS'), serverSelectionTimeoutMS=500) try: return Response(json.dumps({ "type": "success", "msg": "Test list_database operation is successful.", "list_databases": client.list_database_names() }), mimetype="application/json"), 200 except OperationFailure: return Response(json.dumps({ "type": "error", "msg": "MongoDB username and/or password may be invalid. Unable to perform list_database_names operation." }), mimetype='application/json'), 403
''' 将图片以二进制格式存入MongoDB中 ''' from pymongo import mongo_client import bson.binary conn = mongo_client('localhost',27017) db = coon.test myset = db.class0 #读取图片 with open('mongo.jpg',rb) as f : data = f.read() #做格式转化 content = bson.binary.Binary(data) #插入数据库 myset.insert_one('filename':'a1.jpg','data':content) conn.close()
def connect(self): try: self.client = pymongo.mongo_client() except Exception as e:
def process_item(self, item, spider): # 创建mongdb数据库储存数据 client = mongo_client() collection = client['163music']['songs'] collection.insert(item) return item
def connectMongo(self): client = mongo_client(self._host, self._port) self._db = client[self._dbname]
def set_environment_new(): # Check all essential environment variable if essential_env_present(): # Returns an access denied response return Response(json.dumps({ "type": "error", "msg": "Access is denied." }), mimetype='application/json'), 403 essential_env = { "UL_DB_HOST": request.values.get('UL_DB_HOST'), "UL_DB_ROOT_USER": request.values.get('UL_DB_ROOT_USER'), "UL_DB_ROOT_PASS": request.values.get('UL_DB_ROOT_PASS'), "UL_DB_NAME_PREFIX": request.values.get('UL_DB_NAME_PREFIX'), "UL_TP_CHECK": request.values.get('UL_TP_CHECK'), "UL_TP_URL": request.values.get('UL_TP_URL'), "UL_TP_REQUEST_FORMAT": request.values.get('UL_TP_REQUEST_FORMAT') } # Perform validation status = MongoSetupController.validate() if status.get('type') == 'error': return Response(json.dumps(status), mimetype='application/json'), 403 # Sets essential env setting for key in essential_env: if essential_env[key] is None: config(key, '') else: config(key, essential_env[key]) # Create a client instance (mongodb) client = mongo_client(host=config('UL_DB_HOST'), username=config('UL_DB_ROOT_USER'), password=config('UL_DB_ROOT_PASS')) # Assigns database name from prefix config('UL_DB_NAME', config('UL_DB_NAME_PREFIX') + 'ul_db') # Creates an initial user: admin:admin # Note that this will have issue if we have third party authentication server. # I will make a separate setup for this, but for now, the implementation would be # just this simple. I will not handle third party auth for now... client[config('UL_DB_NAME')].users.insert_one({ "username": "******", "password": bcrypt.hashpw(b'admin', bcrypt.gensalt()) }) # Assign a random username and password for universal login user account config('UL_DB_USER', 'ul_user_' + random_str(5)) config('UL_DB_PASS', random_str(16)) # Creates a user dedicated for universal login only client[config('UL_DB_NAME')].command("createUser", config('UL_DB_USER'), pwd=config('UL_DB_PASS'), roles=["readWrite"]) # Generate a symmetric key # Note: I wish to use asymmetric key, but I have issues with EC or RSA. # Errors: TypeError: object of type '_RSAPrivateKey' has no len() or # TypeError: object of type '_EllipticCurvePrivateKey' has no len() config('UL_KEY', jwk.JWK.generate(kty='oct', size=256).export()) return Response(json.dumps({ "type": "success", "msg": "Environment has been successfuly set." }), mimetype='application/json'), 200
""" This is my personal Jupyter Notebook I used for testing and curating my MongoDB database of lyrics Cells are separated by #%% Character, I used this notebook in VSCode's Jupyter notebook extension """ #%% import config import json import functools import pymongo from bson.objectid import ObjectId from pprint import pprint as pp import analyzeSong mongoconnect = config.get("CLIENT", "MONGODB") cluster = pymongo.mongo_client(f"{mongoconnect}") db = cluster["Lyrics_Actual"] colnames = db.list_collection_names() #%% print(colnames) print(db[col].find()) for col in colnames: collection = db[col] docscursor = collection.find() docs = [doc for doc in docscursor] keys = [(doc.keys(), idx) for idx, doc in enumerate(docs) if "song" not in doc.keys()] print(keys) # check if artist has proper formatting, if first entry isnt formatted, whole artist wont be
from multiprocessing import pool import requests from lxml import etree import pymongo from multiprocessing import pool client=pymongo.mongo_client('localhost',27017) mydb=client['mydb'] douban=mydb['douban'] def get_douban(url): html=requests.get(url) selector=etree.HTML(html.text) infos=selector.xpath('//div[@class="indent"]/tr') for info in infos: try: top=info.xpath('div/tr/td[2]/a/@title') except IndexError pass if__name__=='__main__': urls=['https://book.douban.com/top250?start={}'.format(str(i) for i in) range(0,250,25)] pool=pool(processes=4) pool.map(get_douban,urls)
def set_environment(): """ Deprecated :return: """ # Check all essential environment variable if essential_env_present(): # Redirects to /login page return redirect('/login') essential_env = { "UL_DB_HOST": request.values.get('UL_DB_HOST'), "UL_DB_ROOT_USER": request.values.get('UL_DB_ROOT_USER'), "UL_DB_ROOT_PASS": request.values.get('UL_DB_ROOT_PASS'), "UL_DB_NAME_PREFIX": request.values.get('UL_DB_NAME_PREFIX'), "UL_TP_CHECK": request.values.get('UL_TP_CHECK'), "UL_TP_URL": request.values.get('UL_TP_URL'), "UL_TP_REQUEST_FORMAT": request.values.get('UL_TP_REQUEST_FORMAT') } # Sets essential env setting for key in essential_env: if essential_env[key] is None: config(key, '') else: config(key, essential_env[key]) # Create a client instance (mongodb) client = mongo_client(host=config('UL_DB_HOST'), username=config('UL_DB_ROOT_USER'), password=config('UL_DB_ROOT_PASS')) # Assigns database name from prefix config('UL_DB_NAME', config('UL_DB_NAME_PREFIX') + 'ul_db') # Creates an initial user: admin:admin # Note that this will have issue if we have third party authentication server. # I will make a separate setup for this, but for now, the implementation would be # just this simple. I will not handle third party auth for now... client[config('UL_DB_NAME')].users.insert_one({ "username": "******", "password": bcrypt.hashpw(b'admin', bcrypt.gensalt()) }) # Assign a random username and password for universal login user account config('UL_DB_USER', 'ul_user_' + random_str(5)) config('UL_DB_PASS', random_str(16)) # Creates a user dedicated for universal login only client[config('UL_DB_NAME')].command("createUser", config('UL_DB_USER'), pwd=config('UL_DB_PASS'), roles=["readWrite"]) # Generate a symmetric key # Note: I wish to use asymmetric key, but I have issues with EC or RSA. # Errors: TypeError: object of type '_RSAPrivateKey' has no len() or # TypeError: object of type '_EllipticCurvePrivateKey' has no len() config('UL_KEY', jwk.JWK.generate(kty='oct', size=256).export()) # Set environment from submitted form with open('/html/setup-done.html', 'r') as html_file: # index.html file html = html_file.read() # Check all essential environment variable if essential_env_present(): # Perform re-setup. return Response(html), 200 # Perform URL redirection return redirect('/')