def get_connection(username, password, db, host, port):
print("inside connection method")
if username and password:
mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host,
port, db)
connection = mongo_client(mongo_uri)
else:
connection = mongo_client(host, port)
return connection
def setup_verify_mongodbadminpower():
client = mongo_client(host='svc-nosql', serverSelectionTimeoutMS=500)
try:
client.admin.authenticate(
name=request.values.get('UL_DB_ROOT_USER'),
password=request.values.get('UL_DB_ROOT_PASS'))
return Response(json.dumps({
"type":
"success",
"msg":
"rolesInfo command succeed. We assume that user has root priviledge.",
"rolesInfo":
client.admin.command({
'rolesInfo': {
'role': 'root',
'db': 'admin'
},
'showPrivileges': True,
'showBuiltinRoles': True
})
}),
mimetype='application/json'), 200
except OperationFailure:
return Response(json.dumps({
"type":
"success",
"msg":
"rolesInfo command failed. We assume that user doesn't have root privilege."
}),
mimetype='application/json'), 403
def conn():
if Mongo.__conn__ is None:
conn = mongo_client(host=config('UL_DB_HOST'))
conn[config('UL_DB_NAME')].authenticate(
name=config('UL_DB_USER'), password=config('UL_DB_PASS'))
Mongo.__conn__ = conn
return Mongo.__conn__
def connect(self):
global db
try:
client = mongo_client(self.connection_string)
except:
print("Failed to connect to the database")
db = client["models"]
def validate_mongo_host():
client = mongo_client(host=request.values.get('UL_DB_HOST'),
serverSelectionTimeoutMS=5000)
try:
client.server_info()
except ServerSelectionTimeoutError:
raise Exception(
json.dumps({
"type":
"error",
"msg":
"Unable to connect to mongodb host. Are you sure that the hostname is reachable?"
}))
def testPyMongo():
client = mongo_client("localhost", 27017)
db = client.test
#db.user.insert_one({"name":u'牟泽波大魔王'})
q = db.user.find_one_and_update({"userId": 1234},
{"$push": {
"msg": "123"
}})
#q=db.user.find_one_and_update({},{"$set":{"msg":[]}})
print(q)
for r in db.user.find():
print(r)
print(r["name"])
def setup_verify_mongodb():
client = mongo_client(host=request.values.get('UL_DB_HOST'),
serverSelectionTimeoutMS=5000)
try:
return Response(json.dumps({
"type": "success",
"msg": "Connection to mongodb host is successful.",
"server_info": client.server_info()
}),
mimetype='application/json'), 200
except ServerSelectionTimeoutError:
return Response(
'{"type":"error", "msg":"Unable to connect to mongodb host. Are you sure that the hostname is reachable?"}',
mimetype='application/json'), 403
def validate_mongo_admin():
client = mongo_client(host=request.values.get('UL_DB_HOST'),
username=request.values.get('UL_DB_ROOT_USER'),
password=request.values.get('UL_DB_ROOT_PASS'),
serverSelectionTimeoutMS=500)
try:
client.list_database_names()
except OperationFailure:
raise Exception(
json.dumps({
"type":
"error",
"msg":
"MongoDB username and/or password may be invalid. Unable to perform list_database_names operation."
}))
def validate_mongo_power():
client = mongo_client(host='svc-nosql', serverSelectionTimeoutMS=500)
try:
client.admin.authenticate(
name=request.values.get('UL_DB_ROOT_USER'),
password=request.values.get('UL_DB_ROOT_PASS'))
client.admin.command({
'rolesInfo': {
'role': 'root',
'db': 'admin'
},
'showPrivileges': True,
'showBuiltinRoles': True
})
except OperationFailure:
raise Exception(
json.dumps({
"type":
"success",
"msg":
"rolesInfo command failed. We assume that user doesn't have root privilege."
}))
def setup_verify_mongodbadminuser():
client = mongo_client(host=request.values.get('UL_DB_HOST'),
username=request.values.get('UL_DB_ROOT_USER'),
password=request.values.get('UL_DB_ROOT_PASS'),
serverSelectionTimeoutMS=500)
try:
return Response(json.dumps({
"type":
"success",
"msg":
"Test list_database operation is successful.",
"list_databases":
client.list_database_names()
}),
mimetype="application/json"), 200
except OperationFailure:
return Response(json.dumps({
"type":
"error",
"msg":
"MongoDB username and/or password may be invalid. Unable to perform list_database_names operation."
}),
mimetype='application/json'), 403
'''
将图片以二进制格式存入MongoDB中
'''
from pymongo import mongo_client
import bson.binary
conn = mongo_client('localhost',27017)
db = coon.test
myset = db.class0
#读取图片
with open('mongo.jpg',rb) as f :
data = f.read()
#做格式转化
content = bson.binary.Binary(data)
#插入数据库
myset.insert_one('filename':'a1.jpg','data':content)
conn.close()
def connect(self):
try:
self.client = pymongo.mongo_client()
except Exception as e:
def process_item(self, item, spider):
# 创建mongdb数据库储存数据
client = mongo_client()
collection = client['163music']['songs']
collection.insert(item)
return item
def connectMongo(self):
client = mongo_client(self._host, self._port)
self._db = client[self._dbname]
def set_environment_new():
# Check all essential environment variable
if essential_env_present():
# Returns an access denied response
return Response(json.dumps({
"type": "error",
"msg": "Access is denied."
}),
mimetype='application/json'), 403
essential_env = {
"UL_DB_HOST": request.values.get('UL_DB_HOST'),
"UL_DB_ROOT_USER": request.values.get('UL_DB_ROOT_USER'),
"UL_DB_ROOT_PASS": request.values.get('UL_DB_ROOT_PASS'),
"UL_DB_NAME_PREFIX": request.values.get('UL_DB_NAME_PREFIX'),
"UL_TP_CHECK": request.values.get('UL_TP_CHECK'),
"UL_TP_URL": request.values.get('UL_TP_URL'),
"UL_TP_REQUEST_FORMAT": request.values.get('UL_TP_REQUEST_FORMAT')
}
# Perform validation
status = MongoSetupController.validate()
if status.get('type') == 'error':
return Response(json.dumps(status),
mimetype='application/json'), 403
# Sets essential env setting
for key in essential_env:
if essential_env[key] is None:
config(key, '')
else:
config(key, essential_env[key])
# Create a client instance (mongodb)
client = mongo_client(host=config('UL_DB_HOST'),
username=config('UL_DB_ROOT_USER'),
password=config('UL_DB_ROOT_PASS'))
# Assigns database name from prefix
config('UL_DB_NAME', config('UL_DB_NAME_PREFIX') + 'ul_db')
# Creates an initial user: admin:admin
# Note that this will have issue if we have third party authentication server.
# I will make a separate setup for this, but for now, the implementation would be
# just this simple. I will not handle third party auth for now...
client[config('UL_DB_NAME')].users.insert_one({
"username":
"******",
"password":
bcrypt.hashpw(b'admin', bcrypt.gensalt())
})
# Assign a random username and password for universal login user account
config('UL_DB_USER', 'ul_user_' + random_str(5))
config('UL_DB_PASS', random_str(16))
# Creates a user dedicated for universal login only
client[config('UL_DB_NAME')].command("createUser",
config('UL_DB_USER'),
pwd=config('UL_DB_PASS'),
roles=["readWrite"])
# Generate a symmetric key
# Note: I wish to use asymmetric key, but I have issues with EC or RSA.
# Errors: TypeError: object of type '_RSAPrivateKey' has no len() or
# TypeError: object of type '_EllipticCurvePrivateKey' has no len()
config('UL_KEY', jwk.JWK.generate(kty='oct', size=256).export())
return Response(json.dumps({
"type":
"success",
"msg":
"Environment has been successfuly set."
}),
mimetype='application/json'), 200
"""
This is my personal Jupyter Notebook I used for testing and curating my MongoDB database of lyrics
Cells are separated by #%% Character, I used this notebook in VSCode's Jupyter notebook extension
"""
#%%
import config
import json
import functools
import pymongo
from bson.objectid import ObjectId
from pprint import pprint as pp
import analyzeSong
mongoconnect = config.get("CLIENT", "MONGODB")
cluster = pymongo.mongo_client(f"{mongoconnect}")
db = cluster["Lyrics_Actual"]
colnames = db.list_collection_names()
#%%
print(colnames)
print(db[col].find())
for col in colnames:
collection = db[col]
docscursor = collection.find()
docs = [doc for doc in docscursor]
keys = [(doc.keys(), idx) for idx, doc in enumerate(docs)
if "song" not in doc.keys()]
print(keys)
# check if artist has proper formatting, if first entry isnt formatted, whole artist wont be
from multiprocessing import pool
import requests
from lxml import etree
import pymongo
from multiprocessing import pool
client=pymongo.mongo_client('localhost',27017)
mydb=client['mydb']
douban=mydb['douban']
def get_douban(url):
html=requests.get(url)
selector=etree.HTML(html.text)
infos=selector.xpath('//div[@class="indent"]/tr')
for info in infos:
try:
top=info.xpath('div/tr/td[2]/a/@title')
except IndexError
pass
if__name__=='__main__':
urls=['https://book.douban.com/top250?start={}'.format(str(i) for i in) range(0,250,25)]
pool=pool(processes=4)
pool.map(get_douban,urls)
def set_environment():
"""
Deprecated
:return:
"""
# Check all essential environment variable
if essential_env_present():
# Redirects to /login page
return redirect('/login')
essential_env = {
"UL_DB_HOST": request.values.get('UL_DB_HOST'),
"UL_DB_ROOT_USER": request.values.get('UL_DB_ROOT_USER'),
"UL_DB_ROOT_PASS": request.values.get('UL_DB_ROOT_PASS'),
"UL_DB_NAME_PREFIX": request.values.get('UL_DB_NAME_PREFIX'),
"UL_TP_CHECK": request.values.get('UL_TP_CHECK'),
"UL_TP_URL": request.values.get('UL_TP_URL'),
"UL_TP_REQUEST_FORMAT": request.values.get('UL_TP_REQUEST_FORMAT')
}
# Sets essential env setting
for key in essential_env:
if essential_env[key] is None:
config(key, '')
else:
config(key, essential_env[key])
# Create a client instance (mongodb)
client = mongo_client(host=config('UL_DB_HOST'),
username=config('UL_DB_ROOT_USER'),
password=config('UL_DB_ROOT_PASS'))
# Assigns database name from prefix
config('UL_DB_NAME', config('UL_DB_NAME_PREFIX') + 'ul_db')
# Creates an initial user: admin:admin
# Note that this will have issue if we have third party authentication server.
# I will make a separate setup for this, but for now, the implementation would be
# just this simple. I will not handle third party auth for now...
client[config('UL_DB_NAME')].users.insert_one({
"username":
"******",
"password":
bcrypt.hashpw(b'admin', bcrypt.gensalt())
})
# Assign a random username and password for universal login user account
config('UL_DB_USER', 'ul_user_' + random_str(5))
config('UL_DB_PASS', random_str(16))
# Creates a user dedicated for universal login only
client[config('UL_DB_NAME')].command("createUser",
config('UL_DB_USER'),
pwd=config('UL_DB_PASS'),
roles=["readWrite"])
# Generate a symmetric key
# Note: I wish to use asymmetric key, but I have issues with EC or RSA.
# Errors: TypeError: object of type '_RSAPrivateKey' has no len() or
# TypeError: object of type '_EllipticCurvePrivateKey' has no len()
config('UL_KEY', jwk.JWK.generate(kty='oct', size=256).export())
# Set environment from submitted form
with open('/html/setup-done.html', 'r') as html_file:
# index.html file
html = html_file.read()
# Check all essential environment variable
if essential_env_present():
# Perform re-setup.
return Response(html), 200
# Perform URL redirection
return redirect('/')
