def resources_query(query_body):
try:
conf = yield mongodb_conf()
client = MotorClient('mongodb://%s:%s/ceilometer' % conf)
db = client.get_default_database()
cursor = db.resource.find(query_body).hint([('_id', 1)])
samples = yield cursor.to_list(None)
except Exception, e:
LOG.error("ceilometer - query resource error: %s" % e)
raise OpenStackException(e.message)
def samples_query(query_body, limit):
try:
conf = yield mongodb_conf()
client = MotorClient('mongodb://%s:%s/ceilometer' % conf)
db = client.get_default_database()
cursor = db.meter.find(query_body).hint([('timestamp', -1)]).limit(limit)
samples = yield cursor.to_list(None)
except Exception, e:
LOG.error("ceilometer - query samples error: %s" % e)
raise OpenStackException(e.message)
def start(debug: bool = False) -> RestServer:
"""Start a Mad Dash REST service."""
config = from_environment(EXPECTED_CONFIG)
for name in config:
logging.info(f"{name} = {config[name]}")
args = RestHandlerSetup(
{
"auth": {
"secret": config["MAD_DASH_AUTH_SECRET"],
"issuer": config["MAD_DASH_AUTH_ISSUER"],
"algorithm": config["MAD_DASH_AUTH_ALGORITHM"],
},
"debug": debug,
}
)
# configure access to MongoDB as a backing store
mongo_user = quote_plus(config["MAD_DASH_MONGODB_AUTH_USER"])
mongo_pass = quote_plus(config["MAD_DASH_MONGODB_AUTH_PASS"])
mongo_host = config["MAD_DASH_MONGODB_HOST"]
mongo_port = int(config["MAD_DASH_MONGODB_PORT"])
mongodb_url = f"mongodb://{mongo_host}:{mongo_port}"
if mongo_user and mongo_pass:
mongodb_url = f"mongodb://{mongo_user}:{mongo_pass}@{mongo_host}:{mongo_port}"
# ensure indexes
md_mc = MadDashMotorClient(MotorClient(mongodb_url))
asyncio.get_event_loop().run_until_complete(md_mc.ensure_all_databases_indexes())
args["motor_client"] = MotorClient(mongodb_url)
# configure REST routes
server = RestServer(debug=debug)
server.add_route(r"/$", MainHandler, args)
server.add_route(
r"/databases/names$", DatabasesNamesHandler, args
) # get database names
server.add_route(
r"/collections/names$", CollectionsNamesHandler, args
) # get collection names
server.add_route(
r"/collections/histograms/names$", CollectionsHistogramsNamesHandler, args
) # get all histogram names in collection
server.add_route(
r"/collections/histograms$", CollectionsHistogramsHandler, args
) # get all histogram objects in collection
server.add_route(r"/histogram$", HistogramHandler, args) # get histogram object
server.add_route(r"/files/names$", FileNamesHandler, args) # get file names
server.startup(
address=config["MAD_DASH_REST_HOST"], port=int(config["MAD_DASH_REST_PORT"])
)
return server
class RequestRepository:
def __init__(self):
db_url = os.getenv("MONGODB_URL")
db_name = os.getenv("MONGODB_DB")
self.collection = MotorClient(db_url)[db_name].pull_requests
async def get_request(self, request_id):
pr = await self.collection.find_one({'_id': request_id})
return pr
async def get_all_pull_requests_per_status(self, status):
cursor = self.collection.find({'status': status})
result = []
while (await cursor.fetch_next):
doc = cursor.next_object()
result.append(doc)
return result
async def get_all_pull_requests_by_user_id(self, user_id):
cursor = self.collection.find({'user_id': user_id})
result = []
while (await cursor.fetch_next):
doc = cursor.next_object()
result.append(doc)
return result
async def create_request(self, request):
try:
app_log.debug('request : {}'.format(request))
await self.collection.insert_one(request)
app_log.debug('requesteee : {}'.format(request))
except Exception as e:
app_log.warn(
'Cannot save pull request {}'.format(request))
async def update_request(self, request):
try:
await self.collection.update_one({'_id': request['_id']}, {"$set": request}, upsert=False)
except Exception as e:
app_log.warn(
'Cannot update pull request {}'.format(request))
async def create_many_requests(self, psr):
try:
await self.collection.insert_many(psr)
except Exception as e:
app_log.warn(
'Cannot save many pull requests {}'.format(psr))
def make_app():
"""
Main entry point for app
Create a MotorClient and assign it to the settings for the Application.
From the Motor Documentation:
Warning It is a common mistake to create a new client object for every request;
this comes at a dire performance cost. Create the client when your application
starts and reuse that one client for the lifetime of the process,
as shown in these examples.
"""
server = config.dbServer
user = urllib.parse.quote_plus(config.dbUser)
password = urllib.parse.quote_plus(config.dbPassword)
dbUri = 'mongodb://{0}:{1}@{2}'.format(user, password, server)
db = MotorClient(dbUri)[config.database_name]
return tornado.web.Application(
[
(r"/api/auth", AuthHandler),
(r"/api/event/(.+)", EventHandler),
# (r"/api/event/(?P<id>[^\/]+])?/", EventHandler),
(r"/api/vote", VoteHandler),
(r"/api/ws/votes", VoteSocketHandler),
(r"/api/events", EventsHandler)
],
db=db,
debug=True)
def configure(settings):
if "KUBE_API_TOKEN_PATH" in os.environ and os.path.exists(
os.environ["KUBE_API_TOKEN_PATH"]):
logging.info("Reading token from '%s'.",
os.environ["KUBE_API_TOKEN_PATH"])
with open(os.environ["KUBE_API_TOKEN_PATH"]) as token:
settings["kube"] = KubeClient(
os.environ["KUBERNETES_SERVICE_HOST"], token=token.read())
else:
settings["kube"] = KubeClient(os.getenv('KUBERNETES_SERVICE_HOST'))
if "HEAPSTER_SERVICE_HOST" in os.environ:
heapster_endpoint = os.getenv("HEAPSTER_SERVICE_HOST")
heapster_port = os.getenv("HEAPSTER_SERVICE_PORT")
endpoint = "http://%s:%s/api/v1/model" % (heapster_endpoint,
heapster_port)
settings["heapster"] = HeapsterClient(endpoint)
else:
endpoint = "%s%s" % (settings['kube'].http_client.get_base_url(
), "/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model"
)
settings["heapster"] = HeapsterClient(endpoint)
mongo_url = "mongodb://{0}:{1}/".format(
os.getenv("ELASTICKUBE_MONGO_SERVICE_HOST", "localhost"),
os.getenv("ELASTICKUBE_MONGO_SERVICE_PORT", 27017))
settings["motor"] = MotorClient(mongo_url)
elastickube_db = settings["motor"].elastickube
elastickube_db.add_son_manipulator(KeyManipulator())
settings["database"] = elastickube_db
def main():
app = Application([
(r'/eval/(?P<tid>[\w/]+)', EvalHandler),
(r'/eval/(?P<tid>[\w/]+)/submit', EvalHandler),
(r'/eval/(?P<tid>[\w/]+)/config.js', ConfigHandler),
(r"/img/(.*)", StaticFileHandler, {
"path": "./img"
}),
(r"/css/(.*)", StaticFileHandler, {
"path": "./css"
}),
(r"/js/(.*)", StaticFileHandler, {
"path": "./js"
}),
],
template_path='static')
server = HTTPServer(app)
server.bind(options.port)
server.start(1)
app.settings['db'] = MotorClient(options.mongodb_url)[options.mongodb_db]
try:
IOLoop.current().asyncio_loop.run_until_complete(app.create_redis())
except Exception as e:
logger.error(f'Cannot connect to redis {e}')
sys.exit(-1)
IOLoop.current().start()
async def change_owner(self, **params):
if params.get("message"):
params = json.loads(params.get("message", "{}"))
if not params:
return {"error": 400, "reason": "Missed required fields"}
coinid = params.get("coinid")
cid = params.get("cid")
public_key = params.get("public_key")
client = MotorClient()
database = client[coinid]
content_collection = database[settings.CONTENT]
content = await content_collection.find_one_and_update(
{"cid": int(cid)}, {"$set": {
"owner": public_key
}})
if not content:
return {
"error": 404,
"reason": "Change owner. Content with cid %s not found" % cid
}
return {i: content[i] for i in content if i != "_id"}
def get_motor(self, name=None):
''' motor client '''
name = name or __conf__.DB_NAME
if not self._motor_clt:
self._motor_clt = MotorClient(host=__conf__.DB_HOST)
return self._motor_clt[name]
async def get_deals(self, **params):
"""Receives all users deals
Accepts:
- buyer public key
"""
if params.get("message"):
params = json.loads(params.get("message", "{}"))
if not params:
return {"error": 400, "reason": "Missed required fields"}
buyer = params.get("buyer")
if not buyer:
return {"error": 400, "reason": "Missed public key"}
deals = {i: [] for i in settings.AVAILABLE_COIN_ID}
client = MotorClient()
for coinid in settings.AVAILABLE_COIN_ID:
database = client[coinid]
collection = database[settings.DEAL]
async for document in collection.find({"owner": buyer}):
deals[coinid].append((document["cid"], document.get("txid")))
return deals
async def share_content(self, **params):
if params.get("message"):
params = json.loads(params.get("message", "{}"))
if not params:
return {"error": 400, "reason": "Missed required fields"}
coinid = params.get("coinid")
cid = params.get("cid")
public_key = params.get("public_key")
client = MotorClient()
database = client[coinid]
content_collection = database[settings.DEAL]
await content_collection.insert_one({
"cid": int(cid),
"user": public_key
})
content = await content_collection.find_one({
"cid": int(cid),
"user": public_key
})
logging.debug(content)
if not content:
return {
"error": 404,
"reason":
"Shared content. Content with cid %s not created" % cid
}
return {i: content[i] for i in content if i != "_id"}
def init(cls, db_data: DatabaseConfiguration):
""" Create database with asynchronous connector."""
cls.get_logger().info('Establishing database connection...')
uri = cls.__create_uri(db_data.host, db_data.port, db_data.user,
db_data.password, db_data.name)
# Create db client
cls.__CLIENT = MotorClient(uri)
cls.DB = cls.__CLIENT.get_default_database()
def new_tornado_engine(cls,
db_name: str,
loop,
host: str = 'localhost',
port: int = 27017):
from motor.motor_tornado import MotorClient
client = MotorClient(host=host, port=port, io_loop=loop)
return cls(loop, client, db_name, host, port)
def __init__(self, dbname, collection):
# Set database parameters
self.client = MotorClient()
self.database = self.client[dbname]
self.collection = self.database[collection]
class StoreClient(object):
def __init__(self, user="******", password="******", database="project_data"):
userAndPass = ""
if user and password:
userAndPass = user + ":" + str(password) + "@"
url = "mongodb+srv://"+ userAndPass + "nibanfinance-lgkjt.gcp.mongodb.net/test?retryWrites=true&w=majority"
self.collection = MotorClient(url).store_collection
@gen.coroutine
def get(self, key):
result = yield self.collection.find_one({'key': key})
return result['value'] if result else 'null'
@gen.coroutine
def set(self, key, value):
result = yield self.collection.insert({'key': key, 'value': value})
return 'ok'
def __init__(self):
handlers = urlpatterns
conn = MotorClient(ConfUtil.getMongoIP(), ConfUtil.getMongoPort())
self.db = conn[ConfUtil.getDBName()]
settings = dict(
template_path=ConfUtil.getTemplatePath(),
static_path=ConfUtil.getStaticPath(),
debug=True,
)
tornado.web.Application.__init__(self, handlers=handlers, **settings)
def __init__(self):
handlers = urlpatterns
conn = MotorClient(Settings.MONGO_HOST, Settings.MONGO_PORT)
self.db = conn[Settings.DB_NAME]
settings = dict(
template_path=Settings.TEMPLATE_PATH,
static_path=Settings.STATIC_PATH,
debug=Settings.DEBUG,
)
tornado.web.Application.__init__(self, handlers=handlers, **settings)
async def main():
# host，port，db_name
# 数据库连接，域名，端口，表名test
db = MotorClient(options.db_host, options.db_port).test
# 把db传入application的全局实例
app = Application(db)
app.listen(options.port)
server = tornado.httpserver.HTTPServer(app, xheaders=True)
server.start(1)
se = tornado.locks.Event()
await se.wait()
def init():
mongodbs = []
for db_key in config.MONGODB:
MONGODBS[db_key] = MotorClient(
config.MONGODB[db_key]["HOSTS"], config.MONGODB[db_key]
["HOST_PORT"])[config.MONGODB[db_key]["DATABASE_NAME"]]
mongodbs.append(MONGODBS[db_key].authenticate(
config.MONGODB[db_key]["DATABASE_USER_NAME"],
config.MONGODB[db_key]["DATABASE_PASSWD"]))
return mongodbs
def init(cls,
host='localhost',
port=27017,
db_name='connecting_health',
user=None,
password=None):
""" Create database with asynchronous connector."""
cls.get_logger().info('Establishing database connection...')
uri = cls.__create_uri(host, port, user, password, db_name)
# Create db client
cls.__CLIENT = MotorClient(uri)
cls.DB = cls.__CLIENT.get_default_database()
def __init__(self):
db = MotorClient(settings.MONGO_URL)[settings.DATABASE]
app_settings = {'debug': settings.DEBUG, 'db': db}
handlers = [(r'/api/auth', AuthHandler), (r'/api/info', InfoHandler),
(r'/api/register', AuthHandler, {
'register': True
}),
(r'/api/shortie/?(?P<shortie>[a-zA-Z0-9]+)?',
ShortenHandler)]
IOLoop.current().add_callback(self.init_database, db)
super().__init__(handlers, **app_settings)
def __init__(self):
self.webSocketsPool = []
self.db = MotorClient(MONGO_HOST, MONGO_PORT).chat
handlers = (
(r'/', MainHandler),
(r'/websocket/?', WebSocket),
(r'/static/(.*)', tornado.web.StaticFileHandler, {
'path': 'static/'
}),
)
super().__init__(handlers)
def init_app_options() -> dict:
db = MotorClient(settings.DB.dsn).default
debug = settings.ENVIRONMENT == Environment.DEVELOPMENT
configurator = get_unit_service_from_env()
docker_client = docker.from_env()
return {
'debug': debug,
'repository': ApplicationRepository(db),
'configurator': configurator,
'docker': docker_client,
}
def configure(settings):
if os.path.exists('/var/run/secrets/kubernetes.io/serviceaccount/token'):
with open('/var/run/secrets/kubernetes.io/serviceaccount/token'
) as token:
settings['kube'] = client.KubeClient(
os.getenv('KUBERNETES_SERVICE_HOST'), token=token.read())
mongo_url = "mongodb://{0}:{1}/".format(
os.getenv('ELASTICKUBE_MONGO_SERVICE_HOST', 'localhost'),
os.getenv('ELASTICKUBE_MONGO_SERVICE_PORT', 27017))
settings['motor'] = MotorClient(mongo_url)
settings['database'] = settings['motor'].elastickube
async def update_contents(self, **params):
"""Updates users content row
Accepts:
- txid
- cid
- description
- write_price
- read_price
- confirmed
- coinid
"""
if params.get("message"):
params = json.loads(params.get("message", "{}"))
if not params:
return {"error": 400, "reason": "Missed required fields"}
txid = params.get("txid")
coinid = params.get("coinid").upper()
client = MotorClient()
database = client[coinid]
content_collection = database[settings.CONTENT]
content = await content_collection.find_one({"txid": txid})
if not content:
return {
"error": 404,
"reason":
"Update content. Content with txid %s not found" % txid
}
if content.get("hash"):
client = SignedTornadoClient(settings.bridges[coinid])
cid = await client.request(method_name="getcid",
hash=content["hash"])
await content_collection.find_one_and_update(
{"txid": txid}, {"$set": {
"cid": int(cid)
}})
await content_collection.find_one_and_update(
{"txid": txid}, {"$set": {
"hash": None
}})
updated = await content_collection.find_one({"txid": txid})
return {i: updated[i] for i in updated if i != "_id"}
def __init__(self, isTest=False):
self.__IFConfigClient = MotorClient('mongodb://{username}:{password}@{address}:{port}/{database}'.format(
username=Config['MongoDB.IFConfig'].Username.asString(),
password=Config['MongoDB.IFConfig'].Password.asString(),
address=Config['MongoDB'].Address.asString(),
port=Config['MongoDB'].Port.asInt(),
database='IFConfig'
))
self.__IFConfig = self.__IFConfigClient.get_database('IFConfig')
if isTest:
self.__IFConfig = self.__IFConfigClient.get_database('IFConfigTest')
self.__IFDataClient = MotorClient('mongodb://{username}:{password}@{address}:{port}/{database}'.format(
username=Config['MongoDB.IFData'].Username.asString(),
password=Config['MongoDB.IFData'].Password.asString(),
address=Config['MongoDB'].Address.asString(),
port=Config['MongoDB'].Port.asInt(),
database='IFData'
))
self.__IFData = self.__IFDataClient.get_database('IFData')
if isTest:
self.__IFData = self.__IFDataClient.get_database('IFDataTest')
self.IFConfig = IFConfigContext(self.__IFConfig)
self.IFData = IFDataContext(self.__IFData)
def initialize():
logging.info("Initializing charts sync")
mongo_url = "mongodb://{0}:{1}/".format(
os.getenv("ELASTICKUBE_MONGO_SERVICE_HOST", "localhost"),
os.getenv("ELASTICKUBE_MONGO_SERVICE_PORT", 27017))
motor_client = MotorClient(mongo_url)
try:
watch.start_monitor(motor_client)
yield GitSync(motor_client.elastickube).sync_loop()
except:
logging.exception("Unexpected error executing GitSync sync loop.")
IOLoop.current().stop()
def create_connection(db_config: Box = None) -> None:
"""Register a database connection
Args:
db_config: Yapconf-generated configuration object
Returns:
None
"""
global motor_db
motor_conn = MotorClient(
host=db_config.connection.host,
port=db_config.connection.port,
)
motor_db = motor_conn[db_config.name]
async def set_contents(self, **params):
"""Writes users content to database
Writes users content to database
Accepts:
- public key (required)
- content (required)
- description
- price
- address
"""
if params.get("message"):
params = json.loads(params.get("message", "{}"))
if not params:
return {"error": 400, "reason": "Missed required fields"}
txid = params.get("txid")
public_key = params.get("public_key")
_hash = params.get("hash")
coinid = params.get("coinid")
access = params.get("access")
cid = params.get("cid")
# Try to get account
account = await self.collection.find_one({"public_key": public_key})
# Return error if does not exist the one
if not account:
return {"error": 404, "reason": "Account was not found"}
client = MotorClient()
database = client[coinid]
content_collection = database[access]
await content_collection.insert_one({
"owner": public_key,
"cid": cid,
"txid": txid,
"hash": _hash
})
success = await content_collection.find_one({"txid": txid})
if not success:
return {
"error": 500,
"reason": "Error while writing content to database"
}
else:
return {"result": "ok"}
def main():
tornado.options.parse_command_line()
if '.' not in options.ns:
sys.stderr.write('Invalid ns "%s", must contain a "."' %
(options.ns, ))
sys.exit(1)
db_name, collection_name = options.ns.split('.', 1)
client = MotorClient(options.mongo)
collection = client[db_name][collection_name]
app = Application()
app.listen(options.port)
loop = tornado.ioloop.IOLoop.current()
# Start watching collection for changes.
loop.add_callback(watch, collection)
loop.start()
