def update_mdb_collections(mdbs):
"""
更新数据库mongodb collection, 不存在的colletion则创建
:param mdbs:
:return:
"""
# 读取配置中的数据库json 数据
coll_json_file = "{}/configs/mdb_collections.json".format(APPS_PATH)
if not os.path.exists(coll_json_file):
return
with open(coll_json_file) as rf:
jsondata = rf.read()
if jsondata:
collections = json.loads(jsondata)
else:
collections = {}
# 检查数据库collections
for dbname, colls in collections.items():
mdb = mdbs[dbname]
for coll in colls:
try:
mdb.dbs.create_collection(coll)
web_start_log.info(
"[DB: {}] Create collection '{}'".format(
mdb.name, coll))
except Exception as e:
if "already exists" in str(e):
web_start_log.info(e)
else:
web_start_log.warning(e)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user):
'''
更新数据库mongodb collection, 不存在的colletion则创建
:param mdb_sys:
:param mdb_web:
:param mdb_user:
:return:
'''
# 读取配置中的数据库json 数据
with open("{}/configs/collections.json".format(APPS_PATH)) as rf:
jsondata = rf.read()
if jsondata:
collections = json.loads(jsondata)
else:
collections = {}
dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web}
# 检查数据库collections
for dbname, colls in collections.items():
mdb = dbs[dbname]
for coll in colls:
try:
mdb.dbs.create_collection(coll)
web_start_log.info("[DB: {}] Create collection '{}'".format(
mdb.name, coll))
except Exception as e:
if "already exists" in str(e):
web_start_log.info(e)
else:
web_start_log.warning(e)
# 将最新的数据库结构写入配置文件, 方便开发者了解结构
new_collections = {}
for dbname, mdb in dbs.items():
new_collections[dbname] = {}
collnames = mdb.dbs.collection_names()
for collname in collnames:
if collname == "system.indexes" or collname.startswith("plug_"):
continue
new_collections[dbname][collname] = {}
data = mdb.dbs[collname].find_one({}, {"_id": 0})
if data:
for k, v in data.items():
new_collections[dbname][collname][k] = str(type(v))
with open("{}/configs/collections.json".format(APPS_PATH), "w") as wf:
collections = json.dumps(new_collections, indent=4, ensure_ascii=False)
wf.write(collections)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user):
'''
更新数据库mongodb collection, 不存在的colletion则创建
:param mdb_sys:
:param mdb_web:
:param mdb_user:
:return:
'''
dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web}
for k, colls in collections.items():
mdb = dbs[k]
for coll in colls:
try:
mdb.dbs.create_collection(coll)
web_start_log.info("[DB: {}] Create collection '{}'".format(
mdb.name, coll))
except Exception as e:
if "already exists" in str(e):
web_start_log.info(e)
else:
web_start_log.warning(e)
def init_core_module(app, **kwargs):
"""
初始化核心模块
:param app:
:return:
"""
csrf_enabled = kwargs.get("csrf_enabled")
is_debug = kwargs.get("is_debug")
# app config
web_start_log.info("Initialize the core module")
# 系统必要配置, 优先导入
app.config.from_object(ConfDictToClass(CONFIG["system"], key="value"))
app.config.from_object(ConfDictToClass(CONFIG["key"], key="value"))
# 数据库
app.config.from_object(DatabaseConfig())
for name, mdb in mdbs.items():
mdb.init_app(app, config_prefix=name.upper())
# 缓存
app.config.from_object(ConfDictToClass(CONFIG["cache"], key="value"))
app.config["CACHE_REDIS"] = redis
app.config["CACHE_MONGODB_DBS"] = mdbs["sys"].dbs
cache.init_app(app)
# Clear CONFIG cache
if not is_debug:
version_info = mdbs["sys"].db.sys_config.find_one(
{"new_version": {
"$exists": True
}})
ago_time = time.time() - 3600 * 24
ago_time_30m = time.time() - 1800
if version_info["sys_version_of_config"] >= SYS_CONFIG_VERSION \
and version_info["update_time"] > ago_time \
and version_info["update_time"]<ago_time_30m:
# 系统正在使用的SYS_CONFIG_VERSION版本和当前机器CONFIG的一样，或更高
# And: 配置24小时内已有更新
# So: 这次不更新
msg = " * [sys configs cache] Not clean cache." \
" The system is using the same or higher configuration version.\n" \
" And it was executed within 24 hours."
start_info_print("\033[33m{}\033[0m".format(msg))
web_start_log.warning(msg)
else:
with app.app_context():
msg = " * Clean configuration cache successfully"
cache.delete(CONFIG_CACHE_KEY)
cache.delete(PLUG_IN_CONFIG_CACHE_KEY)
web_start_log.info(msg)
start_info_print(msg)
# 异常错误信息
app.config["PRESERVE_CONTEXT_ON_EXCEPTION"] = PRESERVE_CONTEXT_ON_EXCEPTION
###################################################
# 在此之前, 任何程序不能调用utils.get_config.py下的方法
###################################################
from apps.core.utils.get_config import get_configs, get_config
from apps.core.flask.request import OsrRequestProcess
from apps.core.flask.errorhandler import ErrorHandler
from apps.core.blueprint import api, admin_view, theme_view, static_html_view, static, open_api
from apps.core.flask.routing import RegexConverter
from apps.core.flask.routing import push_url_to_db
# 最大请求大小
app.config["MAX_CONTENT_LENGTH"] = get_config(
"system", "MAX_CONTENT_LENGTH") * 1024 * 1024
# Session会话配置
session_config = get_configs("session")
session_config["SESSION_PROTECTION"] = SESSION_PROTECTION
session_config["SESSION_COOKIE_PATH"] = SESSION_COOKIE_PATH
session_config["SESSION_COOKIE_HTTPONLY"] = SESSION_COOKIE_HTTPONLY
session_config["SESSION_COOKIE_SECURE"] = SESSION_COOKIE_SECURE
session_config["SESSION_USE_SIGNER"] = SESSION_USE_SIGNER
session_config["SESSION_MONGODB_DB"] = mdbs["sys"].name
app.config.from_object(ConfDictToClass(session_config))
app.config["SESSION_REDIS"] = redis
app.config["SESSION_MONGODB"] = mdbs["sys"].connection
sess.init_app(app)
rest_session.init_app(app)
# 邮件
app.config.from_object(ConfDictToClass(get_configs("email")))
mail.init_app(app)
# Csrf token
csrf_config = {}
if csrf_enabled:
csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = True
start_info_print(" * Security authentication is turned on")
else:
csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = False
start_info_print(
"\033[31m WARNING: security verification is turned off\033[0m")
# 这两个csrf参数这里关闭，request程序会根据CLIENT_TOKEN_AUTH_ENABLED判断处理
csrf_config["WTF_CSRF_CHECK_DEFAULT"] = False
csrf_config["CSRF_ENABLED"] = False
csrf_config["WTF_CSRF_METHODS"] = WTF_CSRF_METHODS
app.config.from_object(ConfDictToClass(csrf_config))
csrf.init_app(app)
# Babel
app.config.from_object(ConfDictToClass(get_configs("babel")))
app.config["BABEL_TRANSLATION_DIRECTORIES"] = BABEL_TRANSLATION_DIRECTORIES
babel.init_app(app)
# 登录管理
login_manager.init_app(app)
# login_manager.anonymous_user = AnonymousUser()
login_manager.session_protection = SESSION_PROTECTION
oauth.init_app(app)
# 让路由支持正则
app.url_map.converters['regex'] = RegexConverter
# 注册蓝图 blueprint
web_start_log.info("Register blueprint, Initialize the routing")
app.register_blueprint(api)
app.register_blueprint(open_api)
app.register_blueprint(admin_view)
app.register_blueprint(theme_view)
app.register_blueprint(static_html_view)
app.register_blueprint(static)
if not is_debug:
st = time.time()
push_url_to_db(app)
start_info_print(
" * Routing updates saved in the database. It tasks time {} sec".
format(int(time.time() - st)))
celery.conf.update(app.config)
# 请求处理
request_process = OsrRequestProcess()
request_process.init_request_process(app=app)
request_process.init_babel_locale_selector(babel=babel)
# 错误处理
ErrorHandler(app)
# Logger
# Other
log_udp = LogServerUDP()
r = log_udp.init_app()
if r:
log_udp.log_server()
weblog = WebLogger()
weblog.init_app(app)
def push_to_db(mdb_sys, local_config=None, now_version=None):
'''
初始化或者更新配置数据到数据库, 顺便更新CONFIG变量的值
config:如果为真, 则使用传入的这个配置
cover: 是否覆盖数据库中的数据
:return:
'''
if not now_version:
return False
web_start_log.info("Push to the config version:{}".format(now_version))
for k, v in local_config.items():
if not isinstance(v, dict):
continue
for k1, v1 in v.items():
if k1.startswith("__"):
continue
try:
if "__restart__" in v:
conf = v1
conf["update_time"] = time.time()
conf["__restart__"] = v["__restart__"]
if "__info__" in v:
conf["__info__"] = v["__info__"]
if "__sort__" in v:
conf["__sort__"] = v["__sort__"]
r = mdb_sys.db.sys_config.update_one(
{
"project": k,
"key": k1,
"conf_version": now_version
}, {"$set": conf},
upsert=True)
if r.modified_count or r.upserted_id:
web_start_log.info(
"Config updates: [file to db] {} {}".format(k, k1))
elif r.matched_count:
web_start_log.info(
"[file to db] Not updated:{} {}".format(k, k1))
else:
web_start_log.warning(
"[file to db] Update failure:{} {}".format(k, k1))
else:
web_start_log.info("[Not import] {} {}".format(k, k1))
except Exception as e:
web_start_log.error("[file to db] {}, {} {}".format(e, k, k1))
# 更新版本信息
# 将当前版本纳入used_versions, 以便之后管理端修改配置时自动生成新版本
mdb_sys.db.sys_config.update_one(
{
"new_version": {
"$exists": True
},
"used_versions": {
"$ne": now_version
}
}, {"$addToSet": {
"used_versions": now_version
}})
# 更新主机信息
host_version = mdb_sys.db.sys_host.find_one({
"type":
"web",
"host_info.local_ip":
host_info["local_ip"]
})
up_version = {"conf_version": now_version, "switch_conf_version": None}
if host_version:
up_version["host_info"] = host_version["host_info"]
else:
up_version["disable_update_conf"] = 0
up_version["host_info"] = {}
for k, v in host_info.items():
up_version["host_info"][k] = v
if not host_version or not host_version["disable_update_conf"]:
up_version["start_log"] = "Normal"
web_start_log.info(up_version["start_log"])
else:
up_version[
"start_log"] = "Update failed. Because there is no open to update"
web_start_log.warning(up_version["start_log"])
# 更新主机信息
mdb_sys.db.sys_host.delete_many({
"$or": [{
"host_info.local_ip": {
"$exists": False
}
}, {
"host_info.local_ip": {
"$in": [None, "", False]
}
}]
})
mdb_sys.db.sys_host.update_one(
{
"type": "web",
"host_info.local_ip": host_info["local_ip"]
}, {"$set": up_version},
upsert=True)
return True
