def update_mdb_collections(mdbs):
"""
更新数据库mongodb collection, 不存在的colletion则创建
:param mdbs:
:return:
"""
# 读取配置中的数据库json 数据
coll_json_file = "{}/configs/mdb_collections.json".format(APPS_PATH)
if not os.path.exists(coll_json_file):
return
with open(coll_json_file) as rf:
jsondata = rf.read()
if jsondata:
collections = json.loads(jsondata)
else:
collections = {}
# 检查数据库collections
for dbname, colls in collections.items():
mdb = mdbs[dbname]
for coll in colls:
try:
mdb.dbs.create_collection(coll)
web_start_log.info(
"[DB: {}] Create collection '{}'".format(
mdb.name, coll))
except Exception as e:
if "already exists" in str(e):
web_start_log.info(e)
else:
web_start_log.warning(e)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user):
'''
更新数据库mongodb collection, 不存在的colletion则创建
:param mdb_sys:
:param mdb_web:
:param mdb_user:
:return:
'''
# 读取配置中的数据库json 数据
with open("{}/configs/collections.json".format(APPS_PATH)) as rf:
jsondata = rf.read()
if jsondata:
collections = json.loads(jsondata)
else:
collections = {}
dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web}
# 检查数据库collections
for dbname, colls in collections.items():
mdb = dbs[dbname]
for coll in colls:
try:
mdb.dbs.create_collection(coll)
web_start_log.info("[DB: {}] Create collection '{}'".format(
mdb.name, coll))
except Exception as e:
if "already exists" in str(e):
web_start_log.info(e)
else:
web_start_log.warning(e)
# 将最新的数据库结构写入配置文件, 方便开发者了解结构
new_collections = {}
for dbname, mdb in dbs.items():
new_collections[dbname] = {}
collnames = mdb.dbs.collection_names()
for collname in collnames:
if collname == "system.indexes" or collname.startswith("plug_"):
continue
new_collections[dbname][collname] = {}
data = mdb.dbs[collname].find_one({}, {"_id": 0})
if data:
for k, v in data.items():
new_collections[dbname][collname][k] = str(type(v))
with open("{}/configs/collections.json".format(APPS_PATH), "w") as wf:
collections = json.dumps(new_collections, indent=4, ensure_ascii=False)
wf.write(collections)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user):
'''
更新数据库mongodb collection, 不存在的colletion则创建
:param mdb_sys:
:param mdb_web:
:param mdb_user:
:return:
'''
dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web}
for k, colls in collections.items():
mdb = dbs[k]
for coll in colls:
try:
mdb.dbs.create_collection(coll)
web_start_log.info("[DB: {}] Create collection '{}'".format(
mdb.name, coll))
except Exception as e:
if "already exists" in str(e):
web_start_log.info(e)
else:
web_start_log.warning(e)
def update_pylib(input_venv_path=True, latest=False):
"""
更新python环境库
:param need_input:
:return:
"""
if input_venv_path:
try:
input_str = input(
"Already running this script in your project python virtual environment?(yes/no):\n")
if input_str.upper() == "YES":
venv_path = None
else:
venv_path = CONFIG["py_venv"]["VENV_PATH"]["value"]
input_str = input(
"The default path is: {}, Use the default(yes/no)\n".format(venv_path))
if input_str.upper() != "YES":
venv_path = input("Enter a virtual environment:\n")
except BaseException:
venv_path = CONFIG["py_venv"]["VENV_PATH"]["value"]
else:
venv_path = CONFIG["py_venv"]["VENV_PATH"]["value"]
if venv_path:
if os.path.exists("{}/bin/activate".format(venv_path)):
venv = ". {}/bin/activate && ".format(venv_path)
else:
venv = ". {}/bin/activate && ".format(sys.prefix)
else:
venv = ""
print(" * Update pip...({})".format(venv))
s, r = subprocess.getstatusoutput("{}pip3 install -U pip".format(venv))
print(r)
s, r = subprocess.getstatusoutput("{}pip3 freeze".format(venv))
old_reqs = r.split()
with open("{}/requirements.txt".format(PROJECT_PATH)) as rf:
new_reqs = rf.read().split()
# 查找需要安装的包
ret_list = list(set(new_reqs) ^ set(old_reqs))
install_list = []
for ret in ret_list:
if (ret not in old_reqs) and (ret in new_reqs):
install_list.append(ret)
if install_list:
msg = " * To install the following libs"
print(msg)
print(install_list)
if not input_venv_path:
web_start_log.info(msg)
web_start_log.info(install_list)
pass
install_failed = []
for sf in install_list:
if latest:
sf = sf.split("==")[0]
print("pip install -U {}".format(sf))
s, r = subprocess.getstatusoutput(
"{}pip install -U {}".format(venv, sf))
if s:
install_failed.append(sf)
for sf in install_failed:
if latest:
sf = sf.split("==")[0]
s, r = subprocess.getstatusoutput(
"{}pip install -U {}".format(venv, sf))
if not s:
install_failed.remove(sf)
if install_failed:
msg = " * Installation failed library, please manually install"
print(msg)
print(install_failed)
web_start_log.info(msg)
web_start_log.info(install_failed)
# 查找需要卸载的包
s, r = subprocess.getstatusoutput("{}pip freeze".format(venv))
old_reqs = r.split()
ret_list = list(set(new_reqs) ^ set(old_reqs))
uninstall_list = []
for ret in ret_list:
if (ret in old_reqs) and (ret not in new_reqs):
uninstall_list.append(ret)
for sf in uninstall_list[:]:
if "==" not in sf:
uninstall_list.remove(sf)
if uninstall_list:
msg = " * Now don't need python library:"
print(msg)
print(uninstall_list)
if not input_venv_path:
web_start_log.info(msg)
web_start_log.info(uninstall_list)
if latest:
subprocess.getstatusoutput(
"{}pip freeze > {}/requirements.txt".format(venv, PROJECT_PATH))
def init_core_module(app, **kwargs):
"""
初始化核心模块
:param app:
:return:
"""
csrf_enabled = kwargs.get("csrf_enabled")
is_debug = kwargs.get("is_debug")
# app config
web_start_log.info("Initialize the core module")
# 系统必要配置, 优先导入
app.config.from_object(ConfDictToClass(CONFIG["system"], key="value"))
app.config.from_object(ConfDictToClass(CONFIG["key"], key="value"))
# 数据库
app.config.from_object(DatabaseConfig())
for name, mdb in mdbs.items():
mdb.init_app(app, config_prefix=name.upper())
# 缓存
app.config.from_object(ConfDictToClass(CONFIG["cache"], key="value"))
app.config["CACHE_REDIS"] = redis
app.config["CACHE_MONGODB_DBS"] = mdbs["sys"].dbs
cache.init_app(app)
# Clear CONFIG cache
if not is_debug:
version_info = mdbs["sys"].db.sys_config.find_one(
{"new_version": {
"$exists": True
}})
ago_time = time.time() - 3600 * 24
ago_time_30m = time.time() - 1800
if version_info["sys_version_of_config"] >= SYS_CONFIG_VERSION \
and version_info["update_time"] > ago_time \
and version_info["update_time"]<ago_time_30m:
# 系统正在使用的SYS_CONFIG_VERSION版本和当前机器CONFIG的一样，或更高
# And: 配置24小时内已有更新
# So: 这次不更新
msg = " * [sys configs cache] Not clean cache." \
" The system is using the same or higher configuration version.\n" \
" And it was executed within 24 hours."
start_info_print("\033[33m{}\033[0m".format(msg))
web_start_log.warning(msg)
else:
with app.app_context():
msg = " * Clean configuration cache successfully"
cache.delete(CONFIG_CACHE_KEY)
cache.delete(PLUG_IN_CONFIG_CACHE_KEY)
web_start_log.info(msg)
start_info_print(msg)
# 异常错误信息
app.config["PRESERVE_CONTEXT_ON_EXCEPTION"] = PRESERVE_CONTEXT_ON_EXCEPTION
###################################################
# 在此之前, 任何程序不能调用utils.get_config.py下的方法
###################################################
from apps.core.utils.get_config import get_configs, get_config
from apps.core.flask.request import OsrRequestProcess
from apps.core.flask.errorhandler import ErrorHandler
from apps.core.blueprint import api, admin_view, theme_view, static_html_view, static, open_api
from apps.core.flask.routing import RegexConverter
from apps.core.flask.routing import push_url_to_db
# 最大请求大小
app.config["MAX_CONTENT_LENGTH"] = get_config(
"system", "MAX_CONTENT_LENGTH") * 1024 * 1024
# Session会话配置
session_config = get_configs("session")
session_config["SESSION_PROTECTION"] = SESSION_PROTECTION
session_config["SESSION_COOKIE_PATH"] = SESSION_COOKIE_PATH
session_config["SESSION_COOKIE_HTTPONLY"] = SESSION_COOKIE_HTTPONLY
session_config["SESSION_COOKIE_SECURE"] = SESSION_COOKIE_SECURE
session_config["SESSION_USE_SIGNER"] = SESSION_USE_SIGNER
session_config["SESSION_MONGODB_DB"] = mdbs["sys"].name
app.config.from_object(ConfDictToClass(session_config))
app.config["SESSION_REDIS"] = redis
app.config["SESSION_MONGODB"] = mdbs["sys"].connection
sess.init_app(app)
rest_session.init_app(app)
# 邮件
app.config.from_object(ConfDictToClass(get_configs("email")))
mail.init_app(app)
# Csrf token
csrf_config = {}
if csrf_enabled:
csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = True
start_info_print(" * Security authentication is turned on")
else:
csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = False
start_info_print(
"\033[31m WARNING: security verification is turned off\033[0m")
# 这两个csrf参数这里关闭，request程序会根据CLIENT_TOKEN_AUTH_ENABLED判断处理
csrf_config["WTF_CSRF_CHECK_DEFAULT"] = False
csrf_config["CSRF_ENABLED"] = False
csrf_config["WTF_CSRF_METHODS"] = WTF_CSRF_METHODS
app.config.from_object(ConfDictToClass(csrf_config))
csrf.init_app(app)
# Babel
app.config.from_object(ConfDictToClass(get_configs("babel")))
app.config["BABEL_TRANSLATION_DIRECTORIES"] = BABEL_TRANSLATION_DIRECTORIES
babel.init_app(app)
# 登录管理
login_manager.init_app(app)
# login_manager.anonymous_user = AnonymousUser()
login_manager.session_protection = SESSION_PROTECTION
oauth.init_app(app)
# 让路由支持正则
app.url_map.converters['regex'] = RegexConverter
# 注册蓝图 blueprint
web_start_log.info("Register blueprint, Initialize the routing")
app.register_blueprint(api)
app.register_blueprint(open_api)
app.register_blueprint(admin_view)
app.register_blueprint(theme_view)
app.register_blueprint(static_html_view)
app.register_blueprint(static)
if not is_debug:
st = time.time()
push_url_to_db(app)
start_info_print(
" * Routing updates saved in the database. It tasks time {} sec".
format(int(time.time() - st)))
celery.conf.update(app.config)
# 请求处理
request_process = OsrRequestProcess()
request_process.init_request_process(app=app)
request_process.init_babel_locale_selector(babel=babel)
# 错误处理
ErrorHandler(app)
# Logger
# Other
log_udp = LogServerUDP()
r = log_udp.init_app()
if r:
log_udp.log_server()
weblog = WebLogger()
weblog.init_app(app)
def push_to_db(mdb_sys, local_config=None, now_version=None):
'''
初始化或者更新配置数据到数据库, 顺便更新CONFIG变量的值
config:如果为真, 则使用传入的这个配置
cover: 是否覆盖数据库中的数据
:return:
'''
if not now_version:
return False
web_start_log.info("Push to the config version:{}".format(now_version))
for k, v in local_config.items():
if not isinstance(v, dict):
continue
for k1, v1 in v.items():
if k1.startswith("__"):
continue
try:
if "__restart__" in v:
conf = v1
conf["update_time"] = time.time()
conf["__restart__"] = v["__restart__"]
if "__info__" in v:
conf["__info__"] = v["__info__"]
if "__sort__" in v:
conf["__sort__"] = v["__sort__"]
r = mdb_sys.db.sys_config.update_one(
{
"project": k,
"key": k1,
"conf_version": now_version
}, {"$set": conf},
upsert=True)
if r.modified_count or r.upserted_id:
web_start_log.info(
"Config updates: [file to db] {} {}".format(k, k1))
elif r.matched_count:
web_start_log.info(
"[file to db] Not updated:{} {}".format(k, k1))
else:
web_start_log.warning(
"[file to db] Update failure:{} {}".format(k, k1))
else:
web_start_log.info("[Not import] {} {}".format(k, k1))
except Exception as e:
web_start_log.error("[file to db] {}, {} {}".format(e, k, k1))
# 更新版本信息
# 将当前版本纳入used_versions, 以便之后管理端修改配置时自动生成新版本
mdb_sys.db.sys_config.update_one(
{
"new_version": {
"$exists": True
},
"used_versions": {
"$ne": now_version
}
}, {"$addToSet": {
"used_versions": now_version
}})
# 更新主机信息
host_version = mdb_sys.db.sys_host.find_one({
"type":
"web",
"host_info.local_ip":
host_info["local_ip"]
})
up_version = {"conf_version": now_version, "switch_conf_version": None}
if host_version:
up_version["host_info"] = host_version["host_info"]
else:
up_version["disable_update_conf"] = 0
up_version["host_info"] = {}
for k, v in host_info.items():
up_version["host_info"][k] = v
if not host_version or not host_version["disable_update_conf"]:
up_version["start_log"] = "Normal"
web_start_log.info(up_version["start_log"])
else:
up_version[
"start_log"] = "Update failed. Because there is no open to update"
web_start_log.warning(up_version["start_log"])
# 更新主机信息
mdb_sys.db.sys_host.delete_many({
"$or": [{
"host_info.local_ip": {
"$exists": False
}
}, {
"host_info.local_ip": {
"$in": [None, "", False]
}
}]
})
mdb_sys.db.sys_host.update_one(
{
"type": "web",
"host_info.local_ip": host_info["local_ip"]
}, {"$set": up_version},
upsert=True)
return True
def update_config_file(mdb_sys, *args, **kwargs):
'''
网站启动的时候, 数据库保存的配置-配置文件 同步更新
config
mdb_sys
redis
'''
local_config = deepcopy(CONFIG)
overwrite_db = OVERWRITE_DB
version_info = mdb_sys.db.sys_config.find_one(
{"new_version": {
"$exists": True
}})
if not version_info:
now_version = time_to_utcdate(time_stamp=now_time,
tformat="%Y_%m_%d_%H_%M_%S")
version_uses = {
"new_version": now_version,
"used_versions": [now_version],
"update_time": now_time
}
mdb_sys.db.sys_config.insert_one(version_uses)
web_start_log.info("Initialize the sys_config version info")
else:
now_version = version_info["new_version"]
if version_info and not overwrite_db:
# 查询当前主机web的的配置版本
cur_h_version_info = mdb_sys.db.sys_host.find_one({
"type":
"web",
"host_info.local_ip":
host_info["local_ip"]
})
if not cur_h_version_info:
cur_h_version_info = {
"type": "web",
"host_info": host_info,
"conf_version": version_info["new_version"],
"switch_conf_version": None,
"disable_update_conf": 0
}
mdb_sys.db.sys_host.insert_one(cur_h_version_info)
web_start_log.info("Initialize the host version data")
if cur_h_version_info["switch_conf_version"] or cur_h_version_info[
"disable_update_conf"]:
# 数据库配置和本地配置合并:保留本地和数据库的key,用于网站版本回滚
if cur_h_version_info[
"switch_conf_version"] and not cur_h_version_info[
"disable_update_conf"]:
# 版本切换
now_version = cur_h_version_info["switch_conf_version"]
else:
# 禁止更新
now_version = cur_h_version_info["conf_version"]
confs = mdb_sys.db.sys_config.find({"conf_version": now_version})
if confs.count(True):
for conf in confs:
if not re.search(r"^__.*__$", conf["key"]):
if not conf["project"] in local_config:
local_config[conf["project"]] = {}
if not conf["key"] in local_config[conf["project"]]:
local_config[conf["project"]][conf["key"]] = {
"value": conf["value"],
"type": conf["type"],
"info": conf["info"]
}
else:
local_config[conf["project"]][
conf["key"]]["value"] = conf["value"]
web_start_log.info(
"Config rollback:[db to file] Update version info")
else:
web_start_log.error(
"Config rollback:[db to file] Rollback failure")
return False
else:
# 数据库最新版配置和本地配置合并:以本地配置的key为准,多余的删除
now_version = version_info["new_version"]
confs = mdb_sys.db.sys_config.find({"conf_version": now_version})
if confs.count(True):
for conf in confs:
is_cft = re.search(r"^__.*__$", conf["key"])
if not is_cft and conf["project"] in local_config.keys(
) and conf["key"] in local_config[conf["project"]].keys():
local_config[conf["project"]][
conf["key"]]["value"] = conf["value"]
web_start_log.info(
"Config merge:[db to file] {} {}".format(
conf["project"], conf["key"]))
else:
mdb_sys.db.sys_config.delete_one({"_id": conf["_id"]})
web_start_log.info("Remove the config:{} {}".format(
conf["project"], conf["key"]))
else:
web_start_log.error("Config merge:[db to file] Merger failure")
return False
else:
web_start_log.info(
"**Local configuration directly covering the latest edition of the configuration database"
)
r = push_to_db(mdb_sys,
local_config=deepcopy(local_config),
now_version=now_version)
if not r:
web_start_log.error("Config update:[file to db] Push failure")
return False
# 写配置文件
info = '''#!/usr/bin/env python\n# -*-coding:utf-8-*-\n__author__ = "Allen Woo"\n'''
doc = "__readme__='''{}'''\n".format(__readme__)
# write config.py
temp_conf = str(json.dumps(local_config, indent=4, ensure_ascii=False))
wf = open("{}/apps/configs/config.py".format(PROJECT_PATH), "wb")
wf.write(bytes(info, "utf-8"))
wf.write(bytes(doc, "utf-8"))
# 修改配置为同步数据库配置到文件
wf.write(
bytes(
"# Danger: If True, the database configuration data will be overwritten\n",
"utf-8"))
wf.write(bytes("# 危险:如果为True, 则会把该文件配置覆盖掉数据库中保存的配置\n", "utf-8"))
wf.write(bytes("OVERWRITE_DB = False\n", "utf-8"))
wf.write(bytes("CONFIG = ", "utf-8"))
wf.write(
bytes(
temp_conf.replace("false",
"False").replace("true",
"True").replace("null", "None"),
"utf-8"))
wf.close()
web_start_log.info("Configuration updates and merge is complete")
return True
def init_core_module(app):
"""
初始化核心模块
:param app:
:return:
"""
# app config
web_start_log.info("Initialize the core module")
# 系统必要配置, 优先导入
app.config.from_object(ConfDictToClass(CONFIG["system"], key="value"))
app.config.from_object(ConfDictToClass(CONFIG["key"], key="value"))
# 数据库
app.config.from_object(DatabaseConfig())
for name, mdb in mdbs.items():
mdb.init_app(app, config_prefix=name.upper())
# 缓存
app.config.from_object(ConfDictToClass(CONFIG["cache"], key="value"))
app.config["CACHE_REDIS"] = redis
app.config["CACHE_MONGODB"] = mdbs["sys"].connection
app.config["CACHE_MONGODB_DB"] = mdbs["sys"].name
cache.init_app(app)
# 清除配置CONFIG的cache
with app.app_context():
msg = " * Clean configuration cache successfully"
cache.delete(CONFIG_CACHE_KEY)
cache.delete(PLUG_IN_CONFIG_CACHE_KEY)
web_start_log.info(msg)
print(msg)
# 异常错误信息
app.config["PRESERVE_CONTEXT_ON_EXCEPTION"] = PRESERVE_CONTEXT_ON_EXCEPTION
###################################################
# 在此之前, 任何程序不能调用utils.get_config.py下的方法
###################################################
from apps.core.utils.get_config import get_configs, get_config
from apps.core.flask.request import OsrRequestProcess
from apps.core.flask.errorhandler import ErrorHandler
from apps.core.blueprint import api, admin_view, theme_view, static_html_view, static, open_api
from apps.core.flask.routing import RegexConverter
from apps.core.flask.routing import push_url_to_db
# 最大请求大小
app.config["MAX_CONTENT_LENGTH"] = get_config(
"system", "MAX_CONTENT_LENGTH") * 1024 * 1024
# Session会话配置
session_config = get_configs("session")
session_config["SESSION_PROTECTION"] = SESSION_PROTECTION
session_config["SESSION_COOKIE_PATH"] = SESSION_COOKIE_PATH
session_config["SESSION_COOKIE_HTTPONLY"] = SESSION_COOKIE_HTTPONLY
session_config["SESSION_COOKIE_SECURE"] = SESSION_COOKIE_SECURE
session_config["SESSION_USE_SIGNER"] = SESSION_USE_SIGNER
session_config["SESSION_MONGODB_DB"] = mdbs["sys"].name
app.config.from_object(ConfDictToClass(session_config))
app.config["SESSION_REDIS"] = redis
app.config["SESSION_MONGODB"] = mdbs["sys"].connection
sess.init_app(app)
rest_session.init_app(app)
# 邮件
app.config.from_object(ConfDictToClass(get_configs("email")))
mail.init_app(app)
# Csrf token
csrf_config = {}
csrf_config["CSRF_ENABLED"] = CSRF_ENABLED
csrf_config["WTF_CSRF_CHECK_DEFAULT"] = WTF_CSRF_CHECK_DEFAULT
csrf_config["WTF_CSRF_METHODS"] = WTF_CSRF_METHODS
app.config.from_object(ConfDictToClass(csrf_config))
csrf.init_app(app)
# Babel
app.config.from_object(ConfDictToClass(get_configs("babel")))
app.config["BABEL_TRANSLATION_DIRECTORIES"] = BABEL_TRANSLATION_DIRECTORIES
babel.init_app(app)
# 登录管理
login_manager.init_app(app)
# login_manager.anonymous_user = AnonymousUser()
login_manager.session_protection = SESSION_PROTECTION
oauth.init_app(app)
# 让路由支持正则
app.url_map.converters['regex'] = RegexConverter
# 注册蓝图 blueprint
web_start_log.info("Register blueprint, Initialize the routing")
app.register_blueprint(api)
app.register_blueprint(open_api)
app.register_blueprint(admin_view)
app.register_blueprint(theme_view)
app.register_blueprint(static_html_view)
app.register_blueprint(static)
push_url_to_db(app)
# 请求处理
request_process = OsrRequestProcess()
request_process.init_request_process(app=app)
request_process.init_babel_locale_selector(babel=babel)
# 错误处理
ErrorHandler(app)
# Logger
# Other
log_udp = LogServerUDP()
r = log_udp.init_app()
if r:
log_udp.log_server()
weblog = WebLogger()
weblog.init_app(app)
from apps.core.db.mongodb import PyMongo
from apps.core.logger.web_logging import WebLogger, web_start_log
from flask_babel import Babel
from flask_mail import Mail
from flask_oauthlib.client import OAuth
from flask_session import Session
from flask_wtf import CSRFProtect
from flask_login import LoginManager
from redis import StrictRedis
from apps.configs.db_config import DB_CONFIG
'''
Flask app 与其他核心模块实例化
注意: 不要将模块初始话设置放在本文件
'''
# 主程序
web_start_log.info("Initialize the OsrApp")
app = OsrApp(__name__)
web_start_log.info("Instance of each module")
mdb_web = PyMongo()
mdb_sys = PyMongo()
mdb_user = PyMongo()
cache = Cache()
babel = Babel()
csrf = CSRFProtect()
login_manager = LoginManager()
sess = Session()
rest_session = RestSession()
mail = Mail()
weblog = WebLogger()
oauth = OAuth()
redis = StrictRedis(host=DB_CONFIG["redis"]["host"][0],
def update_config_file(mdbs, *args, **kwargs):
"""
网站启动的时候, 数据库保存的配置-配置文件 同步更新
config
mdb_sys
redis
"""
local_config = deepcopy(CONFIG)
overwrite_db = OVERWRITE_DB
version_info = mdbs["sys"].db.sys_config.find_one(
{"new_version": {
"$exists": True
}})
if not version_info:
now_version = time_to_utcdate(time_stamp=now_time,
tformat="%Y_%m_%d_%H_%M_%S")
version_uses = {
"new_version": now_version,
"used_versions": [now_version],
"update_time": now_time
}
mdbs["sys"].db.sys_config.insert_one(version_uses)
sys_version_of_config = SYS_CONFIG_VERSION
last_update_time = now_time
web_start_log.info("Initialize the sys_config version info")
else:
now_version = version_info["new_version"]
last_update_time = version_info["update_time"]
if "sys_version_of_config" in version_info:
sys_version_of_config = version_info["sys_version_of_config"]
else:
sys_version_of_config = SYS_CONFIG_VERSION
mdbs["sys"].db.sys_config.update_one(
{"new_version": {
"$exists": True
}}, {"$set": {
"sys_version_of_config": SYS_CONFIG_VERSION
}})
if version_info and not overwrite_db:
# 查询当前主机web的的配置版本
cur_h_version_info = mdbs["sys"].db.sys_host.find_one({
"type":
"web",
"host_info.local_ip":
host_info["local_ip"]
})
if not cur_h_version_info:
cur_h_version_info = {
"type": "web",
"host_info": host_info,
"conf_version": version_info["new_version"],
"switch_conf_version": None,
"disable_update_conf": 0
}
mdbs["sys"].db.sys_host.insert_one(cur_h_version_info)
web_start_log.info("Initialize the host version data")
if cur_h_version_info["switch_conf_version"] or cur_h_version_info[
"disable_update_conf"]:
# 数据库配置和本地配置合并:保留本地和数据库的key,用于网站版本回滚
if cur_h_version_info[
"switch_conf_version"] and not cur_h_version_info[
"disable_update_conf"]:
# 版本切换
now_version = cur_h_version_info["switch_conf_version"]
else:
# 禁止更新
now_version = cur_h_version_info["conf_version"]
confs = mdbs["sys"].db.sys_config.find(
{"conf_version": now_version})
if confs.count(True):
for conf in confs:
if not re.search(r"^__.*__$", conf["key"]):
if not conf["project"] in local_config:
local_config[conf["project"]] = {}
if not conf["key"] in local_config[conf["project"]]:
local_config[conf["project"]][conf["key"]] = {
"value": conf["value"],
"type": conf["type"],
"info": conf["info"]
}
else:
local_config[conf["project"]][
conf["key"]]["value"] = conf["value"]
web_start_log.info(
"Config rollback:[db to file] Update version info")
else:
web_start_log.error(
"Config rollback:[db to file] Rollback failure")
return False
else:
# 数据库最新版配置和本地配置合并:以本地配置的key为准,多余的删除
# 数据库最新版配置和本地配置合并:以本地配置的key为准,多余的删除
ago_time = now_time - 3600 * 24
if sys_version_of_config >= SYS_CONFIG_VERSION and last_update_time > ago_time:
# 系统正在使用的SYS_CONFIG_VERSION版本和当前机器CONFIG的一样，或更高
# And: 配置24小时内已有更新
# So: 这次不更新
msg = " * [sys configs] Not updated. The system is using the same or higher configuration version.\n" \
" And it was executed within 24 hours.\n"
start_info_print("\033[33m{}\033[0m".format(msg))
return True
now_version = version_info["new_version"]
confs = mdbs["sys"].db.sys_config.find(
{"conf_version": now_version})
if confs.count(True):
for conf in confs:
is_cft = re.search(r"^__.*__$", conf["key"])
if not is_cft and conf["project"] in local_config.keys() \
and conf["key"] in local_config[conf["project"]].keys():
local_config[conf["project"]][
conf["key"]]["value"] = conf["value"]
web_start_log.info(
"Config merge:[db to file] {} {}".format(
conf["project"], conf["key"]))
else:
mdbs["sys"].db.sys_config.delete_one(
{"_id": conf["_id"]})
web_start_log.info("Remove the config:{} {}".format(
conf["project"], conf["key"]))
else:
web_start_log.error("Config merge:[db to file] Merger failure")
return False
else:
web_start_log.info(
"**Local configuration directly covering the latest edition of the configuration database"
)
r = push_to_db(mdbs,
local_config=deepcopy(local_config),
now_version=now_version)
if not r:
web_start_log.error("Config update:[file to db] Push failure")
return False
# 写配置文件
# info = """#!/usr/bin/env python\n# -*-coding:utf-8-*-\n__author__ = "Allen Woo"\n"""
# doc = "__readme__= '''{}'''\n".format(__readme__)
#
# # write config.py
#
# # 把password类型替换成*, 再写入文件，防止提交代码时把密码上传到git
# for k, v in local_config.items():
# for k1, v1 in v.items():
# if k1.startswith("__") and k1.endswith("__"):
# continue
# if "type" in v1 and v1["type"] == "password":
# # 由于上一个版的password已被替换，现在需要把把它写写入到CONFIG中
# CONFIG[k][k1]["value"] = v1["value"]
# # 替换密码
# v1["value"] = "<Your password>"
#
# temp_conf = str(json.dumps(local_config, indent=4, ensure_ascii=False))
# wf = open("{}/apps/configs/config.py".format(PROJECT_PATH), "wb")
# wf.write(bytes(info, "utf-8"))
# wf.write(bytes(doc, "utf-8"))
# # 修改配置为同步数据库配置到文件
# wf.write(bytes("# Danger: If True, the database configuration data will be overwritten\n", "utf-8"))
# wf.write(bytes("# 危险:如果为True, 则会把该文件配置覆盖掉数据库中保存的配置\n", "utf-8"))
# wf.write(bytes("OVERWRITE_DB = False\n", "utf-8"))
# wf.write(bytes("CONFIG = ", "utf-8"))
# wf.write(bytes(temp_conf.replace("false",
# "False").replace("true",
# "True").replace("null",
# "None"),
# "utf-8"))
# wf.close()
web_start_log.info("Configuration updates and merge is complete")
return True
