def update_mdb_collections(mdbs): """ 更新数据库mongodb collection, 不存在的colletion则创建 :param mdbs: :return: """ # 读取配置中的数据库json 数据 coll_json_file = "{}/configs/mdb_collections.json".format(APPS_PATH) if not os.path.exists(coll_json_file): return with open(coll_json_file) as rf: jsondata = rf.read() if jsondata: collections = json.loads(jsondata) else: collections = {} # 检查数据库collections for dbname, colls in collections.items(): mdb = mdbs[dbname] for coll in colls: try: mdb.dbs.create_collection(coll) web_start_log.info( "[DB: {}] Create collection '{}'".format( mdb.name, coll)) except Exception as e: if "already exists" in str(e): web_start_log.info(e) else: web_start_log.warning(e)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user): ''' 更新数据库mongodb collection, 不存在的colletion则创建 :param mdb_sys: :param mdb_web: :param mdb_user: :return: ''' # 读取配置中的数据库json 数据 with open("{}/configs/collections.json".format(APPS_PATH)) as rf: jsondata = rf.read() if jsondata: collections = json.loads(jsondata) else: collections = {} dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web} # 检查数据库collections for dbname, colls in collections.items(): mdb = dbs[dbname] for coll in colls: try: mdb.dbs.create_collection(coll) web_start_log.info("[DB: {}] Create collection '{}'".format( mdb.name, coll)) except Exception as e: if "already exists" in str(e): web_start_log.info(e) else: web_start_log.warning(e) # 将最新的数据库结构写入配置文件, 方便开发者了解结构 new_collections = {} for dbname, mdb in dbs.items(): new_collections[dbname] = {} collnames = mdb.dbs.collection_names() for collname in collnames: if collname == "system.indexes" or collname.startswith("plug_"): continue new_collections[dbname][collname] = {} data = mdb.dbs[collname].find_one({}, {"_id": 0}) if data: for k, v in data.items(): new_collections[dbname][collname][k] = str(type(v)) with open("{}/configs/collections.json".format(APPS_PATH), "w") as wf: collections = json.dumps(new_collections, indent=4, ensure_ascii=False) wf.write(collections)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user): ''' 更新数据库mongodb collection, 不存在的colletion则创建 :param mdb_sys: :param mdb_web: :param mdb_user: :return: ''' dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web} for k, colls in collections.items(): mdb = dbs[k] for coll in colls: try: mdb.dbs.create_collection(coll) web_start_log.info("[DB: {}] Create collection '{}'".format( mdb.name, coll)) except Exception as e: if "already exists" in str(e): web_start_log.info(e) else: web_start_log.warning(e)
def update_pylib(input_venv_path=True, latest=False): """ 更新python环境库 :param need_input: :return: """ if input_venv_path: try: input_str = input( "Already running this script in your project python virtual environment?(yes/no):\n") if input_str.upper() == "YES": venv_path = None else: venv_path = CONFIG["py_venv"]["VENV_PATH"]["value"] input_str = input( "The default path is: {}, Use the default(yes/no)\n".format(venv_path)) if input_str.upper() != "YES": venv_path = input("Enter a virtual environment:\n") except BaseException: venv_path = CONFIG["py_venv"]["VENV_PATH"]["value"] else: venv_path = CONFIG["py_venv"]["VENV_PATH"]["value"] if venv_path: if os.path.exists("{}/bin/activate".format(venv_path)): venv = ". {}/bin/activate && ".format(venv_path) else: venv = ". {}/bin/activate && ".format(sys.prefix) else: venv = "" print(" * Update pip...({})".format(venv)) s, r = subprocess.getstatusoutput("{}pip3 install -U pip".format(venv)) print(r) s, r = subprocess.getstatusoutput("{}pip3 freeze".format(venv)) old_reqs = r.split() with open("{}/requirements.txt".format(PROJECT_PATH)) as rf: new_reqs = rf.read().split() # 查找需要安装的包 ret_list = list(set(new_reqs) ^ set(old_reqs)) install_list = [] for ret in ret_list: if (ret not in old_reqs) and (ret in new_reqs): install_list.append(ret) if install_list: msg = " * To install the following libs" print(msg) print(install_list) if not input_venv_path: web_start_log.info(msg) web_start_log.info(install_list) pass install_failed = [] for sf in install_list: if latest: sf = sf.split("==")[0] print("pip install -U {}".format(sf)) s, r = subprocess.getstatusoutput( "{}pip install -U {}".format(venv, sf)) if s: install_failed.append(sf) for sf in install_failed: if latest: sf = sf.split("==")[0] s, r = subprocess.getstatusoutput( "{}pip install -U {}".format(venv, sf)) if not s: install_failed.remove(sf) if install_failed: msg = " * Installation failed library, please manually install" print(msg) print(install_failed) web_start_log.info(msg) web_start_log.info(install_failed) # 查找需要卸载的包 s, r = subprocess.getstatusoutput("{}pip freeze".format(venv)) old_reqs = r.split() ret_list = list(set(new_reqs) ^ set(old_reqs)) uninstall_list = [] for ret in ret_list: if (ret in old_reqs) and (ret not in new_reqs): uninstall_list.append(ret) for sf in uninstall_list[:]: if "==" not in sf: uninstall_list.remove(sf) if uninstall_list: msg = " * Now don't need python library:" print(msg) print(uninstall_list) if not input_venv_path: web_start_log.info(msg) web_start_log.info(uninstall_list) if latest: subprocess.getstatusoutput( "{}pip freeze > {}/requirements.txt".format(venv, PROJECT_PATH))
def init_core_module(app, **kwargs): """ 初始化核心模块 :param app: :return: """ csrf_enabled = kwargs.get("csrf_enabled") is_debug = kwargs.get("is_debug") # app config web_start_log.info("Initialize the core module") # 系统必要配置, 优先导入 app.config.from_object(ConfDictToClass(CONFIG["system"], key="value")) app.config.from_object(ConfDictToClass(CONFIG["key"], key="value")) # 数据库 app.config.from_object(DatabaseConfig()) for name, mdb in mdbs.items(): mdb.init_app(app, config_prefix=name.upper()) # 缓存 app.config.from_object(ConfDictToClass(CONFIG["cache"], key="value")) app.config["CACHE_REDIS"] = redis app.config["CACHE_MONGODB_DBS"] = mdbs["sys"].dbs cache.init_app(app) # Clear CONFIG cache if not is_debug: version_info = mdbs["sys"].db.sys_config.find_one( {"new_version": { "$exists": True }}) ago_time = time.time() - 3600 * 24 ago_time_30m = time.time() - 1800 if version_info["sys_version_of_config"] >= SYS_CONFIG_VERSION \ and version_info["update_time"] > ago_time \ and version_info["update_time"]<ago_time_30m: # 系统正在使用的SYS_CONFIG_VERSION版本和当前机器CONFIG的一样,或更高 # And: 配置24小时内已有更新 # So: 这次不更新 msg = " * [sys configs cache] Not clean cache." \ " The system is using the same or higher configuration version.\n" \ " And it was executed within 24 hours." start_info_print("\033[33m{}\033[0m".format(msg)) web_start_log.warning(msg) else: with app.app_context(): msg = " * Clean configuration cache successfully" cache.delete(CONFIG_CACHE_KEY) cache.delete(PLUG_IN_CONFIG_CACHE_KEY) web_start_log.info(msg) start_info_print(msg) # 异常错误信息 app.config["PRESERVE_CONTEXT_ON_EXCEPTION"] = PRESERVE_CONTEXT_ON_EXCEPTION ################################################### # 在此之前, 任何程序不能调用utils.get_config.py下的方法 ################################################### from apps.core.utils.get_config import get_configs, get_config from apps.core.flask.request import OsrRequestProcess from apps.core.flask.errorhandler import ErrorHandler from apps.core.blueprint import api, admin_view, theme_view, static_html_view, static, open_api from apps.core.flask.routing import RegexConverter from apps.core.flask.routing import push_url_to_db # 最大请求大小 app.config["MAX_CONTENT_LENGTH"] = get_config( "system", "MAX_CONTENT_LENGTH") * 1024 * 1024 # Session会话配置 session_config = get_configs("session") session_config["SESSION_PROTECTION"] = SESSION_PROTECTION session_config["SESSION_COOKIE_PATH"] = SESSION_COOKIE_PATH session_config["SESSION_COOKIE_HTTPONLY"] = SESSION_COOKIE_HTTPONLY session_config["SESSION_COOKIE_SECURE"] = SESSION_COOKIE_SECURE session_config["SESSION_USE_SIGNER"] = SESSION_USE_SIGNER session_config["SESSION_MONGODB_DB"] = mdbs["sys"].name app.config.from_object(ConfDictToClass(session_config)) app.config["SESSION_REDIS"] = redis app.config["SESSION_MONGODB"] = mdbs["sys"].connection sess.init_app(app) rest_session.init_app(app) # 邮件 app.config.from_object(ConfDictToClass(get_configs("email"))) mail.init_app(app) # Csrf token csrf_config = {} if csrf_enabled: csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = True start_info_print(" * Security authentication is turned on") else: csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = False start_info_print( "\033[31m WARNING: security verification is turned off\033[0m") # 这两个csrf参数这里关闭,request程序会根据CLIENT_TOKEN_AUTH_ENABLED判断处理 csrf_config["WTF_CSRF_CHECK_DEFAULT"] = False csrf_config["CSRF_ENABLED"] = False csrf_config["WTF_CSRF_METHODS"] = WTF_CSRF_METHODS app.config.from_object(ConfDictToClass(csrf_config)) csrf.init_app(app) # Babel app.config.from_object(ConfDictToClass(get_configs("babel"))) app.config["BABEL_TRANSLATION_DIRECTORIES"] = BABEL_TRANSLATION_DIRECTORIES babel.init_app(app) # 登录管理 login_manager.init_app(app) # login_manager.anonymous_user = AnonymousUser() login_manager.session_protection = SESSION_PROTECTION oauth.init_app(app) # 让路由支持正则 app.url_map.converters['regex'] = RegexConverter # 注册蓝图 blueprint web_start_log.info("Register blueprint, Initialize the routing") app.register_blueprint(api) app.register_blueprint(open_api) app.register_blueprint(admin_view) app.register_blueprint(theme_view) app.register_blueprint(static_html_view) app.register_blueprint(static) if not is_debug: st = time.time() push_url_to_db(app) start_info_print( " * Routing updates saved in the database. It tasks time {} sec". format(int(time.time() - st))) celery.conf.update(app.config) # 请求处理 request_process = OsrRequestProcess() request_process.init_request_process(app=app) request_process.init_babel_locale_selector(babel=babel) # 错误处理 ErrorHandler(app) # Logger # Other log_udp = LogServerUDP() r = log_udp.init_app() if r: log_udp.log_server() weblog = WebLogger() weblog.init_app(app)
def push_to_db(mdb_sys, local_config=None, now_version=None): ''' 初始化或者更新配置数据到数据库, 顺便更新CONFIG变量的值 config:如果为真, 则使用传入的这个配置 cover: 是否覆盖数据库中的数据 :return: ''' if not now_version: return False web_start_log.info("Push to the config version:{}".format(now_version)) for k, v in local_config.items(): if not isinstance(v, dict): continue for k1, v1 in v.items(): if k1.startswith("__"): continue try: if "__restart__" in v: conf = v1 conf["update_time"] = time.time() conf["__restart__"] = v["__restart__"] if "__info__" in v: conf["__info__"] = v["__info__"] if "__sort__" in v: conf["__sort__"] = v["__sort__"] r = mdb_sys.db.sys_config.update_one( { "project": k, "key": k1, "conf_version": now_version }, {"$set": conf}, upsert=True) if r.modified_count or r.upserted_id: web_start_log.info( "Config updates: [file to db] {} {}".format(k, k1)) elif r.matched_count: web_start_log.info( "[file to db] Not updated:{} {}".format(k, k1)) else: web_start_log.warning( "[file to db] Update failure:{} {}".format(k, k1)) else: web_start_log.info("[Not import] {} {}".format(k, k1)) except Exception as e: web_start_log.error("[file to db] {}, {} {}".format(e, k, k1)) # 更新版本信息 # 将当前版本纳入used_versions, 以便之后管理端修改配置时自动生成新版本 mdb_sys.db.sys_config.update_one( { "new_version": { "$exists": True }, "used_versions": { "$ne": now_version } }, {"$addToSet": { "used_versions": now_version }}) # 更新主机信息 host_version = mdb_sys.db.sys_host.find_one({ "type": "web", "host_info.local_ip": host_info["local_ip"] }) up_version = {"conf_version": now_version, "switch_conf_version": None} if host_version: up_version["host_info"] = host_version["host_info"] else: up_version["disable_update_conf"] = 0 up_version["host_info"] = {} for k, v in host_info.items(): up_version["host_info"][k] = v if not host_version or not host_version["disable_update_conf"]: up_version["start_log"] = "Normal" web_start_log.info(up_version["start_log"]) else: up_version[ "start_log"] = "Update failed. Because there is no open to update" web_start_log.warning(up_version["start_log"]) # 更新主机信息 mdb_sys.db.sys_host.delete_many({ "$or": [{ "host_info.local_ip": { "$exists": False } }, { "host_info.local_ip": { "$in": [None, "", False] } }] }) mdb_sys.db.sys_host.update_one( { "type": "web", "host_info.local_ip": host_info["local_ip"] }, {"$set": up_version}, upsert=True) return True
def update_config_file(mdb_sys, *args, **kwargs): ''' 网站启动的时候, 数据库保存的配置-配置文件 同步更新 config mdb_sys redis ''' local_config = deepcopy(CONFIG) overwrite_db = OVERWRITE_DB version_info = mdb_sys.db.sys_config.find_one( {"new_version": { "$exists": True }}) if not version_info: now_version = time_to_utcdate(time_stamp=now_time, tformat="%Y_%m_%d_%H_%M_%S") version_uses = { "new_version": now_version, "used_versions": [now_version], "update_time": now_time } mdb_sys.db.sys_config.insert_one(version_uses) web_start_log.info("Initialize the sys_config version info") else: now_version = version_info["new_version"] if version_info and not overwrite_db: # 查询当前主机web的的配置版本 cur_h_version_info = mdb_sys.db.sys_host.find_one({ "type": "web", "host_info.local_ip": host_info["local_ip"] }) if not cur_h_version_info: cur_h_version_info = { "type": "web", "host_info": host_info, "conf_version": version_info["new_version"], "switch_conf_version": None, "disable_update_conf": 0 } mdb_sys.db.sys_host.insert_one(cur_h_version_info) web_start_log.info("Initialize the host version data") if cur_h_version_info["switch_conf_version"] or cur_h_version_info[ "disable_update_conf"]: # 数据库配置和本地配置合并:保留本地和数据库的key,用于网站版本回滚 if cur_h_version_info[ "switch_conf_version"] and not cur_h_version_info[ "disable_update_conf"]: # 版本切换 now_version = cur_h_version_info["switch_conf_version"] else: # 禁止更新 now_version = cur_h_version_info["conf_version"] confs = mdb_sys.db.sys_config.find({"conf_version": now_version}) if confs.count(True): for conf in confs: if not re.search(r"^__.*__$", conf["key"]): if not conf["project"] in local_config: local_config[conf["project"]] = {} if not conf["key"] in local_config[conf["project"]]: local_config[conf["project"]][conf["key"]] = { "value": conf["value"], "type": conf["type"], "info": conf["info"] } else: local_config[conf["project"]][ conf["key"]]["value"] = conf["value"] web_start_log.info( "Config rollback:[db to file] Update version info") else: web_start_log.error( "Config rollback:[db to file] Rollback failure") return False else: # 数据库最新版配置和本地配置合并:以本地配置的key为准,多余的删除 now_version = version_info["new_version"] confs = mdb_sys.db.sys_config.find({"conf_version": now_version}) if confs.count(True): for conf in confs: is_cft = re.search(r"^__.*__$", conf["key"]) if not is_cft and conf["project"] in local_config.keys( ) and conf["key"] in local_config[conf["project"]].keys(): local_config[conf["project"]][ conf["key"]]["value"] = conf["value"] web_start_log.info( "Config merge:[db to file] {} {}".format( conf["project"], conf["key"])) else: mdb_sys.db.sys_config.delete_one({"_id": conf["_id"]}) web_start_log.info("Remove the config:{} {}".format( conf["project"], conf["key"])) else: web_start_log.error("Config merge:[db to file] Merger failure") return False else: web_start_log.info( "**Local configuration directly covering the latest edition of the configuration database" ) r = push_to_db(mdb_sys, local_config=deepcopy(local_config), now_version=now_version) if not r: web_start_log.error("Config update:[file to db] Push failure") return False # 写配置文件 info = '''#!/usr/bin/env python\n# -*-coding:utf-8-*-\n__author__ = "Allen Woo"\n''' doc = "__readme__='''{}'''\n".format(__readme__) # write config.py temp_conf = str(json.dumps(local_config, indent=4, ensure_ascii=False)) wf = open("{}/apps/configs/config.py".format(PROJECT_PATH), "wb") wf.write(bytes(info, "utf-8")) wf.write(bytes(doc, "utf-8")) # 修改配置为同步数据库配置到文件 wf.write( bytes( "# Danger: If True, the database configuration data will be overwritten\n", "utf-8")) wf.write(bytes("# 危险:如果为True, 则会把该文件配置覆盖掉数据库中保存的配置\n", "utf-8")) wf.write(bytes("OVERWRITE_DB = False\n", "utf-8")) wf.write(bytes("CONFIG = ", "utf-8")) wf.write( bytes( temp_conf.replace("false", "False").replace("true", "True").replace("null", "None"), "utf-8")) wf.close() web_start_log.info("Configuration updates and merge is complete") return True
def init_core_module(app): """ 初始化核心模块 :param app: :return: """ # app config web_start_log.info("Initialize the core module") # 系统必要配置, 优先导入 app.config.from_object(ConfDictToClass(CONFIG["system"], key="value")) app.config.from_object(ConfDictToClass(CONFIG["key"], key="value")) # 数据库 app.config.from_object(DatabaseConfig()) for name, mdb in mdbs.items(): mdb.init_app(app, config_prefix=name.upper()) # 缓存 app.config.from_object(ConfDictToClass(CONFIG["cache"], key="value")) app.config["CACHE_REDIS"] = redis app.config["CACHE_MONGODB"] = mdbs["sys"].connection app.config["CACHE_MONGODB_DB"] = mdbs["sys"].name cache.init_app(app) # 清除配置CONFIG的cache with app.app_context(): msg = " * Clean configuration cache successfully" cache.delete(CONFIG_CACHE_KEY) cache.delete(PLUG_IN_CONFIG_CACHE_KEY) web_start_log.info(msg) print(msg) # 异常错误信息 app.config["PRESERVE_CONTEXT_ON_EXCEPTION"] = PRESERVE_CONTEXT_ON_EXCEPTION ################################################### # 在此之前, 任何程序不能调用utils.get_config.py下的方法 ################################################### from apps.core.utils.get_config import get_configs, get_config from apps.core.flask.request import OsrRequestProcess from apps.core.flask.errorhandler import ErrorHandler from apps.core.blueprint import api, admin_view, theme_view, static_html_view, static, open_api from apps.core.flask.routing import RegexConverter from apps.core.flask.routing import push_url_to_db # 最大请求大小 app.config["MAX_CONTENT_LENGTH"] = get_config( "system", "MAX_CONTENT_LENGTH") * 1024 * 1024 # Session会话配置 session_config = get_configs("session") session_config["SESSION_PROTECTION"] = SESSION_PROTECTION session_config["SESSION_COOKIE_PATH"] = SESSION_COOKIE_PATH session_config["SESSION_COOKIE_HTTPONLY"] = SESSION_COOKIE_HTTPONLY session_config["SESSION_COOKIE_SECURE"] = SESSION_COOKIE_SECURE session_config["SESSION_USE_SIGNER"] = SESSION_USE_SIGNER session_config["SESSION_MONGODB_DB"] = mdbs["sys"].name app.config.from_object(ConfDictToClass(session_config)) app.config["SESSION_REDIS"] = redis app.config["SESSION_MONGODB"] = mdbs["sys"].connection sess.init_app(app) rest_session.init_app(app) # 邮件 app.config.from_object(ConfDictToClass(get_configs("email"))) mail.init_app(app) # Csrf token csrf_config = {} csrf_config["CSRF_ENABLED"] = CSRF_ENABLED csrf_config["WTF_CSRF_CHECK_DEFAULT"] = WTF_CSRF_CHECK_DEFAULT csrf_config["WTF_CSRF_METHODS"] = WTF_CSRF_METHODS app.config.from_object(ConfDictToClass(csrf_config)) csrf.init_app(app) # Babel app.config.from_object(ConfDictToClass(get_configs("babel"))) app.config["BABEL_TRANSLATION_DIRECTORIES"] = BABEL_TRANSLATION_DIRECTORIES babel.init_app(app) # 登录管理 login_manager.init_app(app) # login_manager.anonymous_user = AnonymousUser() login_manager.session_protection = SESSION_PROTECTION oauth.init_app(app) # 让路由支持正则 app.url_map.converters['regex'] = RegexConverter # 注册蓝图 blueprint web_start_log.info("Register blueprint, Initialize the routing") app.register_blueprint(api) app.register_blueprint(open_api) app.register_blueprint(admin_view) app.register_blueprint(theme_view) app.register_blueprint(static_html_view) app.register_blueprint(static) push_url_to_db(app) # 请求处理 request_process = OsrRequestProcess() request_process.init_request_process(app=app) request_process.init_babel_locale_selector(babel=babel) # 错误处理 ErrorHandler(app) # Logger # Other log_udp = LogServerUDP() r = log_udp.init_app() if r: log_udp.log_server() weblog = WebLogger() weblog.init_app(app)
from apps.core.db.mongodb import PyMongo from apps.core.logger.web_logging import WebLogger, web_start_log from flask_babel import Babel from flask_mail import Mail from flask_oauthlib.client import OAuth from flask_session import Session from flask_wtf import CSRFProtect from flask_login import LoginManager from redis import StrictRedis from apps.configs.db_config import DB_CONFIG ''' Flask app 与其他核心模块实例化 注意: 不要将模块初始话设置放在本文件 ''' # 主程序 web_start_log.info("Initialize the OsrApp") app = OsrApp(__name__) web_start_log.info("Instance of each module") mdb_web = PyMongo() mdb_sys = PyMongo() mdb_user = PyMongo() cache = Cache() babel = Babel() csrf = CSRFProtect() login_manager = LoginManager() sess = Session() rest_session = RestSession() mail = Mail() weblog = WebLogger() oauth = OAuth() redis = StrictRedis(host=DB_CONFIG["redis"]["host"][0],
def update_config_file(mdbs, *args, **kwargs): """ 网站启动的时候, 数据库保存的配置-配置文件 同步更新 config mdb_sys redis """ local_config = deepcopy(CONFIG) overwrite_db = OVERWRITE_DB version_info = mdbs["sys"].db.sys_config.find_one( {"new_version": { "$exists": True }}) if not version_info: now_version = time_to_utcdate(time_stamp=now_time, tformat="%Y_%m_%d_%H_%M_%S") version_uses = { "new_version": now_version, "used_versions": [now_version], "update_time": now_time } mdbs["sys"].db.sys_config.insert_one(version_uses) sys_version_of_config = SYS_CONFIG_VERSION last_update_time = now_time web_start_log.info("Initialize the sys_config version info") else: now_version = version_info["new_version"] last_update_time = version_info["update_time"] if "sys_version_of_config" in version_info: sys_version_of_config = version_info["sys_version_of_config"] else: sys_version_of_config = SYS_CONFIG_VERSION mdbs["sys"].db.sys_config.update_one( {"new_version": { "$exists": True }}, {"$set": { "sys_version_of_config": SYS_CONFIG_VERSION }}) if version_info and not overwrite_db: # 查询当前主机web的的配置版本 cur_h_version_info = mdbs["sys"].db.sys_host.find_one({ "type": "web", "host_info.local_ip": host_info["local_ip"] }) if not cur_h_version_info: cur_h_version_info = { "type": "web", "host_info": host_info, "conf_version": version_info["new_version"], "switch_conf_version": None, "disable_update_conf": 0 } mdbs["sys"].db.sys_host.insert_one(cur_h_version_info) web_start_log.info("Initialize the host version data") if cur_h_version_info["switch_conf_version"] or cur_h_version_info[ "disable_update_conf"]: # 数据库配置和本地配置合并:保留本地和数据库的key,用于网站版本回滚 if cur_h_version_info[ "switch_conf_version"] and not cur_h_version_info[ "disable_update_conf"]: # 版本切换 now_version = cur_h_version_info["switch_conf_version"] else: # 禁止更新 now_version = cur_h_version_info["conf_version"] confs = mdbs["sys"].db.sys_config.find( {"conf_version": now_version}) if confs.count(True): for conf in confs: if not re.search(r"^__.*__$", conf["key"]): if not conf["project"] in local_config: local_config[conf["project"]] = {} if not conf["key"] in local_config[conf["project"]]: local_config[conf["project"]][conf["key"]] = { "value": conf["value"], "type": conf["type"], "info": conf["info"] } else: local_config[conf["project"]][ conf["key"]]["value"] = conf["value"] web_start_log.info( "Config rollback:[db to file] Update version info") else: web_start_log.error( "Config rollback:[db to file] Rollback failure") return False else: # 数据库最新版配置和本地配置合并:以本地配置的key为准,多余的删除 # 数据库最新版配置和本地配置合并:以本地配置的key为准,多余的删除 ago_time = now_time - 3600 * 24 if sys_version_of_config >= SYS_CONFIG_VERSION and last_update_time > ago_time: # 系统正在使用的SYS_CONFIG_VERSION版本和当前机器CONFIG的一样,或更高 # And: 配置24小时内已有更新 # So: 这次不更新 msg = " * [sys configs] Not updated. The system is using the same or higher configuration version.\n" \ " And it was executed within 24 hours.\n" start_info_print("\033[33m{}\033[0m".format(msg)) return True now_version = version_info["new_version"] confs = mdbs["sys"].db.sys_config.find( {"conf_version": now_version}) if confs.count(True): for conf in confs: is_cft = re.search(r"^__.*__$", conf["key"]) if not is_cft and conf["project"] in local_config.keys() \ and conf["key"] in local_config[conf["project"]].keys(): local_config[conf["project"]][ conf["key"]]["value"] = conf["value"] web_start_log.info( "Config merge:[db to file] {} {}".format( conf["project"], conf["key"])) else: mdbs["sys"].db.sys_config.delete_one( {"_id": conf["_id"]}) web_start_log.info("Remove the config:{} {}".format( conf["project"], conf["key"])) else: web_start_log.error("Config merge:[db to file] Merger failure") return False else: web_start_log.info( "**Local configuration directly covering the latest edition of the configuration database" ) r = push_to_db(mdbs, local_config=deepcopy(local_config), now_version=now_version) if not r: web_start_log.error("Config update:[file to db] Push failure") return False # 写配置文件 # info = """#!/usr/bin/env python\n# -*-coding:utf-8-*-\n__author__ = "Allen Woo"\n""" # doc = "__readme__= '''{}'''\n".format(__readme__) # # # write config.py # # # 把password类型替换成*, 再写入文件,防止提交代码时把密码上传到git # for k, v in local_config.items(): # for k1, v1 in v.items(): # if k1.startswith("__") and k1.endswith("__"): # continue # if "type" in v1 and v1["type"] == "password": # # 由于上一个版的password已被替换,现在需要把把它写写入到CONFIG中 # CONFIG[k][k1]["value"] = v1["value"] # # 替换密码 # v1["value"] = "<Your password>" # # temp_conf = str(json.dumps(local_config, indent=4, ensure_ascii=False)) # wf = open("{}/apps/configs/config.py".format(PROJECT_PATH), "wb") # wf.write(bytes(info, "utf-8")) # wf.write(bytes(doc, "utf-8")) # # 修改配置为同步数据库配置到文件 # wf.write(bytes("# Danger: If True, the database configuration data will be overwritten\n", "utf-8")) # wf.write(bytes("# 危险:如果为True, 则会把该文件配置覆盖掉数据库中保存的配置\n", "utf-8")) # wf.write(bytes("OVERWRITE_DB = False\n", "utf-8")) # wf.write(bytes("CONFIG = ", "utf-8")) # wf.write(bytes(temp_conf.replace("false", # "False").replace("true", # "True").replace("null", # "None"), # "utf-8")) # wf.close() web_start_log.info("Configuration updates and merge is complete") return True