def update_mdb_collections(mdbs): """ 更新数据库mongodb collection, 不存在的colletion则创建 :param mdbs: :return: """ # 读取配置中的数据库json 数据 coll_json_file = "{}/configs/mdb_collections.json".format(APPS_PATH) if not os.path.exists(coll_json_file): return with open(coll_json_file) as rf: jsondata = rf.read() if jsondata: collections = json.loads(jsondata) else: collections = {} # 检查数据库collections for dbname, colls in collections.items(): mdb = mdbs[dbname] for coll in colls: try: mdb.dbs.create_collection(coll) web_start_log.info( "[DB: {}] Create collection '{}'".format( mdb.name, coll)) except Exception as e: if "already exists" in str(e): web_start_log.info(e) else: web_start_log.warning(e)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user): ''' 更新数据库mongodb collection, 不存在的colletion则创建 :param mdb_sys: :param mdb_web: :param mdb_user: :return: ''' # 读取配置中的数据库json 数据 with open("{}/configs/collections.json".format(APPS_PATH)) as rf: jsondata = rf.read() if jsondata: collections = json.loads(jsondata) else: collections = {} dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web} # 检查数据库collections for dbname, colls in collections.items(): mdb = dbs[dbname] for coll in colls: try: mdb.dbs.create_collection(coll) web_start_log.info("[DB: {}] Create collection '{}'".format( mdb.name, coll)) except Exception as e: if "already exists" in str(e): web_start_log.info(e) else: web_start_log.warning(e) # 将最新的数据库结构写入配置文件, 方便开发者了解结构 new_collections = {} for dbname, mdb in dbs.items(): new_collections[dbname] = {} collnames = mdb.dbs.collection_names() for collname in collnames: if collname == "system.indexes" or collname.startswith("plug_"): continue new_collections[dbname][collname] = {} data = mdb.dbs[collname].find_one({}, {"_id": 0}) if data: for k, v in data.items(): new_collections[dbname][collname][k] = str(type(v)) with open("{}/configs/collections.json".format(APPS_PATH), "w") as wf: collections = json.dumps(new_collections, indent=4, ensure_ascii=False) wf.write(collections)
def update_mdb_collections(mdb_sys, mdb_web, mdb_user): ''' 更新数据库mongodb collection, 不存在的colletion则创建 :param mdb_sys: :param mdb_web: :param mdb_user: :return: ''' dbs = {"mdb_sys": mdb_sys, "mdb_user": mdb_user, "mdb_web": mdb_web} for k, colls in collections.items(): mdb = dbs[k] for coll in colls: try: mdb.dbs.create_collection(coll) web_start_log.info("[DB: {}] Create collection '{}'".format( mdb.name, coll)) except Exception as e: if "already exists" in str(e): web_start_log.info(e) else: web_start_log.warning(e)
def init_core_module(app, **kwargs): """ 初始化核心模块 :param app: :return: """ csrf_enabled = kwargs.get("csrf_enabled") is_debug = kwargs.get("is_debug") # app config web_start_log.info("Initialize the core module") # 系统必要配置, 优先导入 app.config.from_object(ConfDictToClass(CONFIG["system"], key="value")) app.config.from_object(ConfDictToClass(CONFIG["key"], key="value")) # 数据库 app.config.from_object(DatabaseConfig()) for name, mdb in mdbs.items(): mdb.init_app(app, config_prefix=name.upper()) # 缓存 app.config.from_object(ConfDictToClass(CONFIG["cache"], key="value")) app.config["CACHE_REDIS"] = redis app.config["CACHE_MONGODB_DBS"] = mdbs["sys"].dbs cache.init_app(app) # Clear CONFIG cache if not is_debug: version_info = mdbs["sys"].db.sys_config.find_one( {"new_version": { "$exists": True }}) ago_time = time.time() - 3600 * 24 ago_time_30m = time.time() - 1800 if version_info["sys_version_of_config"] >= SYS_CONFIG_VERSION \ and version_info["update_time"] > ago_time \ and version_info["update_time"]<ago_time_30m: # 系统正在使用的SYS_CONFIG_VERSION版本和当前机器CONFIG的一样,或更高 # And: 配置24小时内已有更新 # So: 这次不更新 msg = " * [sys configs cache] Not clean cache." \ " The system is using the same or higher configuration version.\n" \ " And it was executed within 24 hours." start_info_print("\033[33m{}\033[0m".format(msg)) web_start_log.warning(msg) else: with app.app_context(): msg = " * Clean configuration cache successfully" cache.delete(CONFIG_CACHE_KEY) cache.delete(PLUG_IN_CONFIG_CACHE_KEY) web_start_log.info(msg) start_info_print(msg) # 异常错误信息 app.config["PRESERVE_CONTEXT_ON_EXCEPTION"] = PRESERVE_CONTEXT_ON_EXCEPTION ################################################### # 在此之前, 任何程序不能调用utils.get_config.py下的方法 ################################################### from apps.core.utils.get_config import get_configs, get_config from apps.core.flask.request import OsrRequestProcess from apps.core.flask.errorhandler import ErrorHandler from apps.core.blueprint import api, admin_view, theme_view, static_html_view, static, open_api from apps.core.flask.routing import RegexConverter from apps.core.flask.routing import push_url_to_db # 最大请求大小 app.config["MAX_CONTENT_LENGTH"] = get_config( "system", "MAX_CONTENT_LENGTH") * 1024 * 1024 # Session会话配置 session_config = get_configs("session") session_config["SESSION_PROTECTION"] = SESSION_PROTECTION session_config["SESSION_COOKIE_PATH"] = SESSION_COOKIE_PATH session_config["SESSION_COOKIE_HTTPONLY"] = SESSION_COOKIE_HTTPONLY session_config["SESSION_COOKIE_SECURE"] = SESSION_COOKIE_SECURE session_config["SESSION_USE_SIGNER"] = SESSION_USE_SIGNER session_config["SESSION_MONGODB_DB"] = mdbs["sys"].name app.config.from_object(ConfDictToClass(session_config)) app.config["SESSION_REDIS"] = redis app.config["SESSION_MONGODB"] = mdbs["sys"].connection sess.init_app(app) rest_session.init_app(app) # 邮件 app.config.from_object(ConfDictToClass(get_configs("email"))) mail.init_app(app) # Csrf token csrf_config = {} if csrf_enabled: csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = True start_info_print(" * Security authentication is turned on") else: csrf_config["CLIENT_TOKEN_AUTH_ENABLED"] = False start_info_print( "\033[31m WARNING: security verification is turned off\033[0m") # 这两个csrf参数这里关闭,request程序会根据CLIENT_TOKEN_AUTH_ENABLED判断处理 csrf_config["WTF_CSRF_CHECK_DEFAULT"] = False csrf_config["CSRF_ENABLED"] = False csrf_config["WTF_CSRF_METHODS"] = WTF_CSRF_METHODS app.config.from_object(ConfDictToClass(csrf_config)) csrf.init_app(app) # Babel app.config.from_object(ConfDictToClass(get_configs("babel"))) app.config["BABEL_TRANSLATION_DIRECTORIES"] = BABEL_TRANSLATION_DIRECTORIES babel.init_app(app) # 登录管理 login_manager.init_app(app) # login_manager.anonymous_user = AnonymousUser() login_manager.session_protection = SESSION_PROTECTION oauth.init_app(app) # 让路由支持正则 app.url_map.converters['regex'] = RegexConverter # 注册蓝图 blueprint web_start_log.info("Register blueprint, Initialize the routing") app.register_blueprint(api) app.register_blueprint(open_api) app.register_blueprint(admin_view) app.register_blueprint(theme_view) app.register_blueprint(static_html_view) app.register_blueprint(static) if not is_debug: st = time.time() push_url_to_db(app) start_info_print( " * Routing updates saved in the database. It tasks time {} sec". format(int(time.time() - st))) celery.conf.update(app.config) # 请求处理 request_process = OsrRequestProcess() request_process.init_request_process(app=app) request_process.init_babel_locale_selector(babel=babel) # 错误处理 ErrorHandler(app) # Logger # Other log_udp = LogServerUDP() r = log_udp.init_app() if r: log_udp.log_server() weblog = WebLogger() weblog.init_app(app)
def push_to_db(mdb_sys, local_config=None, now_version=None): ''' 初始化或者更新配置数据到数据库, 顺便更新CONFIG变量的值 config:如果为真, 则使用传入的这个配置 cover: 是否覆盖数据库中的数据 :return: ''' if not now_version: return False web_start_log.info("Push to the config version:{}".format(now_version)) for k, v in local_config.items(): if not isinstance(v, dict): continue for k1, v1 in v.items(): if k1.startswith("__"): continue try: if "__restart__" in v: conf = v1 conf["update_time"] = time.time() conf["__restart__"] = v["__restart__"] if "__info__" in v: conf["__info__"] = v["__info__"] if "__sort__" in v: conf["__sort__"] = v["__sort__"] r = mdb_sys.db.sys_config.update_one( { "project": k, "key": k1, "conf_version": now_version }, {"$set": conf}, upsert=True) if r.modified_count or r.upserted_id: web_start_log.info( "Config updates: [file to db] {} {}".format(k, k1)) elif r.matched_count: web_start_log.info( "[file to db] Not updated:{} {}".format(k, k1)) else: web_start_log.warning( "[file to db] Update failure:{} {}".format(k, k1)) else: web_start_log.info("[Not import] {} {}".format(k, k1)) except Exception as e: web_start_log.error("[file to db] {}, {} {}".format(e, k, k1)) # 更新版本信息 # 将当前版本纳入used_versions, 以便之后管理端修改配置时自动生成新版本 mdb_sys.db.sys_config.update_one( { "new_version": { "$exists": True }, "used_versions": { "$ne": now_version } }, {"$addToSet": { "used_versions": now_version }}) # 更新主机信息 host_version = mdb_sys.db.sys_host.find_one({ "type": "web", "host_info.local_ip": host_info["local_ip"] }) up_version = {"conf_version": now_version, "switch_conf_version": None} if host_version: up_version["host_info"] = host_version["host_info"] else: up_version["disable_update_conf"] = 0 up_version["host_info"] = {} for k, v in host_info.items(): up_version["host_info"][k] = v if not host_version or not host_version["disable_update_conf"]: up_version["start_log"] = "Normal" web_start_log.info(up_version["start_log"]) else: up_version[ "start_log"] = "Update failed. Because there is no open to update" web_start_log.warning(up_version["start_log"]) # 更新主机信息 mdb_sys.db.sys_host.delete_many({ "$or": [{ "host_info.local_ip": { "$exists": False } }, { "host_info.local_ip": { "$in": [None, "", False] } }] }) mdb_sys.db.sys_host.update_one( { "type": "web", "host_info.local_ip": host_info["local_ip"] }, {"$set": up_version}, upsert=True) return True