def check_php_tag_package_standard(svn_tag_package_list_item, check_php_tag_package_path, forbid_dir_list): """ :param svn_tag_package_list_item: :param check_php_tag_package_path: :param forbid_dir_list: :return: """ last_index = check_php_tag_package_path.rfind(".") base_check_dir_path = check_php_tag_package_path[0:last_index] trunk_path = os.path.join(base_check_dir_path, "trunk") if not os.path.exists(trunk_path): raise MyServiceException("检查: " + svn_tag_package_list_item + " 时发现文件不符合规范") if forbid_dir_list: forbid_dir_lists = forbid_dir_list.split(";") for item in forbid_dir_lists: if not item or item == "": continue forbid_path = os.path.join(trunk_path, item) if os.path.exists(forbid_path): raise MyServiceException("检查: " + svn_tag_package_list_item + " 时发现文件不符合规范,文件存在禁止的目录(%s)" % forbid_path) # 禁止目录中是否包含通配符 if forbid_path.find("*") > 0: if len(glob.glob(forbid_path)) > 0: raise MyServiceException("检查: " + svn_tag_package_list_item + " 时发现文件不符合规范,文件存在禁止的目录(%s)" % forbid_path)
def check_php_url_tag_package(url, svn_server_host): url_prefix = url[0:len(svn_server_host)] url_suffix_zip = url[len(url)-len(".zip"):] url_suffix_rar = url[len(url)-len(".rar"):] if url_prefix != svn_server_host: raise MyServiceException("检查: " + url + " 时发现文件地址不为允许的svn服务器地址") if url_suffix_zip != ".zip" or url_suffix_rar == ".rar": raise MyServiceException("检查: " + url + " 文件后缀不为允许的后缀(.zip/.rar)")
def do_sql_invoke(): try: request_data = common_service.check_request_dat_not_null([ "title", "env", "host_port", "database", "sql_content", "apply_process", "process_template_id", "action_type" ]) title = request_data["title"] env = request_data["env"] host_port = request_data["host_port"] database = request_data["database"] sql_content = request_data["sql_content"] apply_process = request_data["apply_process"] process_template_id = request_data["process_template_id"] action_type = request_data["action_type"] if "TRUNCATE" in sql_content: raise MyServiceException("禁止执行TRUNCATE语法") if action_type != DANGER_SQL_STR: # 检测是否为危险SQL sql_content_list = split_sql_content(sql_content) for sql_content_item in sql_content_list: for forbid_sql_type_item in forbid_sql_type_list: if sql_content_item.upper().startswith( forbid_sql_type_item): raise MyServiceException("不支持的SQL类型: %s" % forbid_sql_type_item) # 检测SQL是否能够执行 check_sql_executable(env, host_port, database, sql_content) # 流程 finish_result = { "method": "POST", "url": "http://localhost:8080/project_manage/associate_db/sql_work_order/finish_sql_invoke", "rb_url": "http://localhost:8080/project_manage/associate_db/sql_work_order/rollback_sql_invoke", } display_content = "执行环境为: %s \n MySQL实例为: %s \n 数据库为: %s \n 动作类型为: %s \n SQL内容为: \n %s \n\n\n " % ( env, host_port, database, action_type, sql_content) process_id, next_step = run_manage.process_start( apply_process, process_type, finish_result, title, display_content, env, process_template_id) # 存储内容等待流程完成时去执行 sql_content = sql_content.replace("`", "") project_manage__work_order_process__run_manage_process_sql_content_co.insert_one( { "process_id": process_id, "env": env, "host_port": host_port, "database": database, "action_type": action_type, "sql_content": sql_content, }) run_manage.process_after_trigger(process_id, next_step) return {} except MyServiceException as e: return ResResult.return500(str(e))
def check_prepare_work_order_list_status(prepare_work_order_list): prepare_work_order_lists = parse_prepare_work_order_list( prepare_work_order_list) for item in prepare_work_order_lists: try: db_res_data = json.loads( dumps( project_manage__work_order_process__run_manage_co.find_one( filter={'_id': ObjectId(item)}))) if db_res_data["status"] != "FINISH": raise MyServiceException("该前置流程未完成(%s)" % item) except Exception as e: print(e) raise MyServiceException("该前置流程未完成(%s) e:%s" % (item, str(e)))
def do_next(process_steps, level=1): process_steps = process_steps.strip() if not process_steps or len(process_steps) < 1: raise MyServiceException("流程(%s)不能为空" % process_steps) # 计算cur_step、next_step、next_steps first_step_sep_index = process_steps.find(">") next_steps = "" next_step = "" # 剩余节点 if first_step_sep_index <= 0: cur_step = process_steps else: cur_step = process_steps[0:first_step_sep_index].strip() if level != 0: next_steps = process_steps[first_step_sep_index + 1:].strip() try: cur_step_temp, next_step_temp, next_steps_temp = do_next( next_steps, level=0) next_step = cur_step_temp except MyServiceException as e: print(e) if level != 0: is_cur_user_process(cur_step) return cur_step, next_step, next_steps
def check_sql_executable(env, host_port, database, sql_content): query_result = json.loads(mysql.get_decrypt()) query_result = query_result[0] level_mysql = query_result["mysql"] level_mysql_instance = level_mysql[env][host_port.replace(".", "_")] host = host_port.split(":")[0] port = host_port.split(":")[1] username = level_mysql_instance["username"] password = level_mysql_instance["password"] sql_content = sql_content.strip() if sql_content.endswith(";"): sql_content = sql_content[0:len(sql_content) - 1] check_execute_sql_result = my_inception.check_execute_sql( host, port, database, username, password, sql_content) is_check_ok = True error_message = "" for item in check_execute_sql_result: errormessage = item["errormessage"] upper_sql = item["SQL"].upper() if "ALTER " in upper_sql or "CREATE " in upper_sql: continue if errormessage != "None": is_check_ok = False error_message += "SQL语句: " + item[ "SQL"] + ": 的问题为" + errormessage + "<br/>" if not is_check_ok: raise MyServiceException("SQL检测失败,原因为: %s <br/>" % error_message)
def do_request(request_data, process_instance_id): method = request_data["method"] url = request_data["url"] data = {"process_instance_id": process_instance_id} net_res_data = requests.request(method=method, url=url, json=data) if net_res_data.status_code != 200: raise MyServiceException("请求失败") print(net_res_data)
def checkout_file(username, password, url): """ 检出文件 :param username: 用户名 :param password: 密码 :param url: url地址 :return: """ print("url: ", url) command_login = "******" % (username, password) # 得到url的文件路径 last_spe = url.rfind("/") + 1 url_dir_path = url[:last_spe] filename = url[last_spe:] dir_path = url_dir_path[url_dir_path.find("//") + len("//"):] dir_path = os.path.join(project_root_path, "temp", dir_path[dir_path.find("/") + len("/"):]) url_dir_path_co = url_dir_path[0:url_dir_path.rfind("/")] url_dir_path_co = url_dir_path_co[0:url_dir_path_co.rfind("/")] dir_path_co = dir_path[0:dir_path.rfind("/")] dir_path_co = dir_path_co[0:dir_path_co.rfind("/")] if not os.path.exists(dir_path_co): command_checkout_dir = "svn co --depth=empty %s %s %s" % ( url_dir_path_co, dir_path_co, command_login) # 检出文件目录 print("command_checkout_dir: ", command_checkout_dir) try: subprocess.run(command_checkout_dir, shell=True, check=True) except Exception as e: print(e) raise MyServiceException("svn检出目录失败") file_path = dir_path + filename update_file_path = file_path[0:file_path.rfind("/")] command_checkout_file = "svn up %s %s" % ( update_file_path, command_login) # 检出指定文件 print("command_checkout_file: ", command_checkout_file) try: subprocess.run(command_checkout_file, shell=True, check=True) except Exception as e: print(e) raise MyServiceException("svn检出文件失败") return file_path
def opt_data_struct(): """ 操作数据结构 """ try: request_data = common_service.check_request_dat_not_null(["opt_name", "code", "meaning", "asset_type"]) asset_type = request_data["asset_type"] opt_name = request_data["opt_name"] code = request_data["code"] meaning = request_data["meaning"] old_data_struct = json.loads(dumps(general_asset_manage_system__data_struct_co.find(filter={ "asset_type": asset_type, "code": code, }))) if "insert" == opt_name: # 校验key唯一性 if old_data_struct: raise MyServiceException("添加失败, 该代码值已经存在,不能重复添加") general_asset_manage_system__data_struct_co.insert_one({ "asset_type": asset_type, "code": code, "meaning": meaning, }) elif "delete" == opt_name: if not old_data_struct: raise MyServiceException("删除失败, 该代码值不存在,不能删除空的代码值") general_asset_manage_system__data_struct_co.delete_one(filter={ "asset_type": asset_type, "code": code, }) elif "update" == opt_name: # 校验key存在性 if not old_data_struct: raise MyServiceException("修改失败, 该代码值不存在,不能修改空的代码值的含义") general_asset_manage_system__data_struct_co.update_one(filter={"asset_type": asset_type, "code": code}, update={'$set': { "meaning": meaning }}) return {} except MyServiceException as e: print(e) return ResResult.return500(str(e))
def process_action_finish(process_instance_id, next_steps, db_res_data): cur_step, next_step, next_steps = do_next(next_steps) ready_to_finish = False if next_step == "": status = "FINISH" # 检查前置工单是否完成 if "prepare_work_order_list" in db_res_data: prepare_work_order_list = db_res_data["prepare_work_order_list"] check_prepare_work_order_list_status(prepare_work_order_list) # 检查是否需要延迟执行 if "special_invoke_datetime_duration" in db_res_data: special_invoke_datetime_duration = db_res_data[ "special_invoke_datetime_duration"] special_invoke_datetime_durations = special_invoke_datetime_duration.split( "-") special_invoke_datetime_duration_start = special_invoke_datetime_durations[ 0] special_invoke_datetime_duration_end = special_invoke_datetime_durations[ 1] cur_datetime = datetime.datetime.now().strftime('%H:%M') if special_invoke_datetime_duration_start > cur_datetime or \ cur_datetime > special_invoke_datetime_duration_end: # 延迟执行 status = "DELAY_INVOKE" if status == "FINISH": ready_to_finish = True else: status = "RUNNING" update_datetime = db_res_data["update_datetime"] convert_update_datetime = [] for item in update_datetime: old_ts = str(item["$date"]) dt1 = datetime.datetime.utcfromtimestamp( float(old_ts) / 10**(len(old_ts) - 10)) convert_update_datetime.append(dt1) convert_update_datetime.append(datetime.datetime.utcnow()) process_instance = { "next_step": next_step, "next_steps": next_steps, "status": status, "update_datetime": convert_update_datetime, "service_invoke_status": "INVOKING" } project_manage__work_order_process__run_manage_co.update_one( filter={'_id': ObjectId(process_instance_id)}, update={'$set': process_instance}) if ready_to_finish: try: do_request(db_res_data["finish_result"], process_instance_id) except Exception as e: print(e) raise MyServiceException("最终执行流程失败,请查询详细日志" + str(e)) if status == "DELAY_INVOKE": create_delay_invoke_thread(process_instance_id) process_after_trigger(process_instance_id, next_step)
def un_archive(filename_path): if not filename_path: return archive_format_index = filename_path.rfind(".") + len(".") archive_format = filename_path[archive_format_index:] filepath = filename_path[:archive_format_index - 1] if "zip" == archive_format: MyArchive.unzip(filepath, filename_path) else: raise MyServiceException("解压压缩文件失败, 不支持的文件类型") return filepath
def process_action_rollback(process_instance_id, db_res_data): if db_res_data["service_type"] in rollback_service_type_only_list: db_res_data_query = json.loads( dumps( project_manage__work_order_process__run_manage_co.find_one( filter={ 'service_type': db_res_data["service_type"], 'system_tag': db_res_data["system_tag"], 'status': 'FINISH', }, sort=[("_id", -1)]))) if db_res_data_query["_id"]["$oid"] != process_instance_id: project_manage__work_order_process__run_manage_co.update_one( filter={'_id': ObjectId(process_instance_id)}, update={'$set': { "is_rollback": True }}) raise MyServiceException("当前系统有新的工单,无法回滚老的工单") method = db_res_data["finish_result"]["method"] rb_url = db_res_data["finish_result"]["rb_url"] update_datetime = db_res_data["update_datetime"] convert_update_datetime = [] for item in update_datetime: old_ts = str(item["$date"]) dt1 = datetime.datetime.utcfromtimestamp( float(old_ts) / 10**(len(old_ts) - 10)) convert_update_datetime.append(dt1) convert_update_datetime.append(datetime.datetime.utcnow()) try: do_request({"method": method, "url": rb_url}, process_instance_id) process_instance = { "is_rollback": True, "update_datetime": convert_update_datetime, } project_manage__work_order_process__run_manage_co.update_one( filter={'_id': ObjectId(process_instance_id)}, update={'$set': process_instance}) except Exception as e: print(e) raise MyServiceException("回滚流程失败" + str(e))
def opt_data(): """ 操作数据 """ try: request_data = common_service.check_request_dat_not_null(["opt_name", "asset_type"]) asset_type = request_data["asset_type"] opt_name = request_data["opt_name"] del request_data["opt_name"] if "insert" == opt_name: # 存在性检测 old_data = json.loads(dumps(general_asset_manage_system__data_struct_co.find(filter=request_data))) # 校验key唯一性 if old_data: raise MyServiceException("添加失败, 数据已经存在,不能重复添加") general_asset_manage_system__data_co.insert_one(request_data) elif "delete" == opt_name: del request_data["_id"] query_old_data = json.loads(dumps(general_asset_manage_system__data_co.find_one(filter=request_data))) if not query_old_data: raise MyServiceException("删除失败, 该数据不存在,不能存在的数据") general_asset_manage_system__data_co.delete_one(filter=request_data) elif "update" == opt_name: request_data = common_service.check_request_dat_not_null(["old_data", "new_data"]) old_data = request_data["old_data"] new_data = request_data["new_data"] del old_data["_id"] del new_data["_id"] # 校验key存在性 query_old_data = json.loads(dumps(general_asset_manage_system__data_co.find_one(filter=old_data))) if not query_old_data: raise MyServiceException("修改失败, 数据不存在,不能修改空的数据") general_asset_manage_system__data_co.update_one(filter=old_data, update={'$set': new_data}) return {} except MyServiceException as e: print(e) return ResResult.return500(str(e))
def check_deploy_php(request_data): svn_tag_package = request_data["svn_tag_package"] project_type = request_data["project_type"] deploy_env = request_data["deploy_env"] project = request_data["project"] # 查询svn资产信息 svn_data = json.loads(svn.get_decrypt())[0]["svn"] host = svn_data["host"] username = svn_data["username"] password = svn_data["password"] if svn_tag_package.find("\n") > 0: svn_tag_package_list = svn_tag_package.splitlines() else: svn_tag_package_list = [svn_tag_package] # 模板数据 projects = json.loads( template.get())[0]["template"][project_type][deploy_env]["projects"] project_template_data = projects[project] # 部署信息 deploy_info = project_template_data["deploy_env"] forbid_dir_list = deploy_info["forbid_dir_list"] display_content_area = "" for svn_tag_package_list_item in svn_tag_package_list: if not svn_tag_package_list_item or svn_tag_package_list_item == "": continue check_php_url_tag_package(svn_tag_package_list_item, host) # 检出svn tag package svn_tag_package_file_path = MySVN.checkout_file( username, password, svn_tag_package_list_item) if not os.path.exists(svn_tag_package_file_path): raise MyServiceException("检查: " + svn_tag_package_list_item + " 时发现包不存在") # 解压文件 un_package_path = MyArchive.extra_file(svn_tag_package_file_path) # 判断该压缩文件中的文件是否符合规范 check_php_tag_package_standard(svn_tag_package_list_item, svn_tag_package_file_path, forbid_dir_list) # 添加svn树状列表文字到日志 un_package_path_path_tree = my_path.get_path_tree(un_package_path) display_content_area += "取包地址为: %s \n" % svn_tag_package_list_item display_content_area += "包中文件为: %s \n" % un_package_path_path_tree return display_content_area
def update(): """ 修改 """ try: request_data = common_service.check_request_dat_not_null( ["id", "name"]) _id = request_data["id"] name = request_data["name"] if not _id or _id == "": raise MyServiceException("不能修改空id节点") process_instance = {"name": name} general_asset_manage_system__metadata_co.update_one( filter={'id': _id}, update={'$set': process_instance}) except MyServiceException as e: print(e) return ResResult.return500(str(e)) return {}
def delete(): """ 删除 """ try: request_data = common_service.check_request_dat_not_null(["id"]) _id = request_data["id"] if not _id or _id == "": raise MyServiceException("不能删除空id节点") # 删除指定id节点 general_asset_manage_system__metadata_co.delete_one(filter={"id": _id}) # 删除子节点 general_asset_manage_system__metadata_co.delete_many( filter={"id": { '$regex': "^" + _id + ".*" }}) return {} except MyServiceException as e: print(e) return ResResult.return500(str(e))
def select_data(): """ 查询数据 分页,搜索 """ try: page_size = 9999999 page_no = 1 # 处理资产类型 request_data = common_service.check_request_dat_not_null([]) if request_data.__contains__("asset_type"): asset_type = request_data["asset_type"] elif request_data.__contains__("asset_type_str"): asset_type_str = request_data["asset_type_str"] asset_type = metadata_manage.get_asset_type_by_asset_type_str(asset_type_str) if not asset_type: raise MyServiceException("查询数据没有指定资产类型") my_filter = {"asset_type": asset_type} if request_data.__contains__("search"): search = request_data["search"] search["asset_type"] = asset_type my_filter = search # 分页 if request_data.__contains__("page"): page = request_data["page"] if page["current"]: page_no = page["current"] if page["page_size"]: page_size = page["page_size"] skip = page_size * (page_no - 1) return dumps({ "data": general_asset_manage_system__data_co.find(filter=my_filter, sort=[("_id", -1)]).limit( page_size).skip(skip), "total": general_asset_manage_system__data_co.find(filter=my_filter, sort=[("_id", -1)]).limit( page_size).skip(skip).count(), }) except MyServiceException as e: print(e) return ResResult.return500(str(e))
def deploy_php(request_data): # 原始方式直接构建 try: process_id = request_data["process_id"] svn_tag_package = request_data["svn_tag_package"] project_type = request_data["project_type"] deploy_env = request_data["deploy_env"] project = request_data["project"] project_manage__work_order_process__run_manage_result_log_co.insert_one( { "process_id": process_id, "log": "开始部署", "is_exe_success": True, }) # 资产数据 jenkins_data = json.loads(jenkins.get_decrypt())[0]["jenkins"] # 模板数据 projects = json.loads(template.get( ))[0]["template"][project_type][deploy_env]["projects"] project_template_data = projects[project] # 构建信息 build_info = project_template_data["build_env"] build_info_host = build_info["host"] build_info_job_name = build_info["job_name"] cur_jenkins = jenkins_data[build_info_host] cur_jenkins_url = cur_jenkins["url"] cur_jenkins_username = cur_jenkins["username"] cur_jenkins_password = cur_jenkins["password"] my_jenkins = MyJenkins(cur_jenkins_url, cur_jenkins_username, cur_jenkins_password) revision, last_build_number = my_jenkins.build_job( build_info_job_name, {"FILELIST": svn_tag_package}) if not revision or revision == "": raise MyServiceException("回滚版本不能为空") project_manage__work_order_process__run_manage_process_process_project_rb_co.insert_one( { "process_id": process_id, "svn_tag_package": svn_tag_package, "project_type": project_type, "deploy_env": deploy_env, "project": project, "revision": revision, }) project_manage__work_order_process__run_manage_result_log_co.insert_one( { "process_id": process_id, "log": "部署成功", "is_exe_success": True, }) project_manage__work_order_process__run_manage_result_log_co.insert_one( { "process_id": process_id, "log": "已经为回滚做好准备,jenkins构建job的id为: %s ,回滚版本为:%s" % (last_build_number, revision), "is_exe_success": True, }) run_manage.upgrade_process_service_invoke_status(process_id, True) except MyServiceException as e: print(e) project_manage__work_order_process__run_manage_result_log_co.insert_one( { "process_id": process_id, "log": "部署失败: " + e.msg, "is_exe_success": True, }) run_manage.upgrade_process_service_invoke_status(process_id, False) raise e
def process_action(): try: request_data = common_service.check_request_dat_not_null( ["process_instance_id", "action"]) cur_user_id = session.get("user_id") process_instance_id = request_data["process_instance_id"] action = request_data["action"] # 动作 is_need_destroy_child_process = False # 查询流程 db_res_data = json.loads( dumps( project_manage__work_order_process__run_manage_co.find_one( filter={'_id': ObjectId(process_instance_id)}))) if action == "FINISH": # 跳转到下一个步骤 # 流程的当前节点必须要是当前登录人 del db_res_data["_id"] next_step = db_res_data["next_step"] next_steps = db_res_data["next_steps"] if next_step != "*" and next_step != cur_user_id: raise MyServiceException("当前登录人(%s)没有权限操作流程(%s)" % (cur_user_id, process_instance_id)) process_action_finish(process_instance_id, next_steps, db_res_data) elif action == "REJECT": # 驳回流程 reject_reason = request_data["reject_reason"] # 驳回理由 print("reject_reason: ", reject_reason) next_steps = db_res_data["next_steps"] do_next(next_steps) process_action_reject(process_instance_id, reject_reason) is_need_destroy_child_process = True elif action == "DESTROY": # 销毁流程 next_steps = db_res_data["next_steps"] if next_steps != "": do_next(next_steps) else: is_cur_user_process(get_last_step(db_res_data["steps"])) process_action_destroy(process_instance_id) is_need_destroy_child_process = True elif action == "ROLLBACK": # 回滚流程 last_step = get_last_step(db_res_data["steps"]) if last_step != "*" and last_step != cur_user_id: raise MyServiceException("当前登录人(%s)没有权限操作流程(%s)" % (cur_user_id, process_instance_id)) process_action_rollback(process_instance_id, db_res_data) is_need_destroy_child_process = True elif action == "SPECIAL_INVOKE_DATETIME_DURATION": # 指定执行时间段 # 流程的当前节点必须要是当前登录人 next_steps = db_res_data["next_steps"] if next_steps == "": is_cur_user_process(get_last_step(db_res_data["steps"])) else: next_step = db_res_data["next_step"] if next_step != "*" and next_step != cur_user_id: raise MyServiceException( "当前登录人(%s)没有权限操作流程(%s)" % (cur_user_id, process_instance_id)) special_invoke_datetime_duration = request_data[ "special_invoke_datetime_duration"] special_invoke_datetime_duration_str = "-".join( special_invoke_datetime_duration) do_special_invoke_datetime_duration( process_instance_id, special_invoke_datetime_duration_str) elif action == "EXCEPTION_INTERRUPT_RESUME": # 异常中断恢复 is_cur_user_process(get_last_step(db_res_data["steps"])) exception_interrupt_resume(process_instance_id, db_res_data) # # # # 处理以当前工单为前置流程的工单 if is_need_destroy_child_process: db_res_data_child_process = json.loads( dumps( project_manage__work_order_process__run_manage_co.find( filter={ 'prepare_work_order_list': { "$regex": process_instance_id } }))) for item in db_res_data_child_process: project_manage__work_order_process__run_manage_co.delete_one( filter={'_id': ObjectId(item["_id"]["$oid"])}) return {} except MyServiceException as e: print(e) return ResResult.return500(str(e))
def check_php_url_tag_package(url, svn_server_host): url_prefix = url[0:len(svn_server_host)] if url_prefix != svn_server_host: raise MyServiceException("检查: " + url + " 时发现文件地址不为允许的svn服务器地址")
def exception_interrupt_resume(process_instance_id, db_res_data): try: do_request(db_res_data["finish_result"], process_instance_id) except Exception as e: print(e) raise MyServiceException("最终执行流程失败,请查询详细日志" + str(e))
def is_cur_user_process(cur_step): cur_user_id = session.get("user_id") if cur_step not in ["*", cur_user_id]: raise MyServiceException("你不是当前流程的指定人")