def rsync_dump(self, passwd, achieve, user, host, mode, port=873, timeout=60): """ :param passwd: :param timeout: :param achieve: :param user: :param host: :param mode: :param port: :return: """ rsync_cmd_str = '''export RSYNC_PASSWORD="******"''' \ ''' && /usr/bin/rsync ''' \ '''-vzrtopgPc ''' \ '''--progress ''' \ '''--remove-source-files ''' \ '''--timeout=%d ''' \ '''--port=%d ''' \ '''--chmod=o+r %s %s@%s::%s''' % ( passwd, int(timeout), port, achieve, user, host, mode ) if not self.exec_command(command=rsync_cmd_str): RecodeLog.error(msg="推送文件失败!{0}".format(rsync_cmd_str)) return False else: RecodeLog.info(msg="推送文件成功!{0}".format(rsync_cmd_str)) return True
def apply_deployment(self, data, namespace, env, name): """ :param data: :param namespace: :param env: :param name: :return: """ env_yaml = os.path.join(KEY_DIR, "{0}.yaml".format(env)) try: self.login(key_file=env_yaml) v1 = client.AppsV1Api() if self.check_deploy(namespace=namespace, env=env, deploy=name): v1.patch_namespaced_deployment(body=data, namespace=namespace, name=name) else: v1.create_namespaced_deployment(body=data, namespace=namespace) RecodeLog.info( msg="Kubernetes容器更新成功,环境:{0},命名空间:{1},服务:{2}".format( env, name, name)) return True except Exception as error: RecodeLog.error( msg="Kubernetes容器更新成功,环境:{0},命名空间:{1},服务:{2},原因:".format( env, name, name, error)) return False
def alert_table_partition(self, db, table, partition_name, partition_type): """ :param db: :param table: :param partition_name: :param partition_type: :return: """ if not HIVE_TABLE_PARTITION: return True partition = time.strftime("%Y%m%d", time.localtime()) sql = "alter table `{2}`.`{3}` add if not exists partition({0}='{1}') location '{1}'".format( partition_name, partition, db, table ) try: self.cursor.execute(sql) return True except Exception as error: RecodeLog.error("创建表分区失败 class {0},function alert_table_partition,{1}.{2} {4}={3}失败{5},{6}".format( self.__class__.__name__, db, table, partition, partition_name, error, sql )) return False
def check_record_time(archives_name, time_offset, max_size=300 * 1024 * 1024): """ :param archives_name: :param time_offset: :param max_size: :return: """ try: archives_unixtime = int( os.path.splitext(archives_name)[0].split("-")[-3]) / 1000000 except Exception as error: RecodeLog.error( "检查文件创建时间class {0},function check_record_time,失败,{1},{2},{3},{4}" .format("RecordData", archives_name, time_offset, max_size, error)) return False archives_name = os.path.join(ROOT_DIR, archives_name) try: if (time.time() - archives_unixtime) > time_offset: raise Exception("已经超出与设定时间,需要新生成文件!") size = os.path.getsize(archives_name) if max_size <= size: raise Exception("文件大小已经超过限制大小。需要生成新文件!") return True except Exception as error: RecodeLog.error( "检查文件创建时间失败class {0},function check_record_time,失败,{1},{2},{3},{4}" .format("RecordData", archives_name, time_offset, max_size, error)) return False
def create_if_not_exist(self, db, table): """ :param db: :param table: :return: """ # ##################################创建数据库############################ try: # 判断并创建数据库 if not self.check_db(db=db): if not self.create_database(db=db): raise Exception("该Hive实例不存在该数据库,创建数据库失败!") else: RecodeLog.info("创建数据库 class {0},function create_if_not_exist,{1}成功".format( self.__class__.__name__, db )) except Exception as error: RecodeLog.error("创建数据库 class {0},function create_if_not_exist,{1}失败{2}".format( self.__class__.__name__, db, error )) return False # ##########################创建表############################## try: if not self.check_table(db=db, table=table): create_data = RecordData.read_create_table_data(db=db, table=table) if HIVE_TABLE_PARTITION: create_params = { "db": db, "table": table, "partition": HIVE_TABLE_PARTITION_NAME, "partition_type": HIVE_TABLE_PARTITION_TYPE, "data": json.loads(create_data) } else: create_params = { "db": db, "table": table, "data": json.loads(create_data) } if not create_data: raise Exception("读取表结构数据失败") sql = self.format_create_table_sql(**create_params) if not sql: raise Exception("产生的建表语句异常") if not self.create_table(sql=sql): raise Exception("创建表失败{0}".format(sql)) return True except Exception as error: RecodeLog.error("创建表 class {0},function create_if_not_exist,{1}.{2}失败{3}".format( self.__class__.__name__, db, table, error )) return False
def __init__(self, **kwargs): assert os.path.exists(os.path.join(HIVE_HOME, 'bin', 'hive')) try: conn = connect(**kwargs) self.cursor = conn.cursor() except Exception as error: RecodeLog.error("初始化class {0},失败,{1}".format(self.__class__.__name__, error)) sys.exit(1)
def get_all_tags(self): """ :return: """ projects = self.get_project() if not projects: RecodeLog.error('没有获取到项目列表!') raise Exception("没有获取到项目列表!") for project in projects: for v in self.get_repo(project_id=project['project_id']): self.get_repo_tags(repo_name=v['repo_name'])
def read_yaml(achieve): """ :param achieve: :return: """ try: with open(achieve, 'r') as fff: data = yaml.load(fff, Loader=yaml.FullLoader) RecodeLog.info(msg="读取文件成功:{0}".format(achieve)) return data except Exception as error: RecodeLog.error(msg="读取文件失败:{0},原因:{1}".format(achieve, error)) return dict()
def create_database(self, db): """ :param db: :return: """ try: self.cursor.execute("create database %s" % db) return True except Exception as error: RecodeLog.error("创建数据库失败class {0},function create_database,失败,{1}".format( self.__class__.__name__, error )) return False
def get_project(self): """ :return: """ if not self.client: RecodeLog.error('harbor还没登录') raise Exception("harbor还没登录") project_list = list() for x in self.client.get_projects(): project_list.append({ 'project_name': x['name'], 'project_id': x['project_id'] }) return project_list
def read_record(archives_name): """ :param archives_name: :return: """ archives_name = os.path.join(ROOT_DIR, archives_name) try: with open(archives_name, 'r') as f: return f.readlines() except Exception as error: RecodeLog.error( "读取文件 class {0},function read_record,失败,{1},{2}".format( "RecordData", archives_name, error)) return False
def touch_record(archives_name): """ :param archives_name: :return: """ archives_name = os.path.join(ROOT_DIR, archives_name) try: with open(archives_name, 'w+') as f: f.close() return archives_name except Exception as error: RecodeLog.error( "生成文件 class {0},function touch_record,失败,{1},{2}".format( "RecordData", archives_name, error)) return False
def write_yaml(achieve, data): """ :param achieve: :param data: :return: """ try: with open(achieve, "w") as f: yaml.dump(data, f) f.close() RecodeLog.info(msg="执行成功, 保存内容是:{0}".format(data)) return True except Exception as error: RecodeLog.error(msg="执行失败, 保存内容是:{0},原因:{1}".format(data, error)) return False
def get_address(): """ :return: """ if sys.version_info < (3, 0): import commands (status, output) = commands.getstatusoutput(cmd=GET_ADDRESS_CMD) else: import subprocess (status, output) = subprocess.getstatusoutput(cmd=GET_ADDRESS_CMD) if status != 0: RecodeLog.error("执行命令异常:{0},原因:{1}".format(GET_ADDRESS_CMD, output)) raise Exception(output) return output
def check_db(self, db): """ :param db: :return: """ try: self.cursor.execute('''show databases like "%s"''' % db) if len(self.cursor.fetchall()) == 0: raise Exception("The database '%s' is not exist!" % db) return True except Exception as error: RecodeLog.error("检查数据库失败class {0},function check_db,失败,{1}".format( self.__class__.__name__, error )) return False
def check_table(self, db, table): """ :param db: :param table: :return: """ try: self.cursor.execute('''show tables in {0} like "{1}" '''.format(db, table)) if len(self.cursor.fetchall()) == 0: raise Exception("The table {0}.{1} is not exist!".format(db, table)) return True except Exception as error: RecodeLog.error("检查表class {0},function check_table,不存在或者失败,{1}".format( self.__class__.__name__, error )) return False
def write_record(archives_name, data): """ :param archives_name: :param data: :return: """ archives_name = os.path.join(ROOT_DIR, archives_name) try: with open(archives_name, "a") as f: f.write("{0}\n".format(data)) f.close() return True except Exception as error: RecodeLog.error( "写入文件 class {0},function write_record,失败,{1},{2},{3}".format( "RecordData", archives_name, data, error)) return False
def postgres_dump(self, params, db_config): """ :param params: :param db_config: :return: """ if not isinstance(db_config, str): raise Exception("输入数据库类型错误!{0}", db_config) psql = os.path.join(EXEC_BIN, 'psql') pg_dump = os.path.join(EXEC_BIN, 'pg_dump') if not os.path.exists(psql) or not os.path.exists(pg_dump): raise EnvironmentError("可执行命令不存在: {0},{1}".format(psql, pg_dump)) dblist = self.get_database_list(db_config=db_config) if len(dblist) == 0 or not dblist: raise Exception("没有获取到数据库列表:{0}".format(db_config)) pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config]) ipaddress = self.get_address() pg_password = pg_params.pop('password') pg_database = pg_params.pop('database') dump_params = "export PGPASSWORD={0} && {1} {2}".format( pg_password, pg_dump, copy.deepcopy(params)) rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config]) for key, value in pg_params.items(): dump_params = "{0} --{1}={2}".format(dump_params, key, value) for db in dblist: achieve = os.path.join( BACKUP_DIR, "{0}_{1}_{2}.gz".format( ipaddress, db[0], datetime.datetime.now().strftime("%Y-%m-%d-%H-%M"))) dump_str = "{0} {1}| /usr/bin/gzip > {2} && md5sum {2} > {2}.md5".format( dump_params, db[0], os.path.join( BACKUP_DIR, "{0}_{1}_{2}.gz".format( ipaddress, db[0], datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))) rsync_params['achieve'] = "{0}.*".format( os.path.splitext(achieve)[0]) if not self.exec_command(command=dump_str): RecodeLog.error(msg="备份数据库失败:{0}".format(dump_str)) else: RecodeLog.info(msg="备份数据库成功:{0}".format(dump_str)) self.rsync_dump(**rsync_params)
def exec_hive_sql_file(self, sql_file, pro_root, data_root): """ :param sql_file: :param pro_root: :param data_root: :return: """ hive_bin = os.path.join(pro_root, 'bin', 'hive') abs_sql_file = os.path.join(data_root, sql_file) exec_str = "{0} -f {1}".format(hive_bin, abs_sql_file) # 获取数据库名称 db = os.path.splitext(sql_file)[0].split("-")[-2] # 获取表名称 table = os.path.splitext(sql_file)[0].split("-")[-1] try: if not self.check_table(db=db, table=table): return True if int(platform.python_version().strip(".")[0]) < 3: status, msg = commands.getstatusoutput(exec_str) else: status, msg = subprocess.getstatusoutput(exec_str) if status != 0: raise Exception(msg) new_name = "{0}.success.{1}".format( sql_file, ''.join(random.sample(string.ascii_letters + string.digits, 8)) ) RecordData.rename_record(archives_name=sql_file, new_archives=new_name) RecodeLog.info("class {0},function command_load,执行完成:{1}成功".format( self.__class__.__name__, exec_str )) return True except Exception as error: new_name = "{0}.error.{1}".format( sql_file, ''.join(random.sample(string.ascii_letters + string.digits, 8)) ) RecordData.rename_record(archives_name=sql_file, new_archives=new_name) RecodeLog.error("class {0},function command_load,{1}执行失败,{2}".format( self.__class__.__name__, exec_str, error )) return False
def get_middleware_config(name): """ :param name: :return: """ if name not in MIDDLEWARE: RecodeLog.error( msg="{0},中间件不存在配置文件中setting.MIDDLEWARE,请检查是否填写错误,或者请添加相关中间件配置!" .format(name)) raise KeyError( "{0},中间件不存在配置文件中setting.MIDDLEWARE,请检查是否填写错误,或者请添加相关中间件配置!". format(name)) env = list() for key, value in MIDDLEWARE[name].items(): if isinstance(value, int) or isinstance(value, float): value = str(value) env.append({"name": key, "value": value}) return env
def get_remove_repo(self, save_count=4): """ :param save_count: :return: """ projects = self.get_project() if not projects: RecodeLog.error('没有获取到项目列表!') raise Exception("没有获取到项目列表!") for project in projects: for v in self.get_repo(project_id=project['project_id']): if v['tags_count'] <= save_count: continue if v['repo_name'].startswith('images'): continue self.get_remove_tags(repo_name=v['repo_name'], remove_count=int(v['tags_count']) - save_count)
def exec_command(self, command): """ :param command: :return: """ try: if sys.version_info < (3, 0): import commands (status, output) = commands.getstatusoutput(cmd=command) else: import subprocess (status, output) = subprocess.getstatusoutput(cmd=command) if status != 0: raise Exception(output) RecodeLog.info(msg="执行命令成功:{0}".format(command)) return True except Exception as error: RecodeLog.error("执行命令异常:{0},原因:{1}".format(command, error)) return False
def create_table(self, sql): """ :param sql: :return: """ try: self.cursor.execute(sql) RecodeLog.info("创建表class {0},function create_table,成功,{1}".format( self.__class__.__name__, sql )) return True except Exception as error: RecodeLog.error("创建表失败class {0},function create_table,{1},失败,{2}".format( self.__class__.__name__, sql, error )) return False
def rename_record(archives_name, new_archives): """ :param archives_name: :param new_archives :return: """ archives_name = os.path.join(ROOT_DIR, archives_name) new_archives = os.path.join(ROOT_DIR, new_archives) try: os.rename(archives_name, new_archives) RecodeLog.info( "重命名文件 class {0},function rename_record,成功,befor={1},after={2}" .format("RecordData", archives_name, new_archives)) return True except Exception as error: RecodeLog.error( "重命名文件 class {0},function rename_record,失败,befor={1},after={2},{3}" .format("RecordData", archives_name, new_archives, error)) return False
def make_deploy_yaml(self, namespace, service_name, image_tag, ports, env, version, replicas, env_args): """ :param namespace: :param service_name: :param image_tag: :param ports: :param env: :param version: :param replicas: :param env_args: :return: """ deploy_yaml = os.path.join( DEPLOY_YAML_DIR, "deploy_{0}_{1}.yaml".format(service_name, version)) if os.path.exists(deploy_yaml): RecodeLog.warn( msg="{0}:文件已经存在,以前可能已经部署过,请检查后再执行!".format(deploy_yaml)) return False # 容器内部端口 container_port = list() # container port获取 for x in ports: container_port.append({'containerPort': int(x)}) # 初始化服务的dict deploy_dict = self.format_config_deploy(namespace=namespace, service_name=service_name, image_tag="{0}:{1}".format( image_tag, version), container_port=container_port, env_args=env_args, replicas=replicas) # 生成yaml if not self.write_yaml(achieve=deploy_yaml, data=deploy_dict): RecodeLog.error(msg="生成yaml文件失败:{0}".format(deploy_yaml)) return False if not self.apply_deployment( data=deploy_dict, namespace=namespace, env=env, name=service_name): return False return True
def check_table_partitioned(self, db, table): """ :param db: :param table: :return: """ try: self.cursor.execute('''show create table `{0}`.{1}'''.format(db, table)) result = self.cursor.fetchall() if len(result) == 0: raise Exception("The database '%s' is not exist!" % db) if "PARTITIONED" in result: return {"status": True} else: return {"status": False} except Exception as error: RecodeLog.error("检查数据库失败class {0},function check_db,失败,{1}".format( self.__class__.__name__, error )) return False
def pg_basedump(self, db_config, params): """ :param db_config: :param params: :return: """ if not isinstance(db_config, str): raise Exception("输入数据库类型错误!{0}", db_config) pg_basebackup = os.path.join(EXEC_BIN, 'pg_basebackup') if not os.path.exists(pg_basebackup): raise EnvironmentError("可执行命令不存在: {0}".format(pg_basebackup)) pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config]) ipaddress = self.get_address() pg_password = pg_params.pop('password') pg_database = pg_params.pop('database') dump_params = "export PGPASSWORD={0} && {1} {2}".format( pg_password, pg_basebackup, copy.deepcopy(params)) rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config]) for key, value in pg_params.items(): dump_params = "{0} --{1}={2}".format(dump_params, key, value) achieve = os.path.join( BACKUP_DIR, "{0}_all-dump_{1}".format( ipaddress, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M"))) if not os.path.exists(achieve): os.makedirs(achieve) pg_basedump_str = "{0} -D {1} ".format(dump_params, achieve) rsync_params['achieve'] = achieve if not self.exec_command(command=pg_basedump_str): RecodeLog.error(msg="全量备份异常!{0}".format(pg_basedump_str)) return False else: RecodeLog.info(msg="全量备份成功!{0}".format(pg_basedump_str)) self.rsync_dump(**rsync_params) return True
def insert(self, sql, db): """ :param sql: :param db: :return: """ if not isinstance(sql, str): RecodeLog.error("输入类型错误 class {0},function insert,{1}失败".format( self.__class__.__name__, sql )) return False if 'INSERT INTO' not in sql.upper(): RecodeLog.error("sql 错误 class {0},function insert,{1}失败".format( self.__class__.__name__, sql )) return False if "CREATE TABLE" in sql.upper(): pass if not self.check_db(db=db): self.create_database(db=db) try: self.cursor.execute("use %s" % db) self.cursor.execute(sql) RecodeLog.info("class {0},function insert,{1}成功".format( self.__class__.__name__, sql )) return True except Exception as error: RecodeLog.error("class {0},function insert,{1}失败,{2}".format( self.__class__.__name__, sql, error )) return False
def complete(self, env, namespace, service, version, port_type, env_args, replicas=1): """ :param env: :param namespace: :param service: :param version: :param port_type: :param env_args: :param replicas: :return: """ # 编译程序所在目录 service_path = os.path.join(WORKSPACE, service) # 镜像标签 tag = os.path.join(HARBOR_URL, service) # 获取到的容器内部存在的node_port node_port = self.get_service_nodeports(env=env) # ##############镜像相关的操作##################### front_app = os.path.join(service_path, 'package.json') backend_app = os.path.join(service_path, 'pom.xml') # front_docker_path = os.path.join(service_path, 'docker') if os.path.exists(front_app) and os.path.exists(backend_app): RecodeLog.error(msg="{2}.同时出现前端和后端的配置:{0},{1}".format( front_app, backend_app, service)) return False elif os.path.exists(front_app): RecodeLog.info(msg="{0},应用为前端应用,开始编译....".format(service)) exec_npm_str = "cd {0} && yarn".format(service_path) if env in ['qa', 'dev']: exec_npm_run_str = "cd {0} && yarn build:{1}".format( service_path, env) else: exec_npm_run_str = "cd {0} && yarn build".format( service_path, env) try: self.exec_command(command=exec_npm_str) RecodeLog.info("执行编译成功:{0}".format(exec_npm_str)) self.exec_command(command=exec_npm_run_str) RecodeLog.info("执行编译成功:{0}".format(exec_npm_run_str)) self.docker_image_build(path=service_path, tag=tag, version=version) RecodeLog.info(msg="生成前端镜像成功:{0}:{1}".format(tag, version)) self.docker_image_push( repository="{0}:{1}".format(tag, version)) RecodeLog.info(msg="推送前端镜像成功:{0}:{1}".format(tag, version)) self.docker_local_image_remove( image_name="{0}:{1}".format(tag, version)) RecodeLog.info(msg="删除镜像成功:{0}:{1}".format(tag, version)) except Exception as error: RecodeLog.error("执行编译异常,原因:{0}".format(error)) return False elif env in ['dev'] and os.path.exists(backend_app): try: self.docker_image_build(path=service_path, tag=tag, version=version) RecodeLog.info(msg="生成后端镜像成功:{0}:{1}".format(tag, version)) self.docker_image_push( repository="{0}:{1}".format(tag, version)) RecodeLog.info(msg="推送后端镜像成功:{0}:{1}".format(tag, version)) self.docker_local_image_remove( image_name="{0}:{1}".format(tag, version)) RecodeLog.info(msg="删除镜像成功:{0}:{1}".format(tag, version)) except Exception as error: RecodeLog.error("执行编译异常,原因:{0}".format(error)) return False ports = self.get_port(path=service_path) if not self.make_service_yaml(namespace=namespace, service_name=service, port_type=port_type, ports=ports, exist_node_port=node_port, env=env): return False if not self.make_deploy_yaml(namespace=namespace, service_name=service, image_tag=tag, ports=ports, env=env, env_args=env_args, version=version, replicas=replicas): return False return True
def front_build_pre(self, path, dockerfile_path, deploy_name): """ :param path: :param dockerfile_path: :param deploy_name: :return: """ bootstrap_dict = { "spring": { "application": { "name": deploy_name }, "cloud": { "consul": { "host": "${consul_host:127.0.0.1}", "port": "${consul_port:8500}", "enabled": True, "discovery": { "enabled": True, "instance-id": "${spring.application.name}:${server.port}", "prefer-ip-address": True, "health-check-interval": "10s", "hostname": "${spring.application.name}", "service-name": "${spring.application.name}" } } }, "mvc": { "favicon": { "enabled": False } }, "boot": { "admin": { "client": { "url": "${nccc_admin_monitor:http://localhost:9020}", "instance": { "prefer-ip": True } } } } }, "server": { "port": "${server-port:8888}" } } if not os.path.exists(dockerfile_path): RecodeLog.error(msg="前端编译依赖的目录不存在,请检查") raise Exception("前端编译依赖的目录不存在,请检查") if os.path.exists(path): RecodeLog.warn(msg="存在之前的目录:{0},删除!".format(path)) shutil.rmtree(path=path) shutil.copytree(src=dockerfile_path, dst=path) # 写入文件bootstrap bootstrap_yaml = os.path.join(path, 'target', 'classes', 'bootstrap.yml') if not self.write_yaml(achieve=bootstrap_yaml, data=bootstrap_dict): raise Exception("写入文件失败:{0},内容:{1}".format(bootstrap_yaml, bootstrap_dict)) self.format_pom_xml(src=SRC_POM_DIR, dsc=os.path.join(path, 'pom.xml'), deploy_name=deploy_name)