def rsync_dump(self,
passwd,
achieve,
user,
host,
mode,
port=873,
timeout=60):
"""
:param passwd:
:param timeout:
:param achieve:
:param user:
:param host:
:param mode:
:param port:
:return:
"""
rsync_cmd_str = '''export RSYNC_PASSWORD="******"''' \
''' && /usr/bin/rsync ''' \
'''-vzrtopgPc ''' \
'''--progress ''' \
'''--remove-source-files ''' \
'''--timeout=%d ''' \
'''--port=%d ''' \
'''--chmod=o+r %s %s@%s::%s''' % (
passwd, int(timeout), port, achieve, user, host, mode
)
if not self.exec_command(command=rsync_cmd_str):
RecodeLog.error(msg="推送文件失败！{0}".format(rsync_cmd_str))
return False
else:
RecodeLog.info(msg="推送文件成功！{0}".format(rsync_cmd_str))
return True
def apply_deployment(self, data, namespace, env, name):
"""
:param data:
:param namespace:
:param env:
:param name:
:return:
"""
env_yaml = os.path.join(KEY_DIR, "{0}.yaml".format(env))
try:
self.login(key_file=env_yaml)
v1 = client.AppsV1Api()
if self.check_deploy(namespace=namespace, env=env, deploy=name):
v1.patch_namespaced_deployment(body=data,
namespace=namespace,
name=name)
else:
v1.create_namespaced_deployment(body=data, namespace=namespace)
RecodeLog.info(
msg="Kubernetes容器更新成功，环境：{0},命名空间：{1},服务：{2}".format(
env, name, name))
return True
except Exception as error:
RecodeLog.error(
msg="Kubernetes容器更新成功，环境：{0},命名空间：{1},服务：{2}，原因：".format(
env, name, name, error))
return False
def alert_table_partition(self, db, table, partition_name, partition_type):
"""
:param db:
:param table:
:param partition_name:
:param partition_type:
:return:
"""
if not HIVE_TABLE_PARTITION:
return True
partition = time.strftime("%Y%m%d", time.localtime())
sql = "alter table `{2}`.`{3}` add if not exists partition({0}='{1}') location '{1}'".format(
partition_name,
partition,
db,
table
)
try:
self.cursor.execute(sql)
return True
except Exception as error:
RecodeLog.error("创建表分区失败 class {0},function alert_table_partition,{1}.{2} {4}={3}失败{5},{6}".format(
self.__class__.__name__,
db,
table,
partition,
partition_name,
error,
sql
))
return False
def check_record_time(archives_name,
time_offset,
max_size=300 * 1024 * 1024):
"""
:param archives_name:
:param time_offset:
:param max_size:
:return:
"""
try:
archives_unixtime = int(
os.path.splitext(archives_name)[0].split("-")[-3]) / 1000000
except Exception as error:
RecodeLog.error(
"检查文件创建时间class {0},function check_record_time,失败,{1},{2},{3},{4}"
.format("RecordData", archives_name, time_offset, max_size,
error))
return False
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
if (time.time() - archives_unixtime) > time_offset:
raise Exception("已经超出与设定时间，需要新生成文件！")
size = os.path.getsize(archives_name)
if max_size <= size:
raise Exception("文件大小已经超过限制大小。需要生成新文件！")
return True
except Exception as error:
RecodeLog.error(
"检查文件创建时间失败class {0},function check_record_time,失败,{1},{2},{3},{4}"
.format("RecordData", archives_name, time_offset, max_size,
error))
return False
def create_if_not_exist(self, db, table):
"""
:param db:
:param table:
:return:
"""
# ##################################创建数据库############################
try:
# 判断并创建数据库
if not self.check_db(db=db):
if not self.create_database(db=db):
raise Exception("该Hive实例不存在该数据库，创建数据库失败！")
else:
RecodeLog.info("创建数据库 class {0},function create_if_not_exist,{1}成功".format(
self.__class__.__name__,
db
))
except Exception as error:
RecodeLog.error("创建数据库 class {0},function create_if_not_exist,{1}失败{2}".format(
self.__class__.__name__,
db,
error
))
return False
# ##########################创建表##############################
try:
if not self.check_table(db=db, table=table):
create_data = RecordData.read_create_table_data(db=db, table=table)
if HIVE_TABLE_PARTITION:
create_params = {
"db": db,
"table": table,
"partition": HIVE_TABLE_PARTITION_NAME,
"partition_type": HIVE_TABLE_PARTITION_TYPE,
"data": json.loads(create_data)
}
else:
create_params = {
"db": db,
"table": table,
"data": json.loads(create_data)
}
if not create_data:
raise Exception("读取表结构数据失败")
sql = self.format_create_table_sql(**create_params)
if not sql:
raise Exception("产生的建表语句异常")
if not self.create_table(sql=sql):
raise Exception("创建表失败{0}".format(sql))
return True
except Exception as error:
RecodeLog.error("创建表 class {0},function create_if_not_exist,{1}.{2}失败{3}".format(
self.__class__.__name__,
db,
table,
error
))
return False
def __init__(self, **kwargs):
assert os.path.exists(os.path.join(HIVE_HOME, 'bin', 'hive'))
try:
conn = connect(**kwargs)
self.cursor = conn.cursor()
except Exception as error:
RecodeLog.error("初始化class {0},失败,{1}".format(self.__class__.__name__, error))
sys.exit(1)
def get_all_tags(self):
"""
:return:
"""
projects = self.get_project()
if not projects:
RecodeLog.error('没有获取到项目列表!')
raise Exception("没有获取到项目列表！")
for project in projects:
for v in self.get_repo(project_id=project['project_id']):
self.get_repo_tags(repo_name=v['repo_name'])
def read_yaml(achieve):
"""
:param achieve:
:return:
"""
try:
with open(achieve, 'r') as fff:
data = yaml.load(fff, Loader=yaml.FullLoader)
RecodeLog.info(msg="读取文件成功:{0}".format(achieve))
return data
except Exception as error:
RecodeLog.error(msg="读取文件失败:{0},原因:{1}".format(achieve, error))
return dict()
def create_database(self, db):
"""
:param db:
:return:
"""
try:
self.cursor.execute("create database %s" % db)
return True
except Exception as error:
RecodeLog.error("创建数据库失败class {0},function create_database,失败,{1}".format(
self.__class__.__name__,
error
))
return False
def get_project(self):
"""
:return:
"""
if not self.client:
RecodeLog.error('harbor还没登录')
raise Exception("harbor还没登录")
project_list = list()
for x in self.client.get_projects():
project_list.append({
'project_name': x['name'],
'project_id': x['project_id']
})
return project_list
def read_record(archives_name):
"""
:param archives_name:
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
with open(archives_name, 'r') as f:
return f.readlines()
except Exception as error:
RecodeLog.error(
"读取文件 class {0},function read_record,失败,{1},{2}".format(
"RecordData", archives_name, error))
return False
def touch_record(archives_name):
"""
:param archives_name:
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
with open(archives_name, 'w+') as f:
f.close()
return archives_name
except Exception as error:
RecodeLog.error(
"生成文件 class {0},function touch_record,失败,{1},{2}".format(
"RecordData", archives_name, error))
return False
def write_yaml(achieve, data):
"""
:param achieve:
:param data:
:return:
"""
try:
with open(achieve, "w") as f:
yaml.dump(data, f)
f.close()
RecodeLog.info(msg="执行成功, 保存内容是：{0}".format(data))
return True
except Exception as error:
RecodeLog.error(msg="执行失败, 保存内容是：{0}，原因:{1}".format(data, error))
return False
def get_address():
"""
:return:
"""
if sys.version_info < (3, 0):
import commands
(status, output) = commands.getstatusoutput(cmd=GET_ADDRESS_CMD)
else:
import subprocess
(status, output) = subprocess.getstatusoutput(cmd=GET_ADDRESS_CMD)
if status != 0:
RecodeLog.error("执行命令异常：{0},原因:{1}".format(GET_ADDRESS_CMD,
output))
raise Exception(output)
return output
def check_db(self, db):
"""
:param db:
:return:
"""
try:
self.cursor.execute('''show databases like "%s"''' % db)
if len(self.cursor.fetchall()) == 0:
raise Exception("The database '%s' is not exist!" % db)
return True
except Exception as error:
RecodeLog.error("检查数据库失败class {0},function check_db,失败,{1}".format(
self.__class__.__name__,
error
))
return False
def check_table(self, db, table):
"""
:param db:
:param table:
:return:
"""
try:
self.cursor.execute('''show tables in {0} like "{1}" '''.format(db, table))
if len(self.cursor.fetchall()) == 0:
raise Exception("The table {0}.{1} is not exist!".format(db, table))
return True
except Exception as error:
RecodeLog.error("检查表class {0},function check_table,不存在或者失败,{1}".format(
self.__class__.__name__,
error
))
return False
def write_record(archives_name, data):
"""
:param archives_name:
:param data:
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
with open(archives_name, "a") as f:
f.write("{0}\n".format(data))
f.close()
return True
except Exception as error:
RecodeLog.error(
"写入文件 class {0},function write_record,失败,{1},{2},{3}".format(
"RecordData", archives_name, data, error))
return False
def postgres_dump(self, params, db_config):
"""
:param params:
:param db_config:
:return:
"""
if not isinstance(db_config, str):
raise Exception("输入数据库类型错误！{0}", db_config)
psql = os.path.join(EXEC_BIN, 'psql')
pg_dump = os.path.join(EXEC_BIN, 'pg_dump')
if not os.path.exists(psql) or not os.path.exists(pg_dump):
raise EnvironmentError("可执行命令不存在: {0},{1}".format(psql, pg_dump))
dblist = self.get_database_list(db_config=db_config)
if len(dblist) == 0 or not dblist:
raise Exception("没有获取到数据库列表:{0}".format(db_config))
pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config])
ipaddress = self.get_address()
pg_password = pg_params.pop('password')
pg_database = pg_params.pop('database')
dump_params = "export PGPASSWORD={0} && {1} {2}".format(
pg_password, pg_dump, copy.deepcopy(params))
rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config])
for key, value in pg_params.items():
dump_params = "{0} --{1}={2}".format(dump_params, key, value)
for db in dblist:
achieve = os.path.join(
BACKUP_DIR, "{0}_{1}_{2}.gz".format(
ipaddress, db[0],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))
dump_str = "{0} {1}| /usr/bin/gzip > {2} && md5sum {2} > {2}.md5".format(
dump_params, db[0],
os.path.join(
BACKUP_DIR, "{0}_{1}_{2}.gz".format(
ipaddress, db[0],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M"))))
rsync_params['achieve'] = "{0}.*".format(
os.path.splitext(achieve)[0])
if not self.exec_command(command=dump_str):
RecodeLog.error(msg="备份数据库失败：{0}".format(dump_str))
else:
RecodeLog.info(msg="备份数据库成功：{0}".format(dump_str))
self.rsync_dump(**rsync_params)
def exec_hive_sql_file(self, sql_file, pro_root, data_root):
"""
:param sql_file:
:param pro_root:
:param data_root:
:return:
"""
hive_bin = os.path.join(pro_root, 'bin', 'hive')
abs_sql_file = os.path.join(data_root, sql_file)
exec_str = "{0} -f {1}".format(hive_bin, abs_sql_file)
# 获取数据库名称
db = os.path.splitext(sql_file)[0].split("-")[-2]
# 获取表名称
table = os.path.splitext(sql_file)[0].split("-")[-1]
try:
if not self.check_table(db=db, table=table):
return True
if int(platform.python_version().strip(".")[0]) < 3:
status, msg = commands.getstatusoutput(exec_str)
else:
status, msg = subprocess.getstatusoutput(exec_str)
if status != 0:
raise Exception(msg)
new_name = "{0}.success.{1}".format(
sql_file,
''.join(random.sample(string.ascii_letters + string.digits, 8))
)
RecordData.rename_record(archives_name=sql_file, new_archives=new_name)
RecodeLog.info("class {0},function command_load,执行完成：{1}成功".format(
self.__class__.__name__,
exec_str
))
return True
except Exception as error:
new_name = "{0}.error.{1}".format(
sql_file,
''.join(random.sample(string.ascii_letters + string.digits, 8))
)
RecordData.rename_record(archives_name=sql_file, new_archives=new_name)
RecodeLog.error("class {0},function command_load,{1}执行失败,{2}".format(
self.__class__.__name__,
exec_str,
error
))
return False
def get_middleware_config(name):
"""
:param name:
:return:
"""
if name not in MIDDLEWARE:
RecodeLog.error(
msg="{0},中间件不存在配置文件中setting.MIDDLEWARE,请检查是否填写错误，或者请添加相关中间件配置！"
.format(name))
raise KeyError(
"{0},中间件不存在配置文件中setting.MIDDLEWARE,请检查是否填写错误，或者请添加相关中间件配置！".
format(name))
env = list()
for key, value in MIDDLEWARE[name].items():
if isinstance(value, int) or isinstance(value, float):
value = str(value)
env.append({"name": key, "value": value})
return env
def get_remove_repo(self, save_count=4):
"""
:param save_count:
:return:
"""
projects = self.get_project()
if not projects:
RecodeLog.error('没有获取到项目列表!')
raise Exception("没有获取到项目列表！")
for project in projects:
for v in self.get_repo(project_id=project['project_id']):
if v['tags_count'] <= save_count:
continue
if v['repo_name'].startswith('images'):
continue
self.get_remove_tags(repo_name=v['repo_name'],
remove_count=int(v['tags_count']) -
save_count)
def exec_command(self, command):
"""
:param command:
:return:
"""
try:
if sys.version_info < (3, 0):
import commands
(status, output) = commands.getstatusoutput(cmd=command)
else:
import subprocess
(status, output) = subprocess.getstatusoutput(cmd=command)
if status != 0:
raise Exception(output)
RecodeLog.info(msg="执行命令成功：{0}".format(command))
return True
except Exception as error:
RecodeLog.error("执行命令异常：{0},原因:{1}".format(command, error))
return False
def create_table(self, sql):
"""
:param sql:
:return:
"""
try:
self.cursor.execute(sql)
RecodeLog.info("创建表class {0},function create_table,成功,{1}".format(
self.__class__.__name__,
sql
))
return True
except Exception as error:
RecodeLog.error("创建表失败class {0},function create_table,{1},失败,{2}".format(
self.__class__.__name__,
sql,
error
))
return False
def rename_record(archives_name, new_archives):
"""
:param archives_name:
:param new_archives
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
new_archives = os.path.join(ROOT_DIR, new_archives)
try:
os.rename(archives_name, new_archives)
RecodeLog.info(
"重命名文件 class {0},function rename_record,成功,befor={1},after={2}"
.format("RecordData", archives_name, new_archives))
return True
except Exception as error:
RecodeLog.error(
"重命名文件 class {0},function rename_record,失败,befor={1},after={2},{3}"
.format("RecordData", archives_name, new_archives, error))
return False
def make_deploy_yaml(self, namespace, service_name, image_tag, ports, env,
version, replicas, env_args):
"""
:param namespace:
:param service_name:
:param image_tag:
:param ports:
:param env:
:param version:
:param replicas:
:param env_args:
:return:
"""
deploy_yaml = os.path.join(
DEPLOY_YAML_DIR,
"deploy_{0}_{1}.yaml".format(service_name, version))
if os.path.exists(deploy_yaml):
RecodeLog.warn(
msg="{0}:文件已经存在，以前可能已经部署过，请检查后再执行！".format(deploy_yaml))
return False
# 容器内部端口
container_port = list()
# container port获取
for x in ports:
container_port.append({'containerPort': int(x)})
# 初始化服务的dict
deploy_dict = self.format_config_deploy(namespace=namespace,
service_name=service_name,
image_tag="{0}:{1}".format(
image_tag, version),
container_port=container_port,
env_args=env_args,
replicas=replicas)
# 生成yaml
if not self.write_yaml(achieve=deploy_yaml, data=deploy_dict):
RecodeLog.error(msg="生成yaml文件失败：{0}".format(deploy_yaml))
return False
if not self.apply_deployment(
data=deploy_dict, namespace=namespace, env=env,
name=service_name):
return False
return True
def check_table_partitioned(self, db, table):
"""
:param db:
:param table:
:return:
"""
try:
self.cursor.execute('''show create table `{0}`.{1}'''.format(db, table))
result = self.cursor.fetchall()
if len(result) == 0:
raise Exception("The database '%s' is not exist!" % db)
if "PARTITIONED" in result:
return {"status": True}
else:
return {"status": False}
except Exception as error:
RecodeLog.error("检查数据库失败class {0},function check_db,失败,{1}".format(
self.__class__.__name__,
error
))
return False
def pg_basedump(self, db_config, params):
"""
:param db_config:
:param params:
:return:
"""
if not isinstance(db_config, str):
raise Exception("输入数据库类型错误！{0}", db_config)
pg_basebackup = os.path.join(EXEC_BIN, 'pg_basebackup')
if not os.path.exists(pg_basebackup):
raise EnvironmentError("可执行命令不存在: {0}".format(pg_basebackup))
pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config])
ipaddress = self.get_address()
pg_password = pg_params.pop('password')
pg_database = pg_params.pop('database')
dump_params = "export PGPASSWORD={0} && {1} {2}".format(
pg_password, pg_basebackup, copy.deepcopy(params))
rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config])
for key, value in pg_params.items():
dump_params = "{0} --{1}={2}".format(dump_params, key, value)
achieve = os.path.join(
BACKUP_DIR, "{0}_all-dump_{1}".format(
ipaddress,
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))
if not os.path.exists(achieve):
os.makedirs(achieve)
pg_basedump_str = "{0} -D {1} ".format(dump_params, achieve)
rsync_params['achieve'] = achieve
if not self.exec_command(command=pg_basedump_str):
RecodeLog.error(msg="全量备份异常！{0}".format(pg_basedump_str))
return False
else:
RecodeLog.info(msg="全量备份成功！{0}".format(pg_basedump_str))
self.rsync_dump(**rsync_params)
return True
def insert(self, sql, db):
"""
:param sql:
:param db:
:return:
"""
if not isinstance(sql, str):
RecodeLog.error("输入类型错误 class {0},function insert,{1}失败".format(
self.__class__.__name__,
sql
))
return False
if 'INSERT INTO' not in sql.upper():
RecodeLog.error("sql 错误 class {0},function insert,{1}失败".format(
self.__class__.__name__,
sql
))
return False
if "CREATE TABLE" in sql.upper():
pass
if not self.check_db(db=db):
self.create_database(db=db)
try:
self.cursor.execute("use %s" % db)
self.cursor.execute(sql)
RecodeLog.info("class {0},function insert,{1}成功".format(
self.__class__.__name__,
sql
))
return True
except Exception as error:
RecodeLog.error("class {0},function insert,{1}失败,{2}".format(
self.__class__.__name__,
sql,
error
))
return False
def complete(self,
env,
namespace,
service,
version,
port_type,
env_args,
replicas=1):
"""
:param env:
:param namespace:
:param service:
:param version:
:param port_type:
:param env_args:
:param replicas:
:return:
"""
# 编译程序所在目录
service_path = os.path.join(WORKSPACE, service)
# 镜像标签
tag = os.path.join(HARBOR_URL, service)
# 获取到的容器内部存在的node_port
node_port = self.get_service_nodeports(env=env)
# ##############镜像相关的操作#####################
front_app = os.path.join(service_path, 'package.json')
backend_app = os.path.join(service_path, 'pom.xml')
# front_docker_path = os.path.join(service_path, 'docker')
if os.path.exists(front_app) and os.path.exists(backend_app):
RecodeLog.error(msg="{2}.同时出现前端和后端的配置：{0},{1}".format(
front_app, backend_app, service))
return False
elif os.path.exists(front_app):
RecodeLog.info(msg="{0},应用为前端应用,开始编译....".format(service))
exec_npm_str = "cd {0} && yarn".format(service_path)
if env in ['qa', 'dev']:
exec_npm_run_str = "cd {0} && yarn build:{1}".format(
service_path, env)
else:
exec_npm_run_str = "cd {0} && yarn build".format(
service_path, env)
try:
self.exec_command(command=exec_npm_str)
RecodeLog.info("执行编译成功：{0}".format(exec_npm_str))
self.exec_command(command=exec_npm_run_str)
RecodeLog.info("执行编译成功：{0}".format(exec_npm_run_str))
self.docker_image_build(path=service_path,
tag=tag,
version=version)
RecodeLog.info(msg="生成前端镜像成功:{0}:{1}".format(tag, version))
self.docker_image_push(
repository="{0}:{1}".format(tag, version))
RecodeLog.info(msg="推送前端镜像成功:{0}:{1}".format(tag, version))
self.docker_local_image_remove(
image_name="{0}:{1}".format(tag, version))
RecodeLog.info(msg="删除镜像成功：{0}:{1}".format(tag, version))
except Exception as error:
RecodeLog.error("执行编译异常，原因：{0}".format(error))
return False
elif env in ['dev'] and os.path.exists(backend_app):
try:
self.docker_image_build(path=service_path,
tag=tag,
version=version)
RecodeLog.info(msg="生成后端镜像成功:{0}:{1}".format(tag, version))
self.docker_image_push(
repository="{0}:{1}".format(tag, version))
RecodeLog.info(msg="推送后端镜像成功:{0}:{1}".format(tag, version))
self.docker_local_image_remove(
image_name="{0}:{1}".format(tag, version))
RecodeLog.info(msg="删除镜像成功：{0}:{1}".format(tag, version))
except Exception as error:
RecodeLog.error("执行编译异常，原因：{0}".format(error))
return False
ports = self.get_port(path=service_path)
if not self.make_service_yaml(namespace=namespace,
service_name=service,
port_type=port_type,
ports=ports,
exist_node_port=node_port,
env=env):
return False
if not self.make_deploy_yaml(namespace=namespace,
service_name=service,
image_tag=tag,
ports=ports,
env=env,
env_args=env_args,
version=version,
replicas=replicas):
return False
return True
def front_build_pre(self, path, dockerfile_path, deploy_name):
"""
:param path:
:param dockerfile_path:
:param deploy_name:
:return:
"""
bootstrap_dict = {
"spring": {
"application": {
"name": deploy_name
},
"cloud": {
"consul": {
"host": "${consul_host:127.0.0.1}",
"port": "${consul_port:8500}",
"enabled": True,
"discovery": {
"enabled": True,
"instance-id":
"${spring.application.name}:${server.port}",
"prefer-ip-address": True,
"health-check-interval": "10s",
"hostname": "${spring.application.name}",
"service-name": "${spring.application.name}"
}
}
},
"mvc": {
"favicon": {
"enabled": False
}
},
"boot": {
"admin": {
"client": {
"url":
"${nccc_admin_monitor:http://localhost:9020}",
"instance": {
"prefer-ip": True
}
}
}
}
},
"server": {
"port": "${server-port:8888}"
}
}
if not os.path.exists(dockerfile_path):
RecodeLog.error(msg="前端编译依赖的目录不存在，请检查")
raise Exception("前端编译依赖的目录不存在，请检查")
if os.path.exists(path):
RecodeLog.warn(msg="存在之前的目录:{0},删除！".format(path))
shutil.rmtree(path=path)
shutil.copytree(src=dockerfile_path, dst=path)
# 写入文件bootstrap
bootstrap_yaml = os.path.join(path, 'target', 'classes',
'bootstrap.yml')
if not self.write_yaml(achieve=bootstrap_yaml, data=bootstrap_dict):
raise Exception("写入文件失败:{0},内容:{1}".format(bootstrap_yaml,
bootstrap_dict))
self.format_pom_xml(src=SRC_POM_DIR,
dsc=os.path.join(path, 'pom.xml'),
deploy_name=deploy_name)
