def rsync_dump(self,
passwd,
achieve,
user,
host,
mode,
port=873,
timeout=60):
"""
:param passwd:
:param timeout:
:param achieve:
:param user:
:param host:
:param mode:
:param port:
:return:
"""
rsync_cmd_str = '''export RSYNC_PASSWORD="******"''' \
''' && /usr/bin/rsync ''' \
'''-vzrtopgPc ''' \
'''--progress ''' \
'''--remove-source-files ''' \
'''--timeout=%d ''' \
'''--port=%d ''' \
'''--chmod=o+r %s %s@%s::%s''' % (
passwd, int(timeout), port, achieve, user, host, mode
)
if not self.exec_command(command=rsync_cmd_str):
RecodeLog.error(msg="推送文件失败！{0}".format(rsync_cmd_str))
return False
else:
RecodeLog.info(msg="推送文件成功！{0}".format(rsync_cmd_str))
return True
def check_record_time(archives_name,
time_offset,
max_size=300 * 1024 * 1024):
"""
:param archives_name:
:param time_offset:
:param max_size:
:return:
"""
try:
archives_unixtime = int(
os.path.splitext(archives_name)[0].split("-")[-3]) / 1000000
except Exception as error:
RecodeLog.error(
"检查文件创建时间class {0},function check_record_time,失败,{1},{2},{3},{4}"
.format("RecordData", archives_name, time_offset, max_size,
error))
return False
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
if (time.time() - archives_unixtime) > time_offset:
raise Exception("已经超出与设定时间，需要新生成文件！")
size = os.path.getsize(archives_name)
if max_size <= size:
raise Exception("文件大小已经超过限制大小。需要生成新文件！")
return True
except Exception as error:
RecodeLog.error(
"检查文件创建时间失败class {0},function check_record_time,失败,{1},{2},{3},{4}"
.format("RecordData", archives_name, time_offset, max_size,
error))
return False
def send_alert(content):
data = {
"msgtype": "text",
"text": {
"content": content
},
"at": {
"isAtAll": False
}
}
headers = {'Content-Type': 'application/json'}
timestamps = long(round(time.time() * 1000))
url = "https://oapi.dingtalk.com/robot/send?access_token={0}".format(
DINGDING_TOKEN) # 说明：这里改为自己创建的机器人的webhook的值
secret_enc = bytes(DINGDING_SECRET).encode('utf-8')
to_sign = '{}\n{}'.format(timestamps, DINGDING_SECRET)
to_sign_enc = bytes(to_sign).encode('utf-8')
hmac_code = hmac.new(secret_enc, to_sign_enc,
digestmod=hashlib.sha256).digest()
sign = urllib.quote_plus(base64.b64encode(hmac_code))
url = "{0}×tamp={1}&sign={2}".format(url, timestamps, sign)
try:
x = requests.post(url=url, data=json.dumps(data), headers=headers)
if x.json()["errcode"] != 0:
raise Exception(x.content)
RecodeLog.info(msg="发送报警成功,url:{0},报警内容:{1}".format(url, data))
except Exception as error:
RecodeLog.info(
msg="发送报警失败,url:{0},报警内容:{1},原因:{2}".format(url, data, error))
def apply_service(self, data, namespace, env, name):
"""
:param data:
:param namespace:
:param env:
:param name:
:return:
"""
env_yaml = os.path.join(KEY_DIR, "{0}.yaml".format(env))
try:
self.login(key_file=env_yaml)
v1 = client.CoreV1Api()
if self.check_service(service=name, namespace=namespace, env=env):
v1.patch_namespaced_service(body=data,
namespace=namespace,
name=name)
else:
v1.create_namespaced_service(body=data, namespace=namespace)
RecodeLog.info(
msg="Kubernetes服务生成成功，环境：{0},命名空间：{1},服务：{2}".format(
env, name, name))
return True
except Exception as error:
RecodeLog.info(
msg="Kubernetes服务生成失败，环境：{0},命名空间：{1},服务：{2}，原因：".format(
env, name, name, error))
return False
def alert_table_partition(self, db, table, partition_name, partition_type):
"""
:param db:
:param table:
:param partition_name:
:param partition_type:
:return:
"""
if not HIVE_TABLE_PARTITION:
return True
partition = time.strftime("%Y%m%d", time.localtime())
sql = "alter table `{2}`.`{3}` add if not exists partition({0}='{1}') location '{1}'".format(
partition_name,
partition,
db,
table
)
try:
self.cursor.execute(sql)
return True
except Exception as error:
RecodeLog.error("创建表分区失败 class {0},function alert_table_partition,{1}.{2} {4}={3}失败{5},{6}".format(
self.__class__.__name__,
db,
table,
partition,
partition_name,
error,
sql
))
return False
def __init__(self, **kwargs):
assert os.path.exists(os.path.join(HIVE_HOME, 'bin', 'hive'))
try:
conn = connect(**kwargs)
self.cursor = conn.cursor()
except Exception as error:
RecodeLog.error("初始化class {0},失败,{1}".format(self.__class__.__name__, error))
sys.exit(1)
def blind(self, **kwargs):
topic = kwargs.pop('topics')
try:
self.__consumer = KafkaConsumer(topic, **kwargs)
RecodeLog.info("卡夫卡初始化成功，{0}成功".format(json.dumps(kwargs)))
except Exception as error:
RecodeLog.info("卡夫卡初始化失败,function blind,{0},{1}".format(
json.dumps(kwargs), str(error)))
sys.exit(1)
def run(self):
RecodeLog.info("初始化成功，开始监听导入任务！")
while True:
for archive in RecordData.get_list_record("*.standby"):
# 获取数据库名称
db = os.path.splitext(archive)[0].split("-")[-2]
# 获取表名称
table = os.path.splitext(archive)[0].split("-")[-1]
# standby 文件时间戳
standby_unixtime = int(os.path.splitext(archive)[0].split("-")[-3])
# 然后将表名修改
alter_sql_list = RecordData.get_list_record("*-{0}-{1}.sql".format(db, table))
if len(alter_sql_list) == 0:
self.command_load(
db=db,
table=table,
data_file=archive,
pro_root=HIVE_HOME
)
else:
for sql in alter_sql_list:
# standby 文件时间戳
sql_unixtime = int(os.path.splitext(sql)[0].split("-")[-3])
# 导入先执行
if sql_unixtime > standby_unixtime:
# 导入
self.command_load(
db=db,
table=table,
data_file=archive,
pro_root=HIVE_HOME
)
# 删除
if not self.exec_hive_sql_file(
sql_file=sql,
data_root=ROOT_DIR,
pro_root=HIVE_HOME
):
return False
else:
# 删除
if not self.exec_hive_sql_file(
sql_file=sql,
data_root=ROOT_DIR,
pro_root=HIVE_HOME
):
return False
# 导入
self.command_load(
db=db,
table=table,
data_file=archive,
pro_root=HIVE_HOME
)
def get_all_tags(self):
"""
:return:
"""
projects = self.get_project()
if not projects:
RecodeLog.error('没有获取到项目列表!')
raise Exception("没有获取到项目列表！")
for project in projects:
for v in self.get_repo(project_id=project['project_id']):
self.get_repo_tags(repo_name=v['repo_name'])
def run(self, exec_function):
"""
:param exec_function:
:return:
"""
if not self.__consumer:
RecodeLog.info("没完成初始化，监听失败")
for msg in self.__consumer:
value = json.loads(msg.value)
if value['type'] == "QUERY":
continue
exec_function(data=value)
def read_yaml(achieve):
"""
:param achieve:
:return:
"""
try:
with open(achieve, 'r') as fff:
data = yaml.load(fff, Loader=yaml.FullLoader)
RecodeLog.info(msg="读取文件成功:{0}".format(achieve))
return data
except Exception as error:
RecodeLog.error(msg="读取文件失败:{0},原因:{1}".format(achieve, error))
return dict()
def create_database(self, db):
"""
:param db:
:return:
"""
try:
self.cursor.execute("create database %s" % db)
return True
except Exception as error:
RecodeLog.error("创建数据库失败class {0},function create_database,失败,{1}".format(
self.__class__.__name__,
error
))
return False
def get_remove_tags(self, repo_name, remove_count):
"""
:param repo_name:
:param remove_count:
:return:
"""
tags = self.client.get_repository_tags(repo_name=repo_name)
health = sorted(tags, key=lambda k: k['created'])
health = [{
'tag': x['name'],
'repo_name': repo_name,
'created': x['created']
} for x in health]
RecodeLog.info("repo:{1}的完整tag列表为:{0}".format(health, repo_name))
RecodeLog.info("开始删除repo:{0}的相关tag,删除tag个数为:{1}".format(
repo_name, remove_count))
for tag in health[0:remove_count]:
result = self.client.delete_repository_tag(
repo_name=tag['repo_name'], tag=tag['tag'])
if result:
RecodeLog.info(msg="删除镜像:{0},tag:{1},成功！".format(
tag['repo_name'], tag['tag']))
else:
RecodeLog.info(msg="删除镜像:{0},tag:{1},失败！原因：{2}".format(
tag['repo_name'], tag['tag'], result))
def read_record(archives_name):
"""
:param archives_name:
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
with open(archives_name, 'r') as f:
return f.readlines()
except Exception as error:
RecodeLog.error(
"读取文件 class {0},function read_record,失败,{1},{2}".format(
"RecordData", archives_name, error))
return False
def get_project(self):
"""
:return:
"""
if not self.client:
RecodeLog.error('harbor还没登录')
raise Exception("harbor还没登录")
project_list = list()
for x in self.client.get_projects():
project_list.append({
'project_name': x['name'],
'project_id': x['project_id']
})
return project_list
def touch_record(archives_name):
"""
:param archives_name:
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
with open(archives_name, 'w+') as f:
f.close()
return archives_name
except Exception as error:
RecodeLog.error(
"生成文件 class {0},function touch_record,失败,{1},{2}".format(
"RecordData", archives_name, error))
return False
def write_yaml(achieve, data):
"""
:param achieve:
:param data:
:return:
"""
try:
with open(achieve, "w") as f:
yaml.dump(data, f)
f.close()
RecodeLog.info(msg="执行成功, 保存内容是：{0}".format(data))
return True
except Exception as error:
RecodeLog.error(msg="执行失败, 保存内容是：{0}，原因:{1}".format(data, error))
return False
def get_address():
"""
:return:
"""
if sys.version_info < (3, 0):
import commands
(status, output) = commands.getstatusoutput(cmd=GET_ADDRESS_CMD)
else:
import subprocess
(status, output) = subprocess.getstatusoutput(cmd=GET_ADDRESS_CMD)
if status != 0:
RecodeLog.error("执行命令异常：{0},原因:{1}".format(GET_ADDRESS_CMD,
output))
raise Exception(output)
return output
def check_db(self, db):
"""
:param db:
:return:
"""
try:
self.cursor.execute('''show databases like "%s"''' % db)
if len(self.cursor.fetchall()) == 0:
raise Exception("The database '%s' is not exist!" % db)
return True
except Exception as error:
RecodeLog.error("检查数据库失败class {0},function check_db,失败,{1}".format(
self.__class__.__name__,
error
))
return False
def check_table(self, db, table):
"""
:param db:
:param table:
:return:
"""
try:
self.cursor.execute('''show tables in {0} like "{1}" '''.format(db, table))
if len(self.cursor.fetchall()) == 0:
raise Exception("The table {0}.{1} is not exist!".format(db, table))
return True
except Exception as error:
RecodeLog.error("检查表class {0},function check_table,不存在或者失败,{1}".format(
self.__class__.__name__,
error
))
return False
def format_pom_xml(self, src, dsc, deploy_name):
"""
:param src:
:param dsc:
:param deploy_name:
:return:
"""
dom_tree = parse(src)
root_node = dom_tree.documentElement
for project in root_node.childNodes:
if project.nodeName != "artifactId":
continue
project.childNodes[0].nodeValue = deploy_name
with open(dsc, 'w') as f:
dom_tree.writexml(f, addindent=' ')
RecodeLog.info("修改pom.xml文件成功：{0}".format(dsc))
def postgres_dump(self, params, db_config):
"""
:param params:
:param db_config:
:return:
"""
if not isinstance(db_config, str):
raise Exception("输入数据库类型错误！{0}", db_config)
psql = os.path.join(EXEC_BIN, 'psql')
pg_dump = os.path.join(EXEC_BIN, 'pg_dump')
if not os.path.exists(psql) or not os.path.exists(pg_dump):
raise EnvironmentError("可执行命令不存在: {0},{1}".format(psql, pg_dump))
dblist = self.get_database_list(db_config=db_config)
if len(dblist) == 0 or not dblist:
raise Exception("没有获取到数据库列表:{0}".format(db_config))
pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config])
ipaddress = self.get_address()
pg_password = pg_params.pop('password')
pg_database = pg_params.pop('database')
dump_params = "export PGPASSWORD={0} && {1} {2}".format(
pg_password, pg_dump, copy.deepcopy(params))
rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config])
for key, value in pg_params.items():
dump_params = "{0} --{1}={2}".format(dump_params, key, value)
for db in dblist:
achieve = os.path.join(
BACKUP_DIR, "{0}_{1}_{2}.gz".format(
ipaddress, db[0],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))
dump_str = "{0} {1}| /usr/bin/gzip > {2} && md5sum {2} > {2}.md5".format(
dump_params, db[0],
os.path.join(
BACKUP_DIR, "{0}_{1}_{2}.gz".format(
ipaddress, db[0],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M"))))
rsync_params['achieve'] = "{0}.*".format(
os.path.splitext(achieve)[0])
if not self.exec_command(command=dump_str):
RecodeLog.error(msg="备份数据库失败：{0}".format(dump_str))
else:
RecodeLog.info(msg="备份数据库成功：{0}".format(dump_str))
self.rsync_dump(**rsync_params)
def write_record(archives_name, data):
"""
:param archives_name:
:param data:
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
try:
with open(archives_name, "a") as f:
f.write("{0}\n".format(data))
f.close()
return True
except Exception as error:
RecodeLog.error(
"写入文件 class {0},function write_record,失败,{1},{2},{3}".format(
"RecordData", archives_name, data, error))
return False
def exec_hive_sql_file(self, sql_file, pro_root, data_root):
"""
:param sql_file:
:param pro_root:
:param data_root:
:return:
"""
hive_bin = os.path.join(pro_root, 'bin', 'hive')
abs_sql_file = os.path.join(data_root, sql_file)
exec_str = "{0} -f {1}".format(hive_bin, abs_sql_file)
# 获取数据库名称
db = os.path.splitext(sql_file)[0].split("-")[-2]
# 获取表名称
table = os.path.splitext(sql_file)[0].split("-")[-1]
try:
if not self.check_table(db=db, table=table):
return True
if int(platform.python_version().strip(".")[0]) < 3:
status, msg = commands.getstatusoutput(exec_str)
else:
status, msg = subprocess.getstatusoutput(exec_str)
if status != 0:
raise Exception(msg)
new_name = "{0}.success.{1}".format(
sql_file,
''.join(random.sample(string.ascii_letters + string.digits, 8))
)
RecordData.rename_record(archives_name=sql_file, new_archives=new_name)
RecodeLog.info("class {0},function command_load,执行完成：{1}成功".format(
self.__class__.__name__,
exec_str
))
return True
except Exception as error:
new_name = "{0}.error.{1}".format(
sql_file,
''.join(random.sample(string.ascii_letters + string.digits, 8))
)
RecordData.rename_record(archives_name=sql_file, new_archives=new_name)
RecodeLog.error("class {0},function command_load,{1}执行失败,{2}".format(
self.__class__.__name__,
exec_str,
error
))
return False
def get_middleware_config(name):
"""
:param name:
:return:
"""
if name not in MIDDLEWARE:
RecodeLog.error(
msg="{0},中间件不存在配置文件中setting.MIDDLEWARE,请检查是否填写错误，或者请添加相关中间件配置！"
.format(name))
raise KeyError(
"{0},中间件不存在配置文件中setting.MIDDLEWARE,请检查是否填写错误，或者请添加相关中间件配置！".
format(name))
env = list()
for key, value in MIDDLEWARE[name].items():
if isinstance(value, int) or isinstance(value, float):
value = str(value)
env.append({"name": key, "value": value})
return env
def get_remove_repo(self, save_count=4):
"""
:param save_count:
:return:
"""
projects = self.get_project()
if not projects:
RecodeLog.error('没有获取到项目列表!')
raise Exception("没有获取到项目列表！")
for project in projects:
for v in self.get_repo(project_id=project['project_id']):
if v['tags_count'] <= save_count:
continue
if v['repo_name'].startswith('images'):
continue
self.get_remove_tags(repo_name=v['repo_name'],
remove_count=int(v['tags_count']) -
save_count)
def exec_command(self, command):
"""
:param command:
:return:
"""
try:
if sys.version_info < (3, 0):
import commands
(status, output) = commands.getstatusoutput(cmd=command)
else:
import subprocess
(status, output) = subprocess.getstatusoutput(cmd=command)
if status != 0:
raise Exception(output)
RecodeLog.info(msg="执行命令成功：{0}".format(command))
return True
except Exception as error:
RecodeLog.error("执行命令异常：{0},原因:{1}".format(command, error))
return False
def create_table(self, sql):
"""
:param sql:
:return:
"""
try:
self.cursor.execute(sql)
RecodeLog.info("创建表class {0},function create_table,成功,{1}".format(
self.__class__.__name__,
sql
))
return True
except Exception as error:
RecodeLog.error("创建表失败class {0},function create_table,{1},失败,{2}".format(
self.__class__.__name__,
sql,
error
))
return False
def rename_record(archives_name, new_archives):
"""
:param archives_name:
:param new_archives
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
new_archives = os.path.join(ROOT_DIR, new_archives)
try:
os.rename(archives_name, new_archives)
RecodeLog.info(
"重命名文件 class {0},function rename_record,成功,befor={1},after={2}"
.format("RecordData", archives_name, new_archives))
return True
except Exception as error:
RecodeLog.error(
"重命名文件 class {0},function rename_record,失败,befor={1},after={2},{3}"
.format("RecordData", archives_name, new_archives, error))
return False
def make_deploy_yaml(self, namespace, service_name, image_tag, ports, env,
version, replicas, env_args):
"""
:param namespace:
:param service_name:
:param image_tag:
:param ports:
:param env:
:param version:
:param replicas:
:param env_args:
:return:
"""
deploy_yaml = os.path.join(
DEPLOY_YAML_DIR,
"deploy_{0}_{1}.yaml".format(service_name, version))
if os.path.exists(deploy_yaml):
RecodeLog.warn(
msg="{0}:文件已经存在，以前可能已经部署过，请检查后再执行！".format(deploy_yaml))
return False
# 容器内部端口
container_port = list()
# container port获取
for x in ports:
container_port.append({'containerPort': int(x)})
# 初始化服务的dict
deploy_dict = self.format_config_deploy(namespace=namespace,
service_name=service_name,
image_tag="{0}:{1}".format(
image_tag, version),
container_port=container_port,
env_args=env_args,
replicas=replicas)
# 生成yaml
if not self.write_yaml(achieve=deploy_yaml, data=deploy_dict):
RecodeLog.error(msg="生成yaml文件失败：{0}".format(deploy_yaml))
return False
if not self.apply_deployment(
data=deploy_dict, namespace=namespace, env=env,
name=service_name):
return False
return True