def send_alert(content):
data = {
"msgtype": "text",
"text": {
"content": content
},
"at": {
"isAtAll": False
}
}
headers = {'Content-Type': 'application/json'}
timestamps = long(round(time.time() * 1000))
url = "https://oapi.dingtalk.com/robot/send?access_token={0}".format(
DINGDING_TOKEN) # 说明：这里改为自己创建的机器人的webhook的值
secret_enc = bytes(DINGDING_SECRET).encode('utf-8')
to_sign = '{}\n{}'.format(timestamps, DINGDING_SECRET)
to_sign_enc = bytes(to_sign).encode('utf-8')
hmac_code = hmac.new(secret_enc, to_sign_enc,
digestmod=hashlib.sha256).digest()
sign = urllib.quote_plus(base64.b64encode(hmac_code))
url = "{0}×tamp={1}&sign={2}".format(url, timestamps, sign)
try:
x = requests.post(url=url, data=json.dumps(data), headers=headers)
if x.json()["errcode"] != 0:
raise Exception(x.content)
RecodeLog.info(msg="发送报警成功,url:{0},报警内容:{1}".format(url, data))
except Exception as error:
RecodeLog.info(
msg="发送报警失败,url:{0},报警内容:{1},原因:{2}".format(url, data, error))
def apply_service(self, data, namespace, env, name):
"""
:param data:
:param namespace:
:param env:
:param name:
:return:
"""
env_yaml = os.path.join(KEY_DIR, "{0}.yaml".format(env))
try:
self.login(key_file=env_yaml)
v1 = client.CoreV1Api()
if self.check_service(service=name, namespace=namespace, env=env):
v1.patch_namespaced_service(body=data,
namespace=namespace,
name=name)
else:
v1.create_namespaced_service(body=data, namespace=namespace)
RecodeLog.info(
msg="Kubernetes服务生成成功，环境：{0},命名空间：{1},服务：{2}".format(
env, name, name))
return True
except Exception as error:
RecodeLog.info(
msg="Kubernetes服务生成失败，环境：{0},命名空间：{1},服务：{2}，原因：".format(
env, name, name, error))
return False
def rsync_dump(self,
passwd,
achieve,
user,
host,
mode,
port=873,
timeout=60):
"""
:param passwd:
:param timeout:
:param achieve:
:param user:
:param host:
:param mode:
:param port:
:return:
"""
rsync_cmd_str = '''export RSYNC_PASSWORD="******"''' \
''' && /usr/bin/rsync ''' \
'''-vzrtopgPc ''' \
'''--progress ''' \
'''--remove-source-files ''' \
'''--timeout=%d ''' \
'''--port=%d ''' \
'''--chmod=o+r %s %s@%s::%s''' % (
passwd, int(timeout), port, achieve, user, host, mode
)
if not self.exec_command(command=rsync_cmd_str):
RecodeLog.error(msg="推送文件失败！{0}".format(rsync_cmd_str))
return False
else:
RecodeLog.info(msg="推送文件成功！{0}".format(rsync_cmd_str))
return True
def create_if_not_exist(self, db, table):
"""
:param db:
:param table:
:return:
"""
# ##################################创建数据库############################
try:
# 判断并创建数据库
if not self.check_db(db=db):
if not self.create_database(db=db):
raise Exception("该Hive实例不存在该数据库，创建数据库失败！")
else:
RecodeLog.info("创建数据库 class {0},function create_if_not_exist,{1}成功".format(
self.__class__.__name__,
db
))
except Exception as error:
RecodeLog.error("创建数据库 class {0},function create_if_not_exist,{1}失败{2}".format(
self.__class__.__name__,
db,
error
))
return False
# ##########################创建表##############################
try:
if not self.check_table(db=db, table=table):
create_data = RecordData.read_create_table_data(db=db, table=table)
if HIVE_TABLE_PARTITION:
create_params = {
"db": db,
"table": table,
"partition": HIVE_TABLE_PARTITION_NAME,
"partition_type": HIVE_TABLE_PARTITION_TYPE,
"data": json.loads(create_data)
}
else:
create_params = {
"db": db,
"table": table,
"data": json.loads(create_data)
}
if not create_data:
raise Exception("读取表结构数据失败")
sql = self.format_create_table_sql(**create_params)
if not sql:
raise Exception("产生的建表语句异常")
if not self.create_table(sql=sql):
raise Exception("创建表失败{0}".format(sql))
return True
except Exception as error:
RecodeLog.error("创建表 class {0},function create_if_not_exist,{1}.{2}失败{3}".format(
self.__class__.__name__,
db,
table,
error
))
return False
def blind(self, **kwargs):
topic = kwargs.pop('topics')
try:
self.__consumer = KafkaConsumer(topic, **kwargs)
RecodeLog.info("卡夫卡初始化成功，{0}成功".format(json.dumps(kwargs)))
except Exception as error:
RecodeLog.info("卡夫卡初始化失败,function blind,{0},{1}".format(
json.dumps(kwargs), str(error)))
sys.exit(1)
def run(self):
RecodeLog.info("初始化成功，开始监听导入任务！")
while True:
for archive in RecordData.get_list_record("*.standby"):
# 获取数据库名称
db = os.path.splitext(archive)[0].split("-")[-2]
# 获取表名称
table = os.path.splitext(archive)[0].split("-")[-1]
# standby 文件时间戳
standby_unixtime = int(os.path.splitext(archive)[0].split("-")[-3])
# 然后将表名修改
alter_sql_list = RecordData.get_list_record("*-{0}-{1}.sql".format(db, table))
if len(alter_sql_list) == 0:
self.command_load(
db=db,
table=table,
data_file=archive,
pro_root=HIVE_HOME
)
else:
for sql in alter_sql_list:
# standby 文件时间戳
sql_unixtime = int(os.path.splitext(sql)[0].split("-")[-3])
# 导入先执行
if sql_unixtime > standby_unixtime:
# 导入
self.command_load(
db=db,
table=table,
data_file=archive,
pro_root=HIVE_HOME
)
# 删除
if not self.exec_hive_sql_file(
sql_file=sql,
data_root=ROOT_DIR,
pro_root=HIVE_HOME
):
return False
else:
# 删除
if not self.exec_hive_sql_file(
sql_file=sql,
data_root=ROOT_DIR,
pro_root=HIVE_HOME
):
return False
# 导入
self.command_load(
db=db,
table=table,
data_file=archive,
pro_root=HIVE_HOME
)
def run(self, exec_function):
"""
:param exec_function:
:return:
"""
if not self.__consumer:
RecodeLog.info("没完成初始化，监听失败")
for msg in self.__consumer:
value = json.loads(msg.value)
if value['type'] == "QUERY":
continue
exec_function(data=value)
def read_yaml(achieve):
"""
:param achieve:
:return:
"""
try:
with open(achieve, 'r') as fff:
data = yaml.load(fff, Loader=yaml.FullLoader)
RecodeLog.info(msg="读取文件成功:{0}".format(achieve))
return data
except Exception as error:
RecodeLog.error(msg="读取文件失败:{0},原因:{1}".format(achieve, error))
return dict()
def get_remove_tags(self, repo_name, remove_count):
"""
:param repo_name:
:param remove_count:
:return:
"""
tags = self.client.get_repository_tags(repo_name=repo_name)
health = sorted(tags, key=lambda k: k['created'])
health = [{
'tag': x['name'],
'repo_name': repo_name,
'created': x['created']
} for x in health]
RecodeLog.info("repo:{1}的完整tag列表为:{0}".format(health, repo_name))
RecodeLog.info("开始删除repo:{0}的相关tag,删除tag个数为:{1}".format(
repo_name, remove_count))
for tag in health[0:remove_count]:
result = self.client.delete_repository_tag(
repo_name=tag['repo_name'], tag=tag['tag'])
if result:
RecodeLog.info(msg="删除镜像:{0},tag:{1},成功！".format(
tag['repo_name'], tag['tag']))
else:
RecodeLog.info(msg="删除镜像:{0},tag:{1},失败！原因：{2}".format(
tag['repo_name'], tag['tag'], result))
def write_yaml(achieve, data):
"""
:param achieve:
:param data:
:return:
"""
try:
with open(achieve, "w") as f:
yaml.dump(data, f)
f.close()
RecodeLog.info(msg="执行成功, 保存内容是：{0}".format(data))
return True
except Exception as error:
RecodeLog.error(msg="执行失败, 保存内容是：{0}，原因:{1}".format(data, error))
return False
def format_pom_xml(self, src, dsc, deploy_name):
"""
:param src:
:param dsc:
:param deploy_name:
:return:
"""
dom_tree = parse(src)
root_node = dom_tree.documentElement
for project in root_node.childNodes:
if project.nodeName != "artifactId":
continue
project.childNodes[0].nodeValue = deploy_name
with open(dsc, 'w') as f:
dom_tree.writexml(f, addindent=' ')
RecodeLog.info("修改pom.xml文件成功：{0}".format(dsc))
def postgres_dump(self, params, db_config):
"""
:param params:
:param db_config:
:return:
"""
if not isinstance(db_config, str):
raise Exception("输入数据库类型错误！{0}", db_config)
psql = os.path.join(EXEC_BIN, 'psql')
pg_dump = os.path.join(EXEC_BIN, 'pg_dump')
if not os.path.exists(psql) or not os.path.exists(pg_dump):
raise EnvironmentError("可执行命令不存在: {0},{1}".format(psql, pg_dump))
dblist = self.get_database_list(db_config=db_config)
if len(dblist) == 0 or not dblist:
raise Exception("没有获取到数据库列表:{0}".format(db_config))
pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config])
ipaddress = self.get_address()
pg_password = pg_params.pop('password')
pg_database = pg_params.pop('database')
dump_params = "export PGPASSWORD={0} && {1} {2}".format(
pg_password, pg_dump, copy.deepcopy(params))
rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config])
for key, value in pg_params.items():
dump_params = "{0} --{1}={2}".format(dump_params, key, value)
for db in dblist:
achieve = os.path.join(
BACKUP_DIR, "{0}_{1}_{2}.gz".format(
ipaddress, db[0],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))
dump_str = "{0} {1}| /usr/bin/gzip > {2} && md5sum {2} > {2}.md5".format(
dump_params, db[0],
os.path.join(
BACKUP_DIR, "{0}_{1}_{2}.gz".format(
ipaddress, db[0],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M"))))
rsync_params['achieve'] = "{0}.*".format(
os.path.splitext(achieve)[0])
if not self.exec_command(command=dump_str):
RecodeLog.error(msg="备份数据库失败：{0}".format(dump_str))
else:
RecodeLog.info(msg="备份数据库成功：{0}".format(dump_str))
self.rsync_dump(**rsync_params)
def exec_hive_sql_file(self, sql_file, pro_root, data_root):
"""
:param sql_file:
:param pro_root:
:param data_root:
:return:
"""
hive_bin = os.path.join(pro_root, 'bin', 'hive')
abs_sql_file = os.path.join(data_root, sql_file)
exec_str = "{0} -f {1}".format(hive_bin, abs_sql_file)
# 获取数据库名称
db = os.path.splitext(sql_file)[0].split("-")[-2]
# 获取表名称
table = os.path.splitext(sql_file)[0].split("-")[-1]
try:
if not self.check_table(db=db, table=table):
return True
if int(platform.python_version().strip(".")[0]) < 3:
status, msg = commands.getstatusoutput(exec_str)
else:
status, msg = subprocess.getstatusoutput(exec_str)
if status != 0:
raise Exception(msg)
new_name = "{0}.success.{1}".format(
sql_file,
''.join(random.sample(string.ascii_letters + string.digits, 8))
)
RecordData.rename_record(archives_name=sql_file, new_archives=new_name)
RecodeLog.info("class {0},function command_load,执行完成：{1}成功".format(
self.__class__.__name__,
exec_str
))
return True
except Exception as error:
new_name = "{0}.error.{1}".format(
sql_file,
''.join(random.sample(string.ascii_letters + string.digits, 8))
)
RecordData.rename_record(archives_name=sql_file, new_archives=new_name)
RecodeLog.error("class {0},function command_load,{1}执行失败,{2}".format(
self.__class__.__name__,
exec_str,
error
))
return False
def exec_command(self, command):
"""
:param command:
:return:
"""
try:
if sys.version_info < (3, 0):
import commands
(status, output) = commands.getstatusoutput(cmd=command)
else:
import subprocess
(status, output) = subprocess.getstatusoutput(cmd=command)
if status != 0:
raise Exception(output)
RecodeLog.info(msg="执行命令成功：{0}".format(command))
return True
except Exception as error:
RecodeLog.error("执行命令异常：{0},原因:{1}".format(command, error))
return False
def create_table(self, sql):
"""
:param sql:
:return:
"""
try:
self.cursor.execute(sql)
RecodeLog.info("创建表class {0},function create_table,成功,{1}".format(
self.__class__.__name__,
sql
))
return True
except Exception as error:
RecodeLog.error("创建表失败class {0},function create_table,{1},失败,{2}".format(
self.__class__.__name__,
sql,
error
))
return False
def rename_record(archives_name, new_archives):
"""
:param archives_name:
:param new_archives
:return:
"""
archives_name = os.path.join(ROOT_DIR, archives_name)
new_archives = os.path.join(ROOT_DIR, new_archives)
try:
os.rename(archives_name, new_archives)
RecodeLog.info(
"重命名文件 class {0},function rename_record,成功,befor={1},after={2}"
.format("RecordData", archives_name, new_archives))
return True
except Exception as error:
RecodeLog.error(
"重命名文件 class {0},function rename_record,失败,befor={1},after={2},{3}"
.format("RecordData", archives_name, new_archives, error))
return False
def pg_basedump(self, db_config, params):
"""
:param db_config:
:param params:
:return:
"""
if not isinstance(db_config, str):
raise Exception("输入数据库类型错误！{0}", db_config)
pg_basebackup = os.path.join(EXEC_BIN, 'pg_basebackup')
if not os.path.exists(pg_basebackup):
raise EnvironmentError("可执行命令不存在: {0}".format(pg_basebackup))
pg_params = copy.deepcopy(DB_CONFIG_DICT[db_config])
ipaddress = self.get_address()
pg_password = pg_params.pop('password')
pg_database = pg_params.pop('database')
dump_params = "export PGPASSWORD={0} && {1} {2}".format(
pg_password, pg_basebackup, copy.deepcopy(params))
rsync_params = copy.deepcopy(RSYNC_CONFIG_DICT[db_config])
for key, value in pg_params.items():
dump_params = "{0} --{1}={2}".format(dump_params, key, value)
achieve = os.path.join(
BACKUP_DIR, "{0}_all-dump_{1}".format(
ipaddress,
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")))
if not os.path.exists(achieve):
os.makedirs(achieve)
pg_basedump_str = "{0} -D {1} ".format(dump_params, achieve)
rsync_params['achieve'] = achieve
if not self.exec_command(command=pg_basedump_str):
RecodeLog.error(msg="全量备份异常！{0}".format(pg_basedump_str))
return False
else:
RecodeLog.info(msg="全量备份成功！{0}".format(pg_basedump_str))
self.rsync_dump(**rsync_params)
return True
def insert(self, sql, db):
"""
:param sql:
:param db:
:return:
"""
if not isinstance(sql, str):
RecodeLog.error("输入类型错误 class {0},function insert,{1}失败".format(
self.__class__.__name__,
sql
))
return False
if 'INSERT INTO' not in sql.upper():
RecodeLog.error("sql 错误 class {0},function insert,{1}失败".format(
self.__class__.__name__,
sql
))
return False
if "CREATE TABLE" in sql.upper():
pass
if not self.check_db(db=db):
self.create_database(db=db)
try:
self.cursor.execute("use %s" % db)
self.cursor.execute(sql)
RecodeLog.info("class {0},function insert,{1}成功".format(
self.__class__.__name__,
sql
))
return True
except Exception as error:
RecodeLog.error("class {0},function insert,{1}失败,{2}".format(
self.__class__.__name__,
sql,
error
))
return False
def complete(self,
env,
namespace,
service,
version,
port_type,
env_args,
replicas=1):
"""
:param env:
:param namespace:
:param service:
:param version:
:param port_type:
:param env_args:
:param replicas:
:return:
"""
# 编译程序所在目录
service_path = os.path.join(WORKSPACE, service)
# 镜像标签
tag = os.path.join(HARBOR_URL, service)
# 获取到的容器内部存在的node_port
node_port = self.get_service_nodeports(env=env)
# ##############镜像相关的操作#####################
front_app = os.path.join(service_path, 'package.json')
backend_app = os.path.join(service_path, 'pom.xml')
# front_docker_path = os.path.join(service_path, 'docker')
if os.path.exists(front_app) and os.path.exists(backend_app):
RecodeLog.error(msg="{2}.同时出现前端和后端的配置：{0},{1}".format(
front_app, backend_app, service))
return False
elif os.path.exists(front_app):
RecodeLog.info(msg="{0},应用为前端应用,开始编译....".format(service))
exec_npm_str = "cd {0} && yarn".format(service_path)
if env in ['qa', 'dev']:
exec_npm_run_str = "cd {0} && yarn build:{1}".format(
service_path, env)
else:
exec_npm_run_str = "cd {0} && yarn build".format(
service_path, env)
try:
self.exec_command(command=exec_npm_str)
RecodeLog.info("执行编译成功：{0}".format(exec_npm_str))
self.exec_command(command=exec_npm_run_str)
RecodeLog.info("执行编译成功：{0}".format(exec_npm_run_str))
self.docker_image_build(path=service_path,
tag=tag,
version=version)
RecodeLog.info(msg="生成前端镜像成功:{0}:{1}".format(tag, version))
self.docker_image_push(
repository="{0}:{1}".format(tag, version))
RecodeLog.info(msg="推送前端镜像成功:{0}:{1}".format(tag, version))
self.docker_local_image_remove(
image_name="{0}:{1}".format(tag, version))
RecodeLog.info(msg="删除镜像成功：{0}:{1}".format(tag, version))
except Exception as error:
RecodeLog.error("执行编译异常，原因：{0}".format(error))
return False
elif env in ['dev'] and os.path.exists(backend_app):
try:
self.docker_image_build(path=service_path,
tag=tag,
version=version)
RecodeLog.info(msg="生成后端镜像成功:{0}:{1}".format(tag, version))
self.docker_image_push(
repository="{0}:{1}".format(tag, version))
RecodeLog.info(msg="推送后端镜像成功:{0}:{1}".format(tag, version))
self.docker_local_image_remove(
image_name="{0}:{1}".format(tag, version))
RecodeLog.info(msg="删除镜像成功：{0}:{1}".format(tag, version))
except Exception as error:
RecodeLog.error("执行编译异常，原因：{0}".format(error))
return False
ports = self.get_port(path=service_path)
if not self.make_service_yaml(namespace=namespace,
service_name=service,
port_type=port_type,
ports=ports,
exist_node_port=node_port,
env=env):
return False
if not self.make_deploy_yaml(namespace=namespace,
service_name=service,
image_tag=tag,
ports=ports,
env=env,
env_args=env_args,
version=version,
replicas=replicas):
return False
return True
def make_service_yaml(self, namespace, service_name, port_type,
exist_node_port, ports, env):
"""
:param namespace:
:param service_name:
:param port_type:
:param exist_node_port:
:param ports:
:param env:
:return:
"""
if len(ports) == 0:
RecodeLog.warn(msg="没有端口")
return True
node_ports = list()
service_achieve = os.path.join(SERVICE_YAML_DIR,
"service_{0}.yaml".format(service_name))
if os.path.exists(service_achieve):
RecodeLog.info(msg="Service:{1}，服务对应的配置文件：{0}，已存在，跳过创建！".format(
service_achieve, service_name))
# 读取已经存在的yaml文件
service_yaml = self.read_yaml(achieve=service_achieve)
# 执行已经存在的yaml文件内容
if not self.apply_service(data=service_yaml,
namespace=namespace,
env=env,
name=service_name):
return False
return True
for port in ports:
if port_type == "NodePort":
node_ports.append({
'name':
"{0}-{1}".format(service_name, port),
'nodePort':
self.make_node_port(data=exist_node_port,
service_name=service_name,
namespace=namespace),
'port':
int(port),
'protocol':
'TCP',
'targetPort':
int(port)
})
else:
node_ports.append({
'name': "{0}-{1}".format(service_name, port),
'port': int(port),
'protocol': 'TCP',
'targetPort': int(port)
})
service_yaml = self.format_service(namespace=namespace,
node_port_set=node_ports,
port_type=port_type,
service_name=service_name)
if not self.write_yaml(achieve=service_achieve, data=service_yaml):
RecodeLog.error(msg="生成yaml文件失败：{0}".format(service_yaml))
# raise Exception("生成yaml文件失败：{0}".format(service_yaml))
return False
if not self.apply_service(
data=service_yaml, namespace=namespace, env=env,
name=service_name):
return False
return True
def command_load(self, db, table, data_file, pro_root):
"""
:param pro_root:
:param db:
:param table:
:param data_file:
:return:
"""
HIVE_BIN = os.path.join(pro_root, 'bin', 'hive')
archives = os.path.join(ROOT_DIR, data_file)
# 带分区的表结构
if HIVE_TABLE_PARTITION:
exec_str = '''{0} -e "use {3}; LOAD DATA LOCAL INPATH '{1}' INTO TABLE \`{2}\` partition ({5}='{4}')"'''.format(
HIVE_BIN,
archives,
table,
db,
time.strftime("%Y%m%d", time.localtime()),
HIVE_TABLE_PARTITION_NAME
)
# 不带分区的表结构
else:
exec_str = '''{0} -e "use {3}; LOAD DATA LOCAL INPATH '{1}' INTO TABLE \`{2}\`"'''.format(
HIVE_BIN,
archives,
table,
db
)
try:
# 创建表
if not self.create_if_not_exist(db=db, table=table):
raise Exception("创建Hive数据库失败")
# 如果有表分区，修改表分区
if not self.alert_table_partition(
db=db,
table=table,
partition_name=HIVE_TABLE_PARTITION_NAME,
partition_type=HIVE_TABLE_PARTITION_TYPE
):
raise Exception("""修改分区信息失败""")
RecodeLog.info("class {0},function command_load,开始导入：{1}".format(
self.__class__.__name__,
exec_str
))
if int(platform.python_version().strip(".")[0]) < 3:
status, msg = commands.getstatusoutput(exec_str)
else:
status, msg = subprocess.getstatusoutput(exec_str)
if status != 0:
raise Exception(msg)
new_name = "{0}.success".format(os.path.splitext(data_file)[0])
RecordData.rename_record(archives_name=data_file, new_archives=new_name)
RecodeLog.info("class {0},function command_load,导入完成：{1}成功".format(
self.__class__.__name__,
exec_str
))
return True
except Exception as error:
new_name = "{0}.error".format(os.path.splitext(data_file)[0])
RecordData.rename_record(archives_name=data_file, new_archives=new_name)
RecodeLog.error("class {0},function command_load,{1}失败,{2}".format(
self.__class__.__name__,
exec_str,
error
))
return False
def service_record_table(self, data):
"""
:param data:
:return:
"""
error_archives = "{0}.error".format(
time.strftime("%Y%m%d%H", time.localtime()))
if not isinstance(data, dict):
RecodeLog.error(
"产生表结构数据 class {0},function service_record_table,失败,输入参数异常，请检查，{1}"
.format("RecordData", data))
return False
if data['type'] != "INSERT":
return True
# if data['type'] not in ("INSERT", "UPDATE", "DELETE", "ERASE", "ALTER"):
# return True
database = data['database']
table = data['table']
prepare_name = "{0}-{3}-{1}-{2}.pre".format(
time.strftime("%Y%m%d%H%M", time.localtime()),
database,
table,
int(time.time() * 1000000),
)
# 处理 DROP ALERT TABLE
if data['type'] in ['ALTER', "ERASE"]:
sql = "ALTER TABLE `{0}`.`{1}` RENAME TO `{0}`.`{1}_{2}`".format(
database, table, time.strftime("%Y%m%d%H%M%S",
time.localtime()))
archives_name = "{2}TableSQL-{3}-{0}-{1}.sql".format(
database,
table,
data['type'],
int(time.time() * 1000000) # 生成检查时间字段
)
# in
if not self.touch_record(archives_name=archives_name):
RecodeLog.error(
"创建 {2} 失败 class {0},function service_record_table，请检查，{1}"
.format("RecordData", sql, archives_name))
return False
if not self.write_record(archives_name=archives_name, data=sql):
RecodeLog.error(
"写入 {2} 失败 class {0},function service_record_table，请检查，{1}"
.format("RecordData", sql, archives_name))
return False
prepare_first_list_alert = self.get_list_record(
archives_regular="*-{0}-{1}.pre".format(database, table))
# 强制修改pre文件为standby
if len(prepare_first_list_alert) != 0:
for i in prepare_first_list_alert:
# 将文件重命名为 准备状态
archives = os.path.splitext(i)[0]
new_name = "{0}.standby".format(archives)
self.rename_record(archives_name=i, new_archives=new_name)
return True
# 产生表结构文件
if not self.make_create_table_data(
db=database, table=table, data=data['mysqlType']):
RecodeLog.error(
"产生表结构数据 class {0},function service_record_table，请检查，{1}".
format("RecordData", str(data['mysqlType'])))
self.write_record(archives_name=error_archives, data=data)
# 以下为检查文件并按照需求生成新的pre文件
# 先获取数据目录下文件列表
prepare_first_list = self.get_list_record(archives_regular="*.pre")
if len(prepare_first_list) == 0:
self.touch_record(archives_name=prepare_name)
RecodeLog.info(
"即将生成pre文件 class {0},function service_record_table,archives_name={1}"
.format(self.__class__.__name__, prepare_name))
else:
for i in prepare_first_list:
if not self.check_record_time(archives_name=i,
time_offset=OFFSET_TIME,
max_size=OFFSET_SIZE):
# 将文件重命名为 准备状态
archives = os.path.splitext(i)[0]
new_name = "{0}.standby".format(archives)
self.rename_record(archives_name=i, new_archives=new_name)
# 再次获取可以写入的文件名
prepare_list = self.get_list_record(archives_regular="*.pre")
if len(prepare_list) > 0:
prepare_archive = prepare_list[0]
else:
# 执行异常将数据写入错误文件中
# 产生一个新文件
prepare_archive = self.touch_record(archives_name=prepare_name)
if not prepare_archive:
self.write_record(archives_name=error_archives, data=data)
return False
if not self.check_archives_write(archives_name=prepare_archive):
# 执行异常将数据写入错误文件中
self.write_record(archives_name=error_archives, data=data)
return False
# 写入数据到文件
if data['data'] == 'null' or not data['data']:
self.write_record(archives_name=error_archives, data=str(data))
return False
for x in range(0, len(data['data'])):
# 以下为处理 DELETE,UPDATE,INSERT 三个事务的操作
new_value = data['data'][x]
new_value_str = ",".join('%s' % x for x in new_value.values())
if data['type'] == "DELETE":
record_line = "{0},-1".format(new_value_str)
elif data['type'] == "INSERT":
# record_line = "{0},1".format(new_value_str)
record_line = new_value_str
else:
old_value = data['old'][x]
record_line = "{0},1".format(new_value_str)
for key, value in old_value.items():
new_value[key] = value
new_value = "{0},-1".format(",".join(new_value.values()))
record_line = "{0}\n{1}".format(record_line, new_value)
self.write_record(archives_name=prepare_archive, data=record_line)
