def SetHaproxyMeta(self, group, reads, master, type=None):
'''加入及master变动设置haproxy配置信息'''
node_path = '{}/{}'.format(GetConf().GetHaproxy(), group)
if type is None:
meta_path = GetConf().GetMetaGroup()
if reads is None:
hosts = self.Get('{}/{}'.format(meta_path, group))
reads = hosts.split(',')
_reads = []
for host in reads:
host_meta = self.GetMeta(type='host', name=host)
host_port = '%s:%d' % (host.replace(
'-', '.'), int(eval(host_meta)['port']))
_reads.append(host_port)
master_host_meta = self.GetMeta(type='host', name=master)
_master = '%s:%d' % (master.replace(
'-', '.'), int(eval(master_host_meta)['port']))
value = '{"read":"%s","write":"%s"}' % (_reads, _master)
state = self.Exists(node_path)
if state is None:
self.Create(path=node_path, value=value, seq=False)
else:
self.Set(path=node_path, value=value)
return True
else:
value = '{"read":"%s","write":"%s"}' % (reads, master)
self.Set(path=node_path, value=value)
return True
def CreateWatch(self,
host,
addition=None,
region=None,
region_for_groupname=None):
'''创建watch,触发时写入task节点'''
online_host_path = GetConf().GetOnlinePath()
_group_name = self.GetMeta(type='host', name=host)
group_name = eval(
_group_name)['group'] if _group_name else region_for_groupname
online_state = self.zk.exists('{}/{}'.format(online_host_path, host))
Logging(msg='master watch :{}'.format(host), level='info')
if online_state is not None:
@self.zk.DataWatch('{}/{}'.format(online_host_path, host))
def my_func(data, stat):
if data is None:
self.CreateDownTask(group_name,
addition=addition,
region=region)
Logging(msg='master({}) has been down!'.format(host),
level='error')
self.zk.stop()
sys.exit()
else:
_name = group_name + '_' + region if region else group_name
Logging(msg="this master {} node not exists".format(host),
level='error')
state = self.Exists('{}/{}'.format(GetConf().GetWatchDown(),
_name))
self.Create(path='{}/{}'.format(GetConf().GetWatchDown(), _name),
value="master not online",
seq=False) if state is None else None
self.zk.stop()
def AddMeta(groupname,hosts):
'''对已经存在的集群增加节点元数据'''
_hosts = hosts.split(':')
group_path = GetConf().GetMetaGroup()
host_path = GetConf().GetMetaHost()
with closing(zkHander()) as zkhander:
group_hosts = zkhander.GetMeta(type='group',name=groupname)
zkhander.Set(path=group_path+'/'+groupname,value=group_hosts+','+_hosts[0].replace('.','-'))
value = {'group':groupname,'port':_hosts[1]}
zkhander.Create(path=host_path+'/'+_hosts[0].replace('.','-'),value=str(value),seq=False)
def __init__(self,host,port):
self.host,self.port = host,int(port)
self.mysqluser,self.mysqlpasswd = GetConf().GetMysqlAcount()
ssl_set = {'ca':GetConf().GetUserSSLCa(),'cert':GetConf().GetUserSSLCert(),'key':GetConf().GetUserSSLKey()}
try:
self.local_conn = MySQLdb.connect(host=self.host, user=self.mysqluser, passwd=self.mysqlpasswd, port=self.port, db='',
charset="utf8mb4",ssl=ssl_set)
self.mysql_cur = self.local_conn.cursor()
self.state = True
except MySQLdb.Error,e:
Logging(msg=traceback.format_exc(),level='error')
self.state = False
def GetMeta(self, **kwargs):
'''获取元数据信息'''
if kwargs['type'] == 'group':
node_path = '{}/{}'.format(GetConf().GetMetaGroup(),
kwargs['name'])
return self.Get(
node_path) if self.Exists(node_path) != None else False
elif kwargs['type'] == 'host':
node_path = '{}/{}'.format(GetConf().GetMetaHost(), kwargs['name'])
return self.Get(
node_path) if self.Exists(node_path) != None else False
else:
Logging(msg="type is error ,only 【group ,host】", level='error')
raise "type is error ,only 【group ,host】"
def __get_online_host(self, region):
"""获取在线列表"""
reg_path = GetConf().GetAdditionRegion() + '/' + region
with closing(zkHander()) as zkhander:
reg_value_dict = eval(
zkhander.Get(reg_path)
) #region存储格式为{'192-168-212-1':{'port':333,'ssl':0/1}.....}
if reg_value_dict:
_reg_online = [
host for host in reg_value_dict
if zkhander.GetOnlineState(host)
]
if _reg_online:
_to_reg = {
'host': _reg_online[0].replace('-', '.'),
'port': reg_value_dict[_reg_online[0]]['port'],
'ssl': reg_value_dict[_reg_online[0]]['ssl']
}
return _to_reg
else:
Logging(
msg=
'This group has replication task ,But all region not online',
level='warning')
return None
else:
Logging(
msg='This group has replication task ,But not region value',
level='warning')
return None
def __init__(self, host, port):
self.host, self.port = host, port
self.mysqluser, self.mysqlpasswd = GetConf().GetMysqlAcount()
self.ssl_set = {
'ca': GetConf().GetUserSSLCa(),
'cert': GetConf().GetUserSSLCert(),
'key': GetConf().GetUserSSLKey()
}
self.conn = pymysql.connect(host=self.host,
user=self.mysqluser,
passwd=self.mysqlpasswd,
port=self.port,
db='',
charset="utf8mb4",
ssl=self.ssl_set)
def __get_master_for_region(self, region, groupname):
with closing(zkHander()) as zkhander:
repl_path = GetConf().GetAdditionRPL(
) + '/' + groupname + '/' + region
master_content = eval(zkhander.Get(repl_path))
host, port = master_content['host'], int(master_content['port'])
return host, port
def Region_meta(self,region):
"""能提供同步的元数据"""
_host_content = {}
for host in self.host_content:
_host_content[host.replace('.','-')] = self.host_content[host]
with closing(zkHander()) as zkhander:
path = GetConf().GetAdditionRegion() + '/' + region
zkhander.Create(path=path,value=str(_host_content),seq=False,mp=True)
def SetMaster(self, groupname, host, onlywatch=None):
if onlywatch:
value = [groupname, host.replace('.', '-'), 'white', 'onlywatch']
else:
value = [groupname, host.replace('.', '-'), 'white']
path = GetConf().GetTaskPath()
zkHander().DeleteWhite(groupname)
return zk.create(path=path + '/task', sequence=True, value=str(value))
def AddRoute(groupname,route):
'''添加路由'''
route_path = GetConf().GetRouter()
with closing(zkHander()) as zkhander:
stat = zkhander.Exists(route_path+'/'+groupname)
if stat is None:
zkhander.Create(path=route_path+'/'+groupname,value=route,seq=False)
else:
print 'this route already exists'
def Run():
SlaveDownCheck() #检查是否有已存在的宕机节点
zkHander().CreateChildrenWatch(path=GetConf().GetSlaveDown(),
func=ManageDownNode)
with closing(zkHander()) as zkhander:
while True:
SlaveCheck(zkhander).WhileCheckSLave()
time.sleep(3) #每3秒扫描一次slave在线状态
def CreateLockWatch(self, taskname):
'''用于任务在其他节点执行，本节点对锁监听，当删除时获取新master启动watch'''
path = '{}/{}'.format(GetConf().GetLockPath(), taskname)
@self.zk.DataWatch(path)
def my_func(data, stat):
if data is None:
masterhost = self.GetMasterMeta(taskname)
self.CreateWatch(masterhost)
sys.exit()
def __checklock(self, taskname):
"""循环检查lock是否存在，当不存在时对新master建立监听"""
import time
path = '{}/{}'.format(GetConf().GetLockPath(), taskname)
while True:
state = self.Exists(path)
if state is None:
return True
break
time.sleep(0.5)
def TaskCheck(self):
'''获取task任务列表，执行并删除任务,并创建task任务实时监听'''
task_path = GetConf().GetTaskPath()
with closing(zkHander()) as zkhander:
task_list = zkhander.GetTaskList()
if task_list:
for task in task_list:
self.TaskFunc(task)
zkHander().CreateChildrenWatch(task_path, self.TaskFunc)
else:
zkHander().CreateChildrenWatch(task_path,self.TaskFunc)
def InsertClusterMeta(groupname,hosts):
'''插入新集群元数据'''
_hosts = hosts.split(',')
_host_list = []
_host_port = {}
for host in _hosts:
_host_port[host.split(':')[0]] = host.split(':')[1]
_host_list.append(host.split(':')[0])
group_path = GetConf().GetMetaGroup()
host_path = GetConf().GetMetaHost()
with closing(zkHander()) as zkhander:
if zkhander.Exists(path=group_path+'/'+groupname) is None:
zkhander.Create(path=group_path+'/'+groupname,value=','.join(_host_list).replace('.','-'),seq=False)
else:
print '%s is already exists' % groupname
for host in _host_port:
value = {'group':groupname,'port':_host_port[host]}
zkhander.Create(path=host_path+'/'+host.replace('.','-'),value=str(value),seq=False)
def SetHa(groupname,host):
'''宕机恢复后修改haproxy配置文件，使恢复的节点接收查询'''
path = GetConf().GetHaproxy()
_value = zkHander().Get(path+'/'+groupname)
value = eval(_value)
meta_path = GetConf().GetMetaHost()
_port = zkHander().Get(meta_path+'/'+host.replace('.','-'))
port = eval(_port)['port']
_add_host = host+':'+str(port)
_read_list = eval(value['read'])
if _add_host not in _read_list:
_read_list.append(host+':'+str(port))
value['read'] = _read_list
print value
zkHander().SetHaproxyMeta(groupname, value['read'], value['write'], type=1)
SendRoute(groupname)
return "OK"
else:
print 'this host already exists'
def Rel_Meta(self, groupname, region):
"""同步任务需要同步的节点信息"""
host_content = {
'host': self.host,
'port': self.port,
'ssl': self.ssl
} if self.host and self.port and self.ssl else None
with closing(zkHander()) as zkhander:
path = GetConf().GetAdditionRPL() + '/' + groupname + '/' + region
zkhander.Create(path=path,
value=str(host_content),
seq=False,
mp=True)
def __check_repl(self, groupname):
repl_path = GetConf().GetAdditionRPL() + '/' + groupname
region_value = {}
with closing(zkHander()) as zkhander:
state = zkhander.Exists(repl_path)
if state:
region_list = zkhander.GetChildren(repl_path)
for region in region_list:
region_value[region] = zkhander.Get(repl_path + '/' +
region)
return region_value
else:
return None
def SetLockTask(self, taskname):
'''创建锁文件，用户多点运行server控制任务'''
path = GetConf().GetLockPath()
if self.Exists(path='{}/{}'.format(path, taskname)) is None:
try:
time.sleep(random.uniform(0, 1))
self.zk.create(path='{}/{}'.format(path, taskname),
value=b'',
ephemeral=False)
return True
except Exception, e:
Logging(msg=traceback.format_exc(), level='error')
return False
def __set_group_region(self, region, host_content):
"""设置该groupname连接对应region的master信息"""
with closing(zkHander()) as zkhander:
host_meta = eval(
zkhander.GetMeta(name=self.host.replace('.', '-'),
type='host'))
groupname = host_meta['group']
path = GetConf().GetAdditionRPL() + '/' + groupname + '/' + region
zkhander.Set(path=path, value=str(host_content))
zkHander().CreateWatch(host=host_content['host'].replace('.', '-'),
addition=True,
region=region,
region_for_groupname=groupname)
with closing(zkHander()) as zkhander:
zkhander.DeleteWatchDown(groupname=groupname + '_' + region)
def CreateDownTask(self, groupname, addition=None, region=None):
task_path = '{}/{}'.format(GetConf().GetTaskPath(), groupname)
if self.Exists(task_path) is None:
time.sleep(random.uniform(0, 1))
try:
if addition:
task_path += '_' + region
self.Create(path=task_path,
value=str([groupname, region, 'append',
'dow']),
seq=False) #跨区域同步主机宕机
else:
self.Create(path=task_path,
value=str([groupname, 'down']),
seq=False)
except:
Logging(msg='Existing outage task for {}'.format(groupname),
level='info')
else:
Logging(msg='Existing outage task for {}'.format(groupname),
level='info')
def ChangeMaster(self,host,port):
'''主从指向'''
repluser,replpassword,ssl_ca,ssl_cert,ssl_key = GetConf().GetReplAcount()
try:
sql = 'reset slave all;'
print self.host
try:
self.mysql_cur.execute(sql)
except:
self.mysql_cur.execute('stop slave')
self.mysql_cur.execute(sql)
change_sql = 'change master to master_host="%s",master_port=%s,master_user="******",master_password="******",master_auto_position=1 for channel "default"' % (host,int(port),repluser,replpassword)
self.mysql_cur.execute(change_sql)
self.__set_variables(type='slave')
return True
except MySQLdb.Warning,e:
start_sql = 'start slave'
self.mysql_cur.execute(start_sql)
self.__set_variables(type='slave')
Logging(msg='Change master to {} state : Warning'.format(host),level='warning')
Logging(msg=traceback.format_exc(),level='warning')
return True
def Change(self, region, host_content):
repluser, replpassword, ssl_ca, ssl_cert, ssl_key = GetConf(
).GetReplAcount(rg=True)
master_host, master_port = host_content['host'], int(
host_content['port'])
if host_content['ssl']:
sql = 'change master to master_host="%s",master_port=%d,master_user="******",master_password="******",master_ssl=1,' \
'master_ssl_ca="%s",' \
'master_ssl_cert="%s",master_ssl_key="%s",master_auto_position=1 for channel "%s" ' % (
master_host, master_port,
repluser, replpassword, ssl_ca,
ssl_cert, ssl_key, region)
else:
sql = 'change master to master_host="%s",master_port=%d,master_user="******",master_password="******",' \
'master_auto_position=1 for channel "%s" ' % (master_host, master_port,
repluser, replpassword, region)
with closing(self.conn.cursor()) as cur:
try:
cur.execute('stop slave for channel "%s"' % region)
cur.execute('reset slave for channel "%s"' % region)
except:
pass
try:
cur.execute(sql)
self.__set_group_region(region, host_content)
except pymysql.Warning, e:
Logging(msg=traceback.format_exc(), level='error')
cur.execute('start slave;')
self.__set_group_region(region, host_content)
except pymysql.Error, e:
Logging(msg=traceback.format_exc(), level='error')
Logging(
msg=
'addition task for {} failed,master to {} in region {} ! ! !'
.format(self.host, host_content['host'], region),
level='error')
return False
def CheckOnlineClient(self, host):
'''检查mysql客户端程序是否在线'''
_path = '{}/{}'.format(GetConf().GetOnlineClientPath(), Replace(host))
return self.Exists(_path)
def ManageDownNode(host):
slave_down_path = GetConf().GetSlaveDown()
with closing(zkHander()) as zkhander:
result = zkhander.Get(path='{}/{}'.format(slave_down_path, host))
SlaveCheck().StaticInfo(result=eval(result), host=host)
def __init__(self, zkhander=None):
self.online_node = GetConf().GetOnlinePath()
self.slave_down_path = GetConf().GetSlaveDown()
self.zkhander = zkhander
def ResetMaster(self,groupname):
try:
'''获取当前binlog读取位置'''
append_stat=None
master_log_file,read_master_log_pos,master_host = self.CheckPos(get_host=True)
'''================'''
#用于mysql宕机，服务器在线追加数据
from zk_handle.zkHandler import zkHander
from Append.AppendValue import Append
from lib.get_conf import GetConf
from contextlib import closing
with closing(zkHander()) as zkhander:
client_stat = zkhander.CheckOnlineClient(master_host)
if client_stat:
__get_content = {'getbinlog': 10010, 'binlog_file': master_log_file, 'start_position': read_master_log_pos}
Logging(msg='gets the unsynchronized data not. info:{}'.format(__get_content),level='info')
append_stat = Append(connection=self.local_conn,cursor=self.mysql_cur,host=master_host,port=GetConf().GetClientPort()).receive(conn_info=str(__get_content))
if append_stat:
Logging(msg='Append OK',level='info')
else:
Logging(msg='Append Failed',level='error')
'''================='''
if master_host:
readbinlog_status = str([groupname,master_log_file,read_master_log_pos])
execute_gtid = str([groupname,self.__CetGtid()])
with closing(zkHander()) as zkhander:
if append_stat:
zkhander.SetExecuteGtid(master_host, execute_gtid)
else:
zkhander.SetReadBinlog(master_host,readbinlog_status)
zkhander.SetExecuteGtid(master_host,execute_gtid)
''''''
#self.mysql_cur.execute('set global read_only=0;')
self.mysql_cur.execute('stop slave')
self.mysql_cur.execute('reset slave all;')
self.__set_variables(type='master')
except MySQLdb.Warning,e:
Logging(msg=traceback.format_exc(),level='warning')
self.mysql_cur.execute('reset slave all;')
self.__set_variables(type='master')
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@author: Great God
'''
import sys
sys.path.append("..")
from lib.SendRoute import SendRoute
from kazoo.client import KazooClient
from lib.get_conf import GetConf
from zk_handle.zkHandler import zkHander
from contextlib import closing
import getopt
zk = KazooClient(hosts=GetConf().GetZKHosts())
zk.start()
class White():
'''手动操作，先执行setwhite再对master进行操作，完成后需执行setmaster'''
def __init__(self):
pass
def SetWhite(self, groupname):
path = GetConf().GetWhitePath()
return zkHander().Create(path=path + '/' + groupname,
value='',
seq=False)
def SetMaster(self, groupname, host, onlywatch=None):
if onlywatch:
def SetWhite(self, groupname):
path = GetConf().GetWhitePath()
return zkHander().Create(path=path + '/' + groupname,
value='',
seq=False)