def _post(self, component=None, pattern=None, node=None, role=None):
# if 'pd_servers' in self._diff:
# reload_pd = True
# else:
# reload_pd = False
self.topology.replace(self._new_topo)
term.info('Update configuration.')
ans = ansibleapi.ANSRunner(user=self.topology.user,
topology=self.topology._topology(
self._new_topo),
tiargs=self._args)
act = Action(ans=ans, topo=self.topology)
if 'pd_servers' in self._diff:
act.deploy_component(component='pd', pattern='pd_servers')
act.deploy_component(component='tikv', pattern='tikv_servers')
act.deploy_component(component='tidb', pattern='tidb_servers')
act.deploy_component(component='pump', pattern='pump_servers')
act.deploy_component(component='drainer',
pattern='drainer_servers')
act.deploy_component(component='prometheus',
pattern='monitoring_server')
act.stop_component(component='prometheus', pattern='monitoring_server')
act.start_component(component='prometheus',
pattern='monitoring_server')
term.notice('Finished scaling out.')
def _post(self, component=None, pattern=None, node=None, role=None):
ans = ansibleapi.ANSRunner(user=self.topology.user,
topology=self.topology(),
tiargs=self._args)
act = Action(ans=ans, topo=self.topology)
if 'pd_servers' in self._diff:
act.deploy_component(component='pd', pattern='pd_servers')
act.deploy_component(component='tikv', pattern='tikv_servers')
act.deploy_component(component='tidb', pattern='tidb_servers')
act.deploy_component(component='pump', pattern='pump_servers')
act.deploy_component(component='drainer',
pattern='drainer_servers')
# self.deploy.deploy_component(component='prometheus', pattern='monitoring_server', ans=ans)
# self.reload.do(component='prometheus', pattern='monitoring_server')
term.notice('Finished scaling in.')
def init_network(self, demo=False):
term.notice(
'Start creating no-password ssh connections between the management machine and other machines'
)
self._check_ip_list()
# get password from prompt
if not self._args.password:
term.info(
'Please enter the password of init user on deployment server ({} password)'
.format(self.init_user))
_passwd = term.getpass()
else:
_passwd = self._args.password
# create ansible runner
initnet = ansibleapi.ANSRunner(ips=self.hosts,
user=self.init_user,
tiargs=self._args,
passwd=_passwd)
term.info('Create {} user on remote machines.'.format(self.user))
initnet.run_model('user',
'name=%s '
'shell=/bin/bash '
'createhome=yes' % (self.user),
become=True)
term.info('Set authorized_keys for {} on cluster machine.'.format(
self.user))
initnet.run_model('authorized_key',
'user=%s '
'key={{ lookup("file", "%s/id_rsa.pub") }}' %
(self.user, utils.profile_path('.ssh')),
become=True)
term.info('Add sudo permissions for {} on cluster machine.'.format(
self.user))
initnet.run_model('lineinfile',
'path=/etc/sudoers '
'line="{} ALL=(ALL) NOPASSWD: ALL" '
'regexp="^{} .*" '
'insertafter=EOF '
'state=present'.format(self.user, self.user),
become=True)
if demo:
term.notice('Finished setting up SSH keys.')
else:
term.notice('Done!!!')
def __init__(self, args=None, topology=None, demo=False, action=None):
try:
self._lock_profile()
except NotImplementedError:
pass
except exceptions.TiOPSException: # TODO: add more specific handlers
raise
self.topology = topology
self._args = args
if not demo and os.path.exists(
self.topology.topology_file) and not action:
term.warn(
'Check TiDB cluster {} status, it may take a few minutes.'.
format(self.topology.cluster_name))
self.check_tombstone()
self.ans = ansibleapi.ANSRunner(user=self.topology.user,
topology=self.topology(),
tiargs=self._args)
def check_tombstone(self, topology=None, args=None):
if not topology:
topology = self.topology
if not args:
args = self._args
_remove_uuid = []
_cluster = ClusterAPI(topology)
_binlog = BinlogAPI(topology)
if _cluster.tikv_stores() and _cluster.tikv_tombstone():
# get tombstone tikv node
for _node in topology()['tikv_servers']:
_tombstone = False
if not _node['offline']:
continue
# online tikv node list
_online_list = [
x['store']['address']
for x in _cluster.tikv_stores()['stores']
]
# tombstone status tikv list
_tombstone_list = [
x['store']['address']
for x in _cluster.tikv_tombstone()['stores']
]
_address = '{}:{}'.format(_node['ip'], _node['port'])
# if node is online, skip it
if _address in _online_list:
continue
# if node is tombstone, will delete it from topology
elif _address in _tombstone_list:
_remove_uuid.append(_node['uuid'])
if _binlog.pump_status:
# get tombstone pump node
for _node in topology()['pump_servers']:
_tombstone = False
if not _node['offline']:
continue
_online_list = [
x['nodeId']
for x in _binlog.pump_status['status'].itervalues()
if x['state'] != 'offline'
]
_tombstone_list = [
x['nodeId']
for x in _binlog.pump_status['status'].itervalues()
if x['state'] == 'offline'
]
if _node['uuid'] in _online_list:
continue
elif _node['uuid'] in _tombstone_list:
_remove_uuid.append(_node['uuid'])
for _node in topology()['drainer_servers']:
_tombstone = False
if not _node['offline']:
continue
_online_list = [
x['nodeId'] for x in _binlog.drainer_status
if x['state'] != 'offline'
]
_tombstone_list = [
x['nodeId'] for x in _binlog.drainer_status
if x['state'] == 'offline'
]
if _node['uuid'] in _online_list:
continue
elif _node['uuid'] in _tombstone_list:
_remove_uuid.append(_node['uuid'])
if not _remove_uuid:
return
_new_topo, _diff = topology.remove(','.join(_remove_uuid), delete=True)
ans = ansibleapi.ANSRunner(user=topology.user,
topology=_diff,
tiargs=args)
act = Action(ans=ans, topo=topology)
for service in [{
'drainer': 'drainer_servers'
}, {
'pump': 'pump_servers'
}, {
'tikv': 'tikv_servers'
}]:
component, pattern = self.check_exist(service, _diff)
if not component and not pattern:
continue
act.stop_component(component=component, pattern=pattern)
act.destroy_component(component=component, pattern=pattern)
topology.replace(_new_topo)
def init_host(self, demo=False):
term.notice('Begin initializing the cluster machine.')
self._check_ip_list()
inithost = ansibleapi.ANSRunner(ips=self.hosts,
user=self.user,
tiargs=self._args)
inithost.run_model('file',
'path=%s '
'state=directory '
'owner=%s '
'group=%s' % (self.check_dir, self.user, self.user),
become=True)
if not os.path.exists(self.host_vars):
os.mkdir(self.host_vars)
elif not os.path.isdir(self.host_vars):
os.remove(self.host_vars)
os.mkdir(self.host_vars)
# get host environment vars
term.info('Get all host environment vars.')
setup = inithost.run_model('setup', 'gather_subset="all" '
'gather_timeout=120')
# record host environment vars
for _host, _vars in setup['success'].iteritems():
__vars = json.loads(json.dumps(_vars))
utils.write_yaml(
os.path.join(self.host_vars, '{}.yaml'.format(_host)), __vars)
if self.enable_checks:
term.info('Check OS platform.')
self.check_os_platform(setup)
term.info('Check OS version.')
self.check_os_version(setup)
# check cpu if support EPOLLEXCLUSIVE
term.info('Check if CPU support EPOLLEXCLUSIVE.')
inithost.run_model('copy',
'src=%s/{{ item }} '
'dest=%s '
'mode=0755' %
(self.script_path, self.check_dir),
with_items=['epollexclusive', 'run_epoll.sh'])
check_epoll = inithost.run_model(
'shell', '%s/run_epoll.sh' % (self.check_dir))
_unsupport_epoll = [
host for host, info in check_epoll['success'].iteritems()
if 'True' not in info['stdout']
]
if _unsupport_epoll:
raise exceptions.TiOPSRuntimeError(
'CPU unsupport epollexclusive on {} machine, please change system version or upgrade kernel verion.'
.format(','.join(_unsupport_epoll).replace('_', '.')),
operation='init')
# check systemd service version (systemd package, not operation system)
_systemd = inithost.run_model('yum', 'list=systemd', become=True)
term.info('Check systemd service version.')
self.check_systemd_version(_systemd)
if not demo:
# check and set cpu mode to performance
term.info('Set CPU performance mode if support.')
# enable cpupower service
inithost.run_model('systemd', 'name="cpupower"', 'state=started',
'enabled=yes')
# set timezone
if self.timezone:
inithost.run_model(
'blockinfile', 'path=/home/%s/.bash_profile '
'insertbefore="# End of file" '
'block="export TZ=%s"' % (self.user, self.timezone))
# install ntp package
if self.ntp_server:
term.info('Install ntpd package.')
inithost.run_model('yum', 'name="ntp" '
'state=present',
become=True)
term.info('Add ntp servers to ntpd config and reload.')
_ntp_config = [
'server {} iburst'.format(server)
for server in self.ntp_server.split(',')
]
inithost.run_model('blockinfile',
'path="/etc/ntp.conf" '
'insertbefore="# End of file" '
'marker="#{mark} TiDB %s MANAGED BLOCK" '
'block="%s"' %
(self.user, '\n'.join(_ntp_config)),
become=True)
inithost.run_model('systemd', 'name=ntpd '
'state=restarted '
'enabled=yes',
become=True)
# check if time synchronize
if self.enable_check_ntp:
term.info('Check if NTP is running to synchronize the time.')
result = inithost.run_model(
'shell', 'ntpstat | grep -w synchronised | wc -l')
_unrun = [
host for host, info in result['success'].iteritems()
if info['stderr']
]
if _unrun:
raise exceptions.TiOPSRuntimeError(
'Ntp server may stopped on {} machine.'.format(
','.join(_unrun).replace('_', '.')),
operation='init')
_unsync = [
host for host, info in result['success'].iteritems()
if info['stdout'] == str(0)
]
if _unsync:
raise exceptions.TiOPSRuntimeError(
'Time unsynchronised on {}, please check ntp status.'.
format(','.join(_unsync).replace('_', '.')),
operation='init')
term.info('Update kernel parameters on cluster machine.')
inithost.run_model('sysctl',
'name={{ item.name }} value={{ item.value }}',
become=True,
with_items=[{
'name': 'net.ipv4.tcp_tw_recycle',
'value': 0
}, {
'name': 'net.core.somaxconn',
"value": 32768
}, {
'name': 'vm.swappiness',
"value": 0
}, {
'name': 'net.ipv4.tcp_syncookies',
"value": 0
}, {
'name': 'fs.file-max',
"value": 1000000
}])
inithost.run_model(
'blockinfile',
'path="/etc/security/limits.conf" '
'insertbefore="# End of file" '
'marker="#{mark} TiDB %s MANAGED BLOCK" '
'block="%s soft nofile 1000000\n'
'%s hard nofile 1000000\n'
'%s soft stack 10240\n"' %
(self.user, self.user, self.user, self.user),
become=True)
if not self.enable_swap:
term.info('Turn off swap on remote machine.')
inithost.run_model('shell', 'swapoff -a', become=True)
else:
term.info('Turn on swap on remote machine')
inithost.run_model('shell', 'swapon -a', become=True)
# disable selinux
term.info('Disable selinux.')
inithost.run_model('selinux', 'state=disabled', become=True)
# set and start irqbalance
if not demo and not self.disable_irqbalance:
term.info('Set and start irqbalance.')
inithost.run_model('lineinfile', 'dest=/etc/sysconfig/irqbalance '
'regexp="(?<!_)ONESHOT=" '
'line="ONESHOT=yes"',
become=True)
inithost.run_model('systemd', 'name=irqbalance.service '
'state=restarted '
'enabled=yes',
become=True)
if demo:
term.notice('Finished init deployment machine.')
else:
term.notice('Done!!!')