def __init__(self, server, appid, appkey):
# NOTE(yikun): There are two reason we only allow this cmd is executed
# in zuul node:
# 1. The app installation interface has been completely supported by
# zuul github driver (but PyGithub is not supported yet), so we use
# it directly to avoid to build the duplicate wheels.
# 2. The app key which is usually only existed in Zuul node.
try:
from zuul.driver.github.githubconnection import GithubConnection
from zuul.driver.github import GithubDriver
except ImportError:
raise exceptions.ClientError(
"Error: 'openlab repo list' only can be used in Zuul node.")
try:
driver = GithubDriver()
connection_config = {
'server': server,
'app_id': appid,
'app_key': appkey,
}
self.conn = GithubConnection(driver, 'github', connection_config)
self.conn._authenticateGithubAPI()
self.conn._prime_installation_map()
except Exception:
raise exceptions.ClientError(
"Failed to load repo list. Please check the specified"
" args:\n--server: %s\n--app-id: %s\n--app-key: %s\n"
"See 'openlab repo list -h' to get more info." % (
server, appid, appkey))
def update_node(self, node_name, maintain=None, role=None, **kwargs):
path = '/ha/%s' % node_name
node_obj = self.get_node(node_name)
if maintain is not None:
if maintain:
if node_obj.status == node.NodeStatus.UP:
node_obj.status = node.NodeStatus.MAINTAINING
else:
raise exceptions.ClientError(
"The node must be in 'up' status when trying to "
"maintain it.")
else:
if node_obj.status == node.NodeStatus.MAINTAINING:
node_obj.status = node.NodeStatus.UP
node_obj.heartbeat = datetime.datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S')
else:
raise exceptions.ClientError(
"The node must be in 'maintaining' status when trying "
"to un-maintain it.")
if role:
node_obj.role = role
switch_status = kwargs.get('switch_status')
if switch_status is not None:
if switch_status.lower() not in ['start', 'end']:
raise exceptions.ClientError(
"switch_status must be 'start', 'end'")
node_obj.update(kwargs)
self.client.set(path, value=node_obj.to_zk_bytes())
node_obj = self.get_node(node_name)
return node_obj
def check(self):
utils.NOCOLOR = self.args.nocolor
cloud_list = self._get_cloud_list(self.args.cloud)
if self.args.type == 'default':
plugins = list(
filter(lambda x: not x.experimental, base.Plugin.plugins))
elif self.args.type == 'all':
plugins = base.Plugin.plugins
else:
# Filter the plugins with specific ptype
plugins = list(
filter(lambda x: x.ptype == self.args.type,
base.Plugin.plugins))
cnt = len(cloud_list)
exit_flag = False
for index, cloud in enumerate(cloud_list):
header = "%s/%s. %s cloud check" % (index + 1, cnt, cloud)
self._header_print(header)
for plugin_class in plugins:
plugin = plugin_class(cloud, self.config)
plugin.check_begin()
plugin.check()
plugin.check_end()
# the failed flag would be record when do check()
if self.args.recover and plugin.failed:
plugin.recover()
if plugin.failed:
exit_flag = True
if exit_flag:
raise exceptions.ClientError("Error: cloud check failed.")
def update_configuration(self, name, value):
path = '/ha/configuration'
configs = self.list_configuration()
if name not in configs.keys():
raise exceptions.ClientError('There is not option %s' % name)
configs[name] = value
self.client.set(path, json.dumps(configs).encode('utf8'))
def get_node(self, node_name):
try:
node_bytes = self.client.get('/ha/%s' % node_name)
node_obj = node.Node.from_zk_bytes(node_bytes)
return node_obj
except kze.NoNodeError:
raise exceptions.ClientError('Node %s not found.' % node_name)
def connect(self, hosts=None, timeout=None, read_only=False):
if not hosts:
if not self.config:
raise exceptions.ClientError('Either config object or hosts '
'string should be provided.')
try:
hosts = hosts or self.config.get('ha', 'zookeeper_hosts')
except (configparser.NoOptionError, configparser.NoSectionError):
raise exceptions.ClientError(
"The config doesn't contain [ha]zookeeper_hosts option.")
if not timeout:
timeout = self.config.get('ha', 'zookeeper_connect_timeout',
fallback=5)
retry_limit = self.config.get('ha', 'zookeeper_connect_retry_limit',
fallback=5)
try:
timeout = int(timeout)
except ValueError:
raise exceptions.ClientError("zookeeper_connect_timeout "
"should be int-like format.")
if timeout <= 0:
raise exceptions.ClientError("zookeeper_connect_timeout "
"should be larger than 0.")
if self.client is None:
self.client = KazooClient(hosts=hosts, timeout=timeout,
read_only=read_only)
self.client.add_listener(self._connection_listener)
# Manually retry initial connection attempt
tried_times = 0
while tried_times < retry_limit:
try:
self.client.start(1)
break
except Exception:
self.logConnectionRetryEvent()
tried_times += 1
if tried_times == retry_limit:
self.client = None
raise exceptions.ClientError(
"Tried %s times, failed connecting "
"zookeeper." % retry_limit)
def create_node(self, name, role, n_type, ip):
existed_nodes = self.list_nodes()
for existed_node in existed_nodes:
if existed_node.role == role and existed_node.role == n_type:
raise exceptions.ClientError(
"The role and type of the node should be unique.")
path = '/ha/%s' % name
new_node = node.Node(name, role, n_type, ip)
try:
self.client.create(path,
value=new_node.to_zk_bytes(),
makepath=True)
except kze.NodeExistsError:
raise exceptions.ClientError("The node %s is already existed."
% name)
self._init_service(name, n_type)
node_obj = self.get_node(name)
return node_obj
def ha_node_update(self):
node_name = self.args.name
if self.args.maintain is None and not self.args.role:
raise exceptions.ClientError("Too few arguments")
maintain = self.args.maintain
role = self.args.role
result = self.zk.update_node(node_name, maintain, role)
if self.args.format == 'pretty':
print(utils.format_output('node', result))
else:
print(result.to_dict())
def get_service(self, service_name, node_name):
service_node = self.get_node(node_name)
path = '/ha/%s/%s/%s' % (service_node.name, service_node.role,
service_name)
try:
service_bytes = self.client.get(path)
except kze.NoNodeError:
raise exceptions.ClientError('Service %s not found.' %
service_name)
service_obj = service.Service.from_zk_bytes(service_bytes)
return service_obj
def print_hints(self):
if self.htype == 'all':
for h in HINTS:
print("\n- Hint of %s:" % h)
for hint in HINTS.get(h):
print(hint)
elif self.htype in HINTS:
print("\n- Hint of %s:" % self.htype)
for hint in HINTS.get(self.htype):
print(hint)
else:
raise exceptions.ClientError("No hints founded.")
def connect(self, hosts=None, read_only=False):
if not hosts:
if not self.config:
raise exceptions.ClientError('Either config object or hosts '
'string should be provided.')
else:
try:
hosts = self.config.get('ha', 'zookeeper_hosts')
except configparser.NoOptionError:
raise exceptions.ClientError(
"The config doesn't [ha]zookeeper_hosts option.")
if self.client is None:
self.client = KazooClient(hosts=hosts, read_only=read_only)
self.client.add_listener(self._connection_listener)
# Manually retry initial connection attempt
while True:
try:
self.client.start(1)
break
except KazooTimeoutError:
self.logConnectionRetryEvent()
def __init__(self, config=None):
"""
Zookeeper Client for OpenLab HA management.
:param config: The config object.
:type: configparser.ConfigParser.
"""
self.client = None
self.config = config
if self.config and not isinstance(self.config,
configparser.ConfigParser):
raise exceptions.ClientError("config should be a ConfigParser "
"object.")
self._last_retry_log = 0
def _initConfig(self):
self.config = configparser.ConfigParser()
if self.args.config:
locations = [self.args.config]
else:
locations = [
'/etc/openlab/openlab.conf', '~/openlab.conf',
'/usr/local/etc/openlab/openlab.conf'
]
for fp in locations:
if os.path.exists(os.path.expanduser(fp)):
self.config.read(os.path.expanduser(fp))
return
raise exceptions.ClientError("Unable to locate config file in "
"%s" % locations)
def _get_cloud_list(self, cloud):
cloud_conf_location = self.config.get(
'check', 'cloud_conf', fallback='/etc/openstack/clouds.yaml')
with open(cloud_conf_location) as f:
clouds = yaml.load(f, Loader=yaml.FullLoader)
clouds_list = [c for c in clouds['clouds']]
if cloud not in clouds_list + ['all']:
raise exceptions.ClientError(
"Error: Cloud %(cloud)s is not found. Please use the cloud "
"in %(clouds_list)s or just use 'all'." % {
'cloud': cloud,
'clouds_list': clouds_list
})
clouds_list = clouds_list if cloud == 'all' else [cloud]
return clouds_list
def check_and_repair_deployment_sg(self, is_dry_run=False):
"""Check and Repair current HA deployment Security Group configuration
This func is called by labkeeper deploy tool. So that operators can
check and repair exist deployment from zookeeper. The function is
for checking Cloud Security Group configuration.
"""
deploy_map = {}
cloud_provide_rules = {}
unexpect_rules = {}
for node in self.list_nodes():
ha_ports_cp = copy.deepcopy(constants.HA_PORTS)
if node.type == 'nodepool':
ha_ports_cp.remove(constants.MYSQL_HA_PORT)
elif node.type == 'zuul':
for p in constants.ZOOKEEPER_HA_PORTS:
ha_ports_cp.remove(p)
elif node.type == 'zookeeper':
ha_ports_cp.remove(constants.RSYNCD_HA_PORT)
ha_ports_cp.remove(constants.MYSQL_HA_PORT)
if node.name.split("-")[0] not in deploy_map:
deploy_map[node.name.split("-")[0]] = {'nodes': [node]}
cloud_provide_rules[node.name.split("-")[0]] = {
node.ip + '/32': ha_ports_cp}
else:
deploy_map[node.name.split("-")[0]]['nodes'].append(node)
cloud_provide_rules[node.name.split("-")[0]][
node.ip + '/32'] = ha_ports_cp
# Fit current expect_rules
expect_rules = {}
sg_map = {}
cloud_names = list(cloud_provide_rules.keys())
for cloud_name, ip_dict in cloud_provide_rules.items():
c_names = copy.deepcopy(cloud_names)
c_names.remove(cloud_name)
expect_rules[cloud_name] = copy.deepcopy(ip_dict)
if len(cloud_provide_rules[cloud_name].keys()) > 1:
for c_name in c_names:
expect_rules[cloud_name].update(
copy.deepcopy(cloud_provide_rules[c_name]))
else:
for c_name in c_names:
for ip in cloud_provide_rules[c_name].keys():
if 2888 in cloud_provide_rules[c_name][ip]:
zk_ha_ports = copy.deepcopy(
constants.ZOOKEEPER_HA_PORTS)
expect_rules[cloud_name][ip] = zk_ha_ports
else:
expect_rules[cloud_name][ip] = [2181]
for cloud_name, nodes_dict in deploy_map.items():
net_client = os_client_config.make_rest_client(
'network', cloud=cloud_name)
for sg_name in constants.HA_SGs:
url = "/security-groups?name=%s" % sg_name
resp = net_client.get(url)
if resp.status_code != 200:
raise exceptions.ClientError(
'Security group %(sg_name)s not found on '
'cloud %(cloud_name)s.' % {'sg_name': sg_name,
'cloud_name': cloud_name})
sgr_data = resp.json()['security_groups'][0]
if cloud_name not in sg_map:
sg_map[cloud_name] = resp.json()[
'security_groups'][0]['id']
for rule in sgr_data['security_group_rules']:
if rule['direction'] != 'ingress':
continue
is_specified_1_port = (
rule['port_range_min'] == rule['port_range_max'])
is_ipv4 = rule['ethertype'] == 'IPv4'
is_tcp = rule['protocol'] == 'tcp'
if not expect_rules[cloud_name].get(
rule['remote_ip_prefix']):
if cloud_name not in unexpect_rules:
unexpect_rules[cloud_name] = [
(rule['remote_ip_prefix'],
rule['port_range_min'], rule['id'])]
else:
unexpect_rules[cloud_name].append(
(rule['remote_ip_prefix'],
rule['port_range_min'], rule['id']))
else:
if (is_specified_1_port and is_ipv4 and is_tcp and
rule['port_range_min'] in expect_rules[
cloud_name][rule['remote_ip_prefix']]):
expect_rules[cloud_name][
rule['remote_ip_prefix']].remove(
rule['port_range_min'])
if len(expect_rules[cloud_name][
rule['remote_ip_prefix']]) ==0:
expect_rules[cloud_name].pop(
rule['remote_ip_prefix'])
else:
if cloud_name not in unexpect_rules:
unexpect_rules[cloud_name] = [
(rule['remote_ip_prefix'],
rule['port_range_min'], rule['id'])]
else:
unexpect_rules[cloud_name].append((
rule['remote_ip_prefix'],
rule['port_range_min'], rule['id']))
if not is_dry_run:
# analysis expect_rules
for cloud_name, ip_dict in expect_rules.items():
if not ip_dict:
print("Cloud %s: PASSED" % cloud_name)
continue
print("Recover security group rules for cloud %s:" %
cloud_name)
# Here means the sg lacks SG_rule settings
net_client = os_client_config.make_rest_client(
'network', cloud=cloud_name)
for ip, ports in ip_dict.items():
req = {
"security_group_rule": {
"direction": "ingress",
"ethertype": "IPv4",
"protocol": "tcp",
"security_group_id": sg_map[cloud_name],
"remote_ip_prefix": ip
}
}
for port in ports:
req["security_group_rule"].update({
"port_range_min": port,
"port_range_max": port
})
resp = net_client.post('/security-group-rules',
json=req)
if resp.status_code != 201:
raise exceptions.ClientError(
'Failed to create security group rule on '
'cloud %(cloud_name)s with summary '
'%(ip)s %(port)s'
% {'cloud_name': cloud_name, 'ip': ip,
'port': port})
print("Create new sg_rule, summary %(ip)s %(port)s" % {
"ip": ip,
"port": str(port)
})
# remove unexpect sg_rules
for cloud_name, ip_port_tuple_list in unexpect_rules.items():
net_client = os_client_config.make_rest_client(
'network', cloud=cloud_name)
print("Unexpect security group rules clean for cloud %s:" %
cloud_name)
for ip_port_tuple in ip_port_tuple_list:
url = "/security-group-rules/%s" % ip_port_tuple[2]
resp = net_client.delete(url)
if resp.status_code != 204:
raise exceptions.ClientError(
'Failed to delete security group rule '
'%(rule_id)s on cloud %(cloud_name)s'
% {'cloud_name': cloud_name,
'rule_id': ip_port_tuple[2]})
print("Remove sg_rule %(rule_id)s, summary %(ip)s "
"%(port)s" % {
"rule_id": ip_port_tuple[2],
"ip": ip_port_tuple[0],
"port": str(ip_port_tuple[1])
})
else:
for cloud_name, ip_dict in expect_rules.items():
if not ip_dict:
print("Cloud %s: PASSED" % cloud_name)
continue
print("Found lack security group rules in cloud %s" %
cloud_name)
for ip, ports in ip_dict.items():
print(" Need to create new rule for (ip)s (ports)s" % {
"ip": ip,
"ports": str(ports)
})
# remove unexpect sg_rules
for cloud_name, ip_port_tuple_list in unexpect_rules.items():
print("Found unexpect security group rules clean for "
"cloud %s:" % cloud_name)
for ip_port_tuple in ip_port_tuple_list:
print(" Need to remove sg_rule %(rule_id)s, "
"summary %(ip)s %(port)s" % {
"rule_id": ip_port_tuple[2],
"ip": ip_port_tuple[0],
"port": str(ip_port_tuple[1])
})
def wrapper(self, *args, **kwargs):
if not self.client:
raise exceptions.ClientError(
"Should call connect function first to initialise "
"zookeeper client")
return func(self, *args, **kwargs)