def __init__(self, server, appid, appkey): # NOTE(yikun): There are two reason we only allow this cmd is executed # in zuul node: # 1. The app installation interface has been completely supported by # zuul github driver (but PyGithub is not supported yet), so we use # it directly to avoid to build the duplicate wheels. # 2. The app key which is usually only existed in Zuul node. try: from zuul.driver.github.githubconnection import GithubConnection from zuul.driver.github import GithubDriver except ImportError: raise exceptions.ClientError( "Error: 'openlab repo list' only can be used in Zuul node.") try: driver = GithubDriver() connection_config = { 'server': server, 'app_id': appid, 'app_key': appkey, } self.conn = GithubConnection(driver, 'github', connection_config) self.conn._authenticateGithubAPI() self.conn._prime_installation_map() except Exception: raise exceptions.ClientError( "Failed to load repo list. Please check the specified" " args:\n--server: %s\n--app-id: %s\n--app-key: %s\n" "See 'openlab repo list -h' to get more info." % ( server, appid, appkey))
def update_node(self, node_name, maintain=None, role=None, **kwargs): path = '/ha/%s' % node_name node_obj = self.get_node(node_name) if maintain is not None: if maintain: if node_obj.status == node.NodeStatus.UP: node_obj.status = node.NodeStatus.MAINTAINING else: raise exceptions.ClientError( "The node must be in 'up' status when trying to " "maintain it.") else: if node_obj.status == node.NodeStatus.MAINTAINING: node_obj.status = node.NodeStatus.UP node_obj.heartbeat = datetime.datetime.utcnow().strftime( '%Y-%m-%d %H:%M:%S') else: raise exceptions.ClientError( "The node must be in 'maintaining' status when trying " "to un-maintain it.") if role: node_obj.role = role switch_status = kwargs.get('switch_status') if switch_status is not None: if switch_status.lower() not in ['start', 'end']: raise exceptions.ClientError( "switch_status must be 'start', 'end'") node_obj.update(kwargs) self.client.set(path, value=node_obj.to_zk_bytes()) node_obj = self.get_node(node_name) return node_obj
def check(self): utils.NOCOLOR = self.args.nocolor cloud_list = self._get_cloud_list(self.args.cloud) if self.args.type == 'default': plugins = list( filter(lambda x: not x.experimental, base.Plugin.plugins)) elif self.args.type == 'all': plugins = base.Plugin.plugins else: # Filter the plugins with specific ptype plugins = list( filter(lambda x: x.ptype == self.args.type, base.Plugin.plugins)) cnt = len(cloud_list) exit_flag = False for index, cloud in enumerate(cloud_list): header = "%s/%s. %s cloud check" % (index + 1, cnt, cloud) self._header_print(header) for plugin_class in plugins: plugin = plugin_class(cloud, self.config) plugin.check_begin() plugin.check() plugin.check_end() # the failed flag would be record when do check() if self.args.recover and plugin.failed: plugin.recover() if plugin.failed: exit_flag = True if exit_flag: raise exceptions.ClientError("Error: cloud check failed.")
def update_configuration(self, name, value): path = '/ha/configuration' configs = self.list_configuration() if name not in configs.keys(): raise exceptions.ClientError('There is not option %s' % name) configs[name] = value self.client.set(path, json.dumps(configs).encode('utf8'))
def get_node(self, node_name): try: node_bytes = self.client.get('/ha/%s' % node_name) node_obj = node.Node.from_zk_bytes(node_bytes) return node_obj except kze.NoNodeError: raise exceptions.ClientError('Node %s not found.' % node_name)
def connect(self, hosts=None, timeout=None, read_only=False): if not hosts: if not self.config: raise exceptions.ClientError('Either config object or hosts ' 'string should be provided.') try: hosts = hosts or self.config.get('ha', 'zookeeper_hosts') except (configparser.NoOptionError, configparser.NoSectionError): raise exceptions.ClientError( "The config doesn't contain [ha]zookeeper_hosts option.") if not timeout: timeout = self.config.get('ha', 'zookeeper_connect_timeout', fallback=5) retry_limit = self.config.get('ha', 'zookeeper_connect_retry_limit', fallback=5) try: timeout = int(timeout) except ValueError: raise exceptions.ClientError("zookeeper_connect_timeout " "should be int-like format.") if timeout <= 0: raise exceptions.ClientError("zookeeper_connect_timeout " "should be larger than 0.") if self.client is None: self.client = KazooClient(hosts=hosts, timeout=timeout, read_only=read_only) self.client.add_listener(self._connection_listener) # Manually retry initial connection attempt tried_times = 0 while tried_times < retry_limit: try: self.client.start(1) break except Exception: self.logConnectionRetryEvent() tried_times += 1 if tried_times == retry_limit: self.client = None raise exceptions.ClientError( "Tried %s times, failed connecting " "zookeeper." % retry_limit)
def create_node(self, name, role, n_type, ip): existed_nodes = self.list_nodes() for existed_node in existed_nodes: if existed_node.role == role and existed_node.role == n_type: raise exceptions.ClientError( "The role and type of the node should be unique.") path = '/ha/%s' % name new_node = node.Node(name, role, n_type, ip) try: self.client.create(path, value=new_node.to_zk_bytes(), makepath=True) except kze.NodeExistsError: raise exceptions.ClientError("The node %s is already existed." % name) self._init_service(name, n_type) node_obj = self.get_node(name) return node_obj
def ha_node_update(self): node_name = self.args.name if self.args.maintain is None and not self.args.role: raise exceptions.ClientError("Too few arguments") maintain = self.args.maintain role = self.args.role result = self.zk.update_node(node_name, maintain, role) if self.args.format == 'pretty': print(utils.format_output('node', result)) else: print(result.to_dict())
def get_service(self, service_name, node_name): service_node = self.get_node(node_name) path = '/ha/%s/%s/%s' % (service_node.name, service_node.role, service_name) try: service_bytes = self.client.get(path) except kze.NoNodeError: raise exceptions.ClientError('Service %s not found.' % service_name) service_obj = service.Service.from_zk_bytes(service_bytes) return service_obj
def print_hints(self): if self.htype == 'all': for h in HINTS: print("\n- Hint of %s:" % h) for hint in HINTS.get(h): print(hint) elif self.htype in HINTS: print("\n- Hint of %s:" % self.htype) for hint in HINTS.get(self.htype): print(hint) else: raise exceptions.ClientError("No hints founded.")
def connect(self, hosts=None, read_only=False): if not hosts: if not self.config: raise exceptions.ClientError('Either config object or hosts ' 'string should be provided.') else: try: hosts = self.config.get('ha', 'zookeeper_hosts') except configparser.NoOptionError: raise exceptions.ClientError( "The config doesn't [ha]zookeeper_hosts option.") if self.client is None: self.client = KazooClient(hosts=hosts, read_only=read_only) self.client.add_listener(self._connection_listener) # Manually retry initial connection attempt while True: try: self.client.start(1) break except KazooTimeoutError: self.logConnectionRetryEvent()
def __init__(self, config=None): """ Zookeeper Client for OpenLab HA management. :param config: The config object. :type: configparser.ConfigParser. """ self.client = None self.config = config if self.config and not isinstance(self.config, configparser.ConfigParser): raise exceptions.ClientError("config should be a ConfigParser " "object.") self._last_retry_log = 0
def _initConfig(self): self.config = configparser.ConfigParser() if self.args.config: locations = [self.args.config] else: locations = [ '/etc/openlab/openlab.conf', '~/openlab.conf', '/usr/local/etc/openlab/openlab.conf' ] for fp in locations: if os.path.exists(os.path.expanduser(fp)): self.config.read(os.path.expanduser(fp)) return raise exceptions.ClientError("Unable to locate config file in " "%s" % locations)
def _get_cloud_list(self, cloud): cloud_conf_location = self.config.get( 'check', 'cloud_conf', fallback='/etc/openstack/clouds.yaml') with open(cloud_conf_location) as f: clouds = yaml.load(f, Loader=yaml.FullLoader) clouds_list = [c for c in clouds['clouds']] if cloud not in clouds_list + ['all']: raise exceptions.ClientError( "Error: Cloud %(cloud)s is not found. Please use the cloud " "in %(clouds_list)s or just use 'all'." % { 'cloud': cloud, 'clouds_list': clouds_list }) clouds_list = clouds_list if cloud == 'all' else [cloud] return clouds_list
def check_and_repair_deployment_sg(self, is_dry_run=False): """Check and Repair current HA deployment Security Group configuration This func is called by labkeeper deploy tool. So that operators can check and repair exist deployment from zookeeper. The function is for checking Cloud Security Group configuration. """ deploy_map = {} cloud_provide_rules = {} unexpect_rules = {} for node in self.list_nodes(): ha_ports_cp = copy.deepcopy(constants.HA_PORTS) if node.type == 'nodepool': ha_ports_cp.remove(constants.MYSQL_HA_PORT) elif node.type == 'zuul': for p in constants.ZOOKEEPER_HA_PORTS: ha_ports_cp.remove(p) elif node.type == 'zookeeper': ha_ports_cp.remove(constants.RSYNCD_HA_PORT) ha_ports_cp.remove(constants.MYSQL_HA_PORT) if node.name.split("-")[0] not in deploy_map: deploy_map[node.name.split("-")[0]] = {'nodes': [node]} cloud_provide_rules[node.name.split("-")[0]] = { node.ip + '/32': ha_ports_cp} else: deploy_map[node.name.split("-")[0]]['nodes'].append(node) cloud_provide_rules[node.name.split("-")[0]][ node.ip + '/32'] = ha_ports_cp # Fit current expect_rules expect_rules = {} sg_map = {} cloud_names = list(cloud_provide_rules.keys()) for cloud_name, ip_dict in cloud_provide_rules.items(): c_names = copy.deepcopy(cloud_names) c_names.remove(cloud_name) expect_rules[cloud_name] = copy.deepcopy(ip_dict) if len(cloud_provide_rules[cloud_name].keys()) > 1: for c_name in c_names: expect_rules[cloud_name].update( copy.deepcopy(cloud_provide_rules[c_name])) else: for c_name in c_names: for ip in cloud_provide_rules[c_name].keys(): if 2888 in cloud_provide_rules[c_name][ip]: zk_ha_ports = copy.deepcopy( constants.ZOOKEEPER_HA_PORTS) expect_rules[cloud_name][ip] = zk_ha_ports else: expect_rules[cloud_name][ip] = [2181] for cloud_name, nodes_dict in deploy_map.items(): net_client = os_client_config.make_rest_client( 'network', cloud=cloud_name) for sg_name in constants.HA_SGs: url = "/security-groups?name=%s" % sg_name resp = net_client.get(url) if resp.status_code != 200: raise exceptions.ClientError( 'Security group %(sg_name)s not found on ' 'cloud %(cloud_name)s.' % {'sg_name': sg_name, 'cloud_name': cloud_name}) sgr_data = resp.json()['security_groups'][0] if cloud_name not in sg_map: sg_map[cloud_name] = resp.json()[ 'security_groups'][0]['id'] for rule in sgr_data['security_group_rules']: if rule['direction'] != 'ingress': continue is_specified_1_port = ( rule['port_range_min'] == rule['port_range_max']) is_ipv4 = rule['ethertype'] == 'IPv4' is_tcp = rule['protocol'] == 'tcp' if not expect_rules[cloud_name].get( rule['remote_ip_prefix']): if cloud_name not in unexpect_rules: unexpect_rules[cloud_name] = [ (rule['remote_ip_prefix'], rule['port_range_min'], rule['id'])] else: unexpect_rules[cloud_name].append( (rule['remote_ip_prefix'], rule['port_range_min'], rule['id'])) else: if (is_specified_1_port and is_ipv4 and is_tcp and rule['port_range_min'] in expect_rules[ cloud_name][rule['remote_ip_prefix']]): expect_rules[cloud_name][ rule['remote_ip_prefix']].remove( rule['port_range_min']) if len(expect_rules[cloud_name][ rule['remote_ip_prefix']]) ==0: expect_rules[cloud_name].pop( rule['remote_ip_prefix']) else: if cloud_name not in unexpect_rules: unexpect_rules[cloud_name] = [ (rule['remote_ip_prefix'], rule['port_range_min'], rule['id'])] else: unexpect_rules[cloud_name].append(( rule['remote_ip_prefix'], rule['port_range_min'], rule['id'])) if not is_dry_run: # analysis expect_rules for cloud_name, ip_dict in expect_rules.items(): if not ip_dict: print("Cloud %s: PASSED" % cloud_name) continue print("Recover security group rules for cloud %s:" % cloud_name) # Here means the sg lacks SG_rule settings net_client = os_client_config.make_rest_client( 'network', cloud=cloud_name) for ip, ports in ip_dict.items(): req = { "security_group_rule": { "direction": "ingress", "ethertype": "IPv4", "protocol": "tcp", "security_group_id": sg_map[cloud_name], "remote_ip_prefix": ip } } for port in ports: req["security_group_rule"].update({ "port_range_min": port, "port_range_max": port }) resp = net_client.post('/security-group-rules', json=req) if resp.status_code != 201: raise exceptions.ClientError( 'Failed to create security group rule on ' 'cloud %(cloud_name)s with summary ' '%(ip)s %(port)s' % {'cloud_name': cloud_name, 'ip': ip, 'port': port}) print("Create new sg_rule, summary %(ip)s %(port)s" % { "ip": ip, "port": str(port) }) # remove unexpect sg_rules for cloud_name, ip_port_tuple_list in unexpect_rules.items(): net_client = os_client_config.make_rest_client( 'network', cloud=cloud_name) print("Unexpect security group rules clean for cloud %s:" % cloud_name) for ip_port_tuple in ip_port_tuple_list: url = "/security-group-rules/%s" % ip_port_tuple[2] resp = net_client.delete(url) if resp.status_code != 204: raise exceptions.ClientError( 'Failed to delete security group rule ' '%(rule_id)s on cloud %(cloud_name)s' % {'cloud_name': cloud_name, 'rule_id': ip_port_tuple[2]}) print("Remove sg_rule %(rule_id)s, summary %(ip)s " "%(port)s" % { "rule_id": ip_port_tuple[2], "ip": ip_port_tuple[0], "port": str(ip_port_tuple[1]) }) else: for cloud_name, ip_dict in expect_rules.items(): if not ip_dict: print("Cloud %s: PASSED" % cloud_name) continue print("Found lack security group rules in cloud %s" % cloud_name) for ip, ports in ip_dict.items(): print(" Need to create new rule for (ip)s (ports)s" % { "ip": ip, "ports": str(ports) }) # remove unexpect sg_rules for cloud_name, ip_port_tuple_list in unexpect_rules.items(): print("Found unexpect security group rules clean for " "cloud %s:" % cloud_name) for ip_port_tuple in ip_port_tuple_list: print(" Need to remove sg_rule %(rule_id)s, " "summary %(ip)s %(port)s" % { "rule_id": ip_port_tuple[2], "ip": ip_port_tuple[0], "port": str(ip_port_tuple[1]) })
def wrapper(self, *args, **kwargs): if not self.client: raise exceptions.ClientError( "Should call connect function first to initialise " "zookeeper client") return func(self, *args, **kwargs)