def _create_bootstrap_vm_nic(self): self.logging.info("Creating bootstrap VM NIC") public_ip = self._create_bootstrap_vm_public_ip() control_plane_subnet = utils.retry_on_error()( self.network_client.subnets.get)( self.cluster_name, "%s-vnet" % self.cluster_name, "%s-controlplane-subnet" % self.cluster_name) nsg = self._create_bootstrap_vm_secgroup() nic_parameters = { "location": self.azure_location, "network_security_group": { "id": nsg.id }, "ip_configurations": [{ "name": "%s-ipconfig" % self.bootstrap_vm_nic_name, "subnet": { "id": control_plane_subnet.id }, "public_ip_address": { "id": public_ip.id } }] } return utils.retry_on_error()( self.network_client.network_interfaces.create_or_update)( self.cluster_name, self.bootstrap_vm_nic_name, nic_parameters).result()
def _add_flannel_cni(self): template_file = os.path.join( os.getcwd(), "cluster-api/flannel/kube-flannel.yaml.j2") context = { "cluster_network_subnet": self.deployer.cluster_network_subnet, "flannel_mode": self.opts.flannel_mode } kube_flannel = "/tmp/kube-flannel.yaml" utils.render_template(template_file, kube_flannel, context) server_core_tag = "windowsservercore-%s" % ( self.opts.base_container_image_tag) mode = "overlay" if self.opts.flannel_mode == constants.FLANNEL_MODE_L2BRIDGE: mode = "l2bridge" context = { "server_core_tag": server_core_tag, "container_runtime": self.opts.container_runtime, "mode": mode } kube_flannel_windows = "/tmp/kube-flannel-windows.yaml" searchpath = os.path.join(os.getcwd(), "cluster-api/flannel") utils.render_template("kube-flannel-windows.yaml.j2", kube_flannel_windows, context, searchpath) cmd = [self.kubectl, "apply", "-f", kube_flannel] utils.retry_on_error()(utils.run_shell_cmd)(cmd) cmd = [self.kubectl, "apply", "-f", kube_flannel_windows] utils.retry_on_error()(utils.run_shell_cmd)(cmd) if self.opts.flannel_mode == constants.FLANNEL_MODE_OVERLAY: self._set_vxlan_devices_mtu()
def upload_to_bootstrap_vm(self, local_path): self.logging.info("Uploading %s to bootstrap VM", local_path) ssh_cmd = ("ssh -i %s -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null" % os.environ["SSH_KEY"]) cmd = [ "rsync", "--chmod=D755,F644", "-r", "-e", "\"%s\"" % ssh_cmd, local_path, "capi@%s:/www/" % self.bootstrap_vm_public_ip ] utils.retry_on_error()(utils.run_shell_cmd)(cmd)
def _wait_for_control_plane(self, timeout=2700): self.logging.info( "Waiting up to %.2f minutes for the control-plane to be ready.", timeout / 60.0) sleep_time = 5 start = time.time() while True: elapsed = time.time() - start if elapsed > timeout: raise Exception("The control-plane didn't become ready " "within %.2f minutes" % (timeout / 60.0)) cmd = [ self.kubectl, "get", "machine", "--kubeconfig", self.kind_kubeconfig_path, "--output=custom-columns=NAME:.metadata.name", "--no-headers" ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) machines = output.decode().strip().split('\n') control_plane_machines = [ m for m in machines if m.startswith("%s-control-plane" % self.cluster_name) ] if len(control_plane_machines) == 0: time.sleep(sleep_time) continue all_ready = True for control_plane_machine in control_plane_machines: cmd = [ self.kubectl, "get", "machine", "--kubeconfig", self.kind_kubeconfig_path, "--output=custom-columns=READY:.status.phase", "--no-headers", control_plane_machine ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) status_phase = output.decode().strip() if status_phase != "Running": all_ready = False continue if all_ready: self.logging.info( "The control plane provisioned in " "%.2f minutes", (time.time() - start) / 60.0) break time.sleep(sleep_time)
def _set_vxlan_devices_mtu(self): self.logging.info( "Set the proper MTU for the k8s master vxlan devices") ssh_key_path = (os.environ.get("SSH_KEY") or os.path.join(os.environ.get("HOME"), ".ssh/id_rsa")) utils.retry_on_error()(utils.run_shell_cmd)([ "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-i", ssh_key_path, "capi@%s" % self.deployer.master_public_address, "'sudo bash -s' < %s" % os.path.join( os.getcwd(), "cluster-api/scripts/set-vxlan-devices-mtu.sh") ])
def on_joined(self, event): if not self.cluster.is_ag_ready: logger.warning('The availability group is not ready. Defering ' 'hacluster on_joined until AG is ready.') event.defer() return logger.info('Installing Microsoft SQL Server HA components') retry_on_error()(apt_install)(packages=self.APT_PACKAGES, fatal=True) self.setup_pacemaker_mssql_login() rel_data = { 'resources': { 'ag_cluster': 'ocf:mssql:ag' }, 'resource_params': { 'ag_cluster': 'params ag_name="{ag_name}" ' 'meta failure-timeout=60s ' 'op start timeout=60s ' 'op stop timeout=60s ' 'op promote timeout=60s ' 'op demote timeout=10s ' 'op monitor timeout=60s interval=10s ' 'op monitor timeout=60s interval=11s role="Master" ' 'op monitor timeout=60s interval=12s role="Slave" ' 'op notify timeout=60s'.format(ag_name=self.cluster.AG_NAME) }, 'ms': { 'ms-ag_cluster': 'ag_cluster meta ' 'master-max="1" master-node-max="1" ' 'clone-max="3" clone-node-max="1" notify="true"' } } update_hacluster_vip('mssql', rel_data) group_name = VIP_GROUP_NAME.format(service='mssql') rel_data.update({ 'colocations': { 'vip_on_master': 'inf: {} ms-ag_cluster:Master'.format(group_name) }, 'orders': { 'ag_first': 'inf: ms-ag_cluster:promote {}:start'.format(group_name) } }) rel = self.model.get_relation(event.relation.name, event.relation.id) for k, v in rel_data.items(): rel.data[self.unit]['json_{}'.format(k)] = json.dumps( v, **JSON_ENCODE_OPTIONS)
def _is_k8s_node_ready(self, node_name=None): if not node_name: self.logging.warning("Empty node_name parameter") return False cmd = [ self.kubectl, "get", "--kubeconfig", self.capz_kubeconfig_path, "node", "-o", "yaml", node_name ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)(cmd, sensitive=True) node = yaml.safe_load(output.decode('ascii')) if "status" not in node: self.logging.info("Node %s didn't report status yet", node_name) return False ready_condition = [ c for c in node["status"]["conditions"] if c["type"] == "Ready" ] if len(ready_condition) == 0: self.logging.info("Node %s didn't report ready condition yet", node_name) return False try: is_ready = strtobool(ready_condition[0]["status"]) except ValueError: is_ready = False if not is_ready: self.logging.info("Node %s is not ready yet", node_name) return False return True
def _validate_k8s_api_container_images(self): self.logging.info("Validating K8s API container images") output, _ = utils.retry_on_error()(utils.run_shell_cmd)([ self.kubectl, "get", "nodes", "-o", "yaml", "-l", "kubernetes.io/os=linux" ]) nodes = yaml.safe_load(output.decode("ascii")) images_tag = self.ci_version.replace("+", "_").strip("v") name_regex = re.compile(r"^(k8s.gcr.io/kube-.*):v(.*)$") for node in nodes["items"]: non_ci_images_names = [] for image in node["status"]["images"]: non_ci_images_names += [ name for name in image["names"] if (name_regex.match(name) and name_regex.match(name).group(2) != images_tag)] if len(non_ci_images_names) > 0: self.logging.error( "Found the following non-CI images %s on the " "node %s.", non_ci_images_names, node["metadata"]["name"]) raise Exception("Found non-CI container images on " "node %s" % node["metadata"]["name"])
def _add_kube_proxy_windows(self): template_file = os.path.join( os.getcwd(), "cluster-api/kube-proxy/kube-proxy-windows.yaml.j2") server_core_tag = "windowsservercore-%s" % ( self.opts.base_container_image_tag) context = { "kubernetes_version": self.kubernetes_version, "server_core_tag": server_core_tag, "enable_win_dsr": str(self.opts.enable_win_dsr).lower(), "flannel_mode": self.opts.flannel_mode } output_file = "/tmp/kube-proxy-windows.yaml" utils.render_template(template_file, output_file, context) cmd = [self.kubectl, "apply", "-f", output_file] utils.retry_on_error()(utils.run_shell_cmd)(cmd)
def _create_bootstrap_vm_secgroup(self): self.logging.info("Creating bootstrap VM security group") secgroup_rules = [ net_models.SecurityRule( protocol="Tcp", priority="1000", source_port_range="*", source_address_prefix="0.0.0.0/0", destination_port_range="22", destination_address_prefix="0.0.0.0/0", destination_address_prefixes=[], destination_application_security_groups=[], access=net_models.SecurityRuleAccess.allow, direction=net_models.SecurityRuleDirection.inbound, name="Allow_SSH") ] secgroup_params = net_models.NetworkSecurityGroup( name=self.bootstrap_vm_secgroup_name, location=self.azure_location, security_rules=secgroup_rules) return utils.retry_on_error()( self.network_client.network_security_groups.create_or_update)( self.cluster_name, self.bootstrap_vm_secgroup_name, secgroup_params).result()
def _wait_for_bootstrap_vm(self, timeout=900): self.logging.info("Waiting up to %.2f minutes for VM %s to provision", timeout / 60.0, self.bootstrap_vm_name) sleep_time = 5 start = time.time() while True: elapsed = time.time() - start if elapsed > timeout: err_msg = "VM %s didn't provision within %.2f minutes" % ( self.bootstrap_vm_name, timeout / 60) self.logging.error(err_msg) raise Exception(err_msg) vm = utils.retry_on_error()( self.compute_client.virtual_machines.get)( self.cluster_name, self.bootstrap_vm_name) if vm.provisioning_state == "Succeeded": break if vm.provisioning_state not in ("Creating", "Updating"): err_msg = 'VM "%s" entered invalid state: "%s"' % ( self.bootstrap_vm_name, vm.provisioning_state) self.logging.error(err_msg) raise Exception(err_msg) time.sleep(sleep_time) return vm
def _get_agents_private_addresses(self, operating_system): output, _ = utils.retry_on_error()(utils.run_shell_cmd)([ self.kubectl, "get", "machine", "--kubeconfig", self.kind_kubeconfig_path, "-o", "yaml" ]) addresses = [] nodes = yaml.safe_load(output) for node in nodes['items']: if (operating_system == "linux" and node["metadata"]["name"].startswith("capi-win")): continue if (operating_system == "windows" and not node["metadata"]["name"].startswith("capi-win")): continue try: node_addresses = [ n['address'] for n in node['status']['addresses'] if n['type'] == 'InternalIP' ] except Exception as ex: self.logging.warning( "Cannot find private address for node %s. Exception " "details: %s. Skipping", node["metadata"]["name"], ex) # pick the first node internal address addresses.append(node_addresses[0]) return addresses
def master_public_port(self): output, _ = utils.retry_on_error()(utils.run_shell_cmd)([ self.kubectl, "get", "cluster", "--kubeconfig", self.kind_kubeconfig_path, self.cluster_name, "-o", "custom-columns=MASTER_PORT:.spec.controlPlaneEndpoint.port", "--no-headers" ]) return output.decode("ascii").strip()
def _wait_for_cluster(self, timeout=900): self.logging.info( "Waiting up to %.2f minutes for the cluster to provision.", timeout / 60.0) sleep_time = 5 start = time.time() cluster_resource_name = "cluster.cluster.x-k8s.io/%s" % ( self.cluster_name) while True: elapsed = time.time() - start if elapsed > timeout: raise Exception("Cluster didn't provision within %.2f " "minutes" % (timeout / 60.0)) cmd = [ self.kubectl, "get", "cluster", "--kubeconfig", self.kind_kubeconfig_path, self.cluster_name, "--output=name" ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) names = output.decode().strip().split('\n') found = [c for c in names if c == cluster_resource_name] if len(found) == 0: time.sleep(sleep_time) continue cmd = [ self.kubectl, "get", "cluster", "--kubeconfig", self.kind_kubeconfig_path, self.cluster_name, "--output=custom-columns=CLUSTER_STATUS:.status.phase", "--no-headers" ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) cluster_status = output.decode().strip() if cluster_status == "Provisioned": self.logging.info("Cluster provisioned in %.2f minutes", (time.time() - start) / 60.0) break time.sleep(sleep_time)
def _create_bootstrap_vm_public_ip(self): self.logging.info("Creating bootstrap VM public IP") public_ip_parameters = { "location": self.azure_location, "public_ip_address_version": "IPV4" } return utils.retry_on_error()( self.network_client.public_ip_addresses.create_or_update)( self.cluster_name, self.bootstrap_vm_public_ip_name, public_ip_parameters).result()
def enable_ip_forwarding(self): self.logging.info("Enabling IP forwarding for the cluster VMs") vm_nics = utils.retry_on_error()( self.network_client.network_interfaces.list)(self.cluster_name) for nic in vm_nics: if nic.name == self.bootstrap_vm_nic_name: continue if nic.enable_ip_forwarding: self.logging.info("IP forwarding is already enabled on nic %s", nic.name) continue self.logging.info("Enabling IP forwarding on nic %s", nic.name) nic_parameters = nic.as_dict() nic_parameters["enable_ip_forwarding"] = True utils.retry_on_error()( self.network_client.network_interfaces.create_or_update)( self.cluster_name, nic.name, nic_parameters).result()
def _setup_capz_kubeconfig(self): self.logging.info("Setting up CAPZ kubeconfig") cmd = [ self.kubectl, "get", "--kubeconfig", self.kind_kubeconfig_path, "secret/%s-kubeconfig" % self.cluster_name, "--output=custom-columns=KUBECONFIG_B64:.data.value", "--no-headers" ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)(cmd) with open(self.capz_kubeconfig_path, 'w') as f: f.write(base64.b64decode(output).decode('ascii'))
def setup_simplesamlphp(self): if os.path.exists(self.DEST_DIR): os.rmdir(self.DEST_DIR) version = self.config.get('simple-saml-php-version') archive_handler = ArchiveUrlFetchHandler() retry_on_error()(archive_handler.install)( source='{0}/v{1}/simplesamlphp-{1}.tar.gz'.format( self.BASE_DOWNLOAD_URL, version), dest=os.path.dirname(self.DEST_DIR)) os.rename('{0}-{1}'.format(self.DEST_DIR, version), self.DEST_DIR) key_file = '{0}/cert/server.pem'.format(self.DEST_DIR) cert_file = '{0}/cert/server.crt'.format(self.DEST_DIR) ssl.generate_selfsigned(keyfile=key_file, certfile=cert_file, keysize=2048, cn=get_unit_hostname()) uid = pwd.getpwnam(self.APACHE_USER).pw_uid gid = grp.getgrnam(self.APACHE_GROUP).gr_gid os.chown(key_file, uid, gid) os.chown(cert_file, uid, gid)
def _prepull_images(self, timeout=3600): prepull_yaml_path = "/tmp/prepull-windows-images.yaml" utils.download_file(self.opts.prepull_yaml, prepull_yaml_path) self.logging.info("Starting Windows images pre-pull") utils.retry_on_error()(utils.run_shell_cmd)( [self.kubectl, "apply", "-f", prepull_yaml_path]) self.logging.info( "Waiting up to %.2f minutes to pre-pull Windows container images", timeout / 60.0) sleep_time = 5 start = time.time() cmd = [self.kubectl, "get", "-o", "yaml", "-f", prepull_yaml_path] while True: elapsed = time.time() - start if elapsed > timeout: raise Exception("Couldn't pre-pull Windows images within " "%.2f minutes." % (timeout / 60.0)) output, _ = utils.retry_on_error()( utils.run_shell_cmd)(cmd, sensitive=True) prepull_daemonset = yaml.safe_load(output.decode("ascii")) if (prepull_daemonset["status"]["numberReady"] == prepull_daemonset["status"]["desiredNumberScheduled"]): break time.sleep(sleep_time) self.logging.info("Windows images pre-pulled in %.2f minutes", (time.time() - start) / 60.0) self.logging.info("Cleaning up") utils.run_shell_cmd( [self.kubectl, "delete", "--wait", "-f", prepull_yaml_path])
def _cleanup_bootstrap_vm(self): self.logging.info("Cleaning up the bootstrap VM") self.logging.info("Deleting bootstrap VM") utils.retry_on_error()(self.compute_client.virtual_machines.delete)( self.cluster_name, self.bootstrap_vm_name).wait() self.logging.info("Deleting bootstrap VM NIC") utils.retry_on_error()(self.network_client.network_interfaces.delete)( self.cluster_name, self.bootstrap_vm_nic_name).wait() self.logging.info("Deleting bootstrap VM public IP") utils.retry_on_error()(self.network_client.public_ip_addresses.delete)( self.cluster_name, self.bootstrap_vm_public_ip_name).wait() self.logging.info("Deleting bootstrap VM security group") utils.retry_on_error()( self.network_client.network_security_groups.delete)( self.cluster_name, self.bootstrap_vm_secgroup_name).wait()
def _validate_k8s_api_versions(self): self.logging.info("Validating K8s API versions") output, _ = utils.retry_on_error()( utils.run_shell_cmd)([self.kubectl, "get", "nodes", "-o", "yaml"]) nodes = yaml.safe_load(output.decode("ascii")) for node in nodes["items"]: node_name = node["metadata"]["name"] node_info = node["status"]["nodeInfo"] if node_info["kubeletVersion"] != self.ci_version: raise Exception( "Wrong kubelet version on node %s. " "Expected %s, but found %s" % (node_name, self.ci_version, node_info["kubeletVersion"])) if node_info["kubeProxyVersion"] != self.ci_version: raise Exception( "Wrong kube-proxy version on node %s. " "Expected %s, but found %s" % (node_name, self.ci_version, node_info["kubeletVersion"]))
def _build_k8s_artifacts(self): k8s_path = utils.get_k8s_folder() utils.clone_git_repo( self.opts.k8s_repo, self.opts.k8s_branch, k8s_path) self.logging.info("Building K8s Linux binaries") cmd = [ 'make', 'WHAT="cmd/kubectl cmd/kubelet cmd/kubeadm"', 'KUBE_BUILD_PLATFORMS="linux/amd64"' ] utils.run_shell_cmd(cmd, k8s_path) del os.environ["KUBECTL_PATH"] self.logging.info("Building K8s Windows binaries") cmd = [ 'make', 'WHAT="cmd/kubectl cmd/kubelet cmd/kubeadm cmd/kube-proxy"', 'KUBE_BUILD_PLATFORMS="windows/amd64"' ] utils.run_shell_cmd(cmd, k8s_path) self.logging.info("Building K8s Linux DaemonSet container images") cmd = ['make', 'quick-release-images'] env = {"KUBE_FASTBUILD": "true", "KUBE_BUILD_CONFORMANCE": "n"} utils.retry_on_error()(utils.run_shell_cmd)(cmd, k8s_path, env) kubeadm_bin = os.path.join(constants.KUBERNETES_LINUX_BINS_LOCATION, 'kubeadm') out, _ = utils.run_shell_cmd( [kubeadm_bin, "version", "-o=short"], k8s_path) self.ci_version = out.decode().strip() self.deployer.ci_version = self.ci_version ci_artifacts_linux_bin_dir = "%s/%s/bin/linux/amd64" % ( self.ci_artifacts_dir, self.ci_version) ci_artifacts_windows_bin_dir = "%s/%s/bin/windows/amd64" % ( self.ci_artifacts_dir, self.ci_version) ci_artifacts_images_dir = "%s/%s/images" % ( self.ci_artifacts_dir, self.ci_version) os.makedirs(ci_artifacts_linux_bin_dir, exist_ok=True) os.makedirs(ci_artifacts_windows_bin_dir, exist_ok=True) os.makedirs(ci_artifacts_images_dir, exist_ok=True) for bin_name in ["kubectl", "kubelet", "kubeadm"]: linux_bin_path = "%s/%s/%s" % ( k8s_path, constants.KUBERNETES_LINUX_BINS_LOCATION, bin_name) shutil.copy(linux_bin_path, ci_artifacts_linux_bin_dir) for bin_name in ["kubectl", "kubelet", "kubeadm", "kube-proxy"]: win_bin_path = "%s/%s/%s.exe" % ( k8s_path, constants.KUBERNETES_WINDOWS_BINS_LOCATION, bin_name) shutil.copy(win_bin_path, ci_artifacts_windows_bin_dir) images_names = [ "kube-apiserver.tar", "kube-controller-manager.tar", "kube-proxy.tar", "kube-scheduler.tar" ] for image_name in images_names: image_path = "%s/%s/%s" % ( k8s_path, constants.KUBERNETES_IMAGES_LOCATION, image_name) shutil.copy(image_path, ci_artifacts_images_dir)
def _connect_control_plane_to_node_subnet(self): self.logging.info( "Adding second NIC from node subnet to control plane") control_plane_vm = self._get_control_plane_vm() if not control_plane_vm: raise Exception("The control plane VM was not found") if len(control_plane_vm.network_profile.network_interfaces) > 1: self.logging.info( "The second NIC is already attached to the control plane") return nic_name = "%s-control-plane-node-nic" % self.cluster_name node_subnet = utils.retry_on_error()(self.network_client.subnets.get)( self.cluster_name, "%s-vnet" % self.cluster_name, "%s-node-subnet" % self.cluster_name) nic_parameters = { "location": self.azure_location, "ip_configurations": [{ "name": "%s-ipconfig" % nic_name, "subnet": { "id": node_subnet.id } }] } self.logging.info("Creating the second NIC") nic = utils.retry_on_error()( self.network_client.network_interfaces.create_or_update)( self.cluster_name, nic_name, nic_parameters).result() self.logging.info("Deallocating the control plane VM") utils.retry_on_error()( self.compute_client.virtual_machines.deallocate)( self.cluster_name, control_plane_vm.name).wait() self.logging.info("Attaching the second NIC to the control plane VM") vm_parameters = control_plane_vm.as_dict() vm_parameters["network_profile"]["network_interfaces"].append({ "id": nic.id, "primary": False }) utils.retry_on_error()( self.compute_client.virtual_machines.create_or_update)( self.cluster_name, control_plane_vm.name, vm_parameters).wait() self.logging.info("Starting the control plane VM") utils.retry_on_error()(self.compute_client.virtual_machines.start)( self.cluster_name, control_plane_vm.name).wait() utils.wait_for_port_connectivity(self.master_public_address, 22) self.logging.info("Updating the control plane route") control_plane_route = self._get_control_plane_route() if not control_plane_route: raise Exception("The control plane route was not found") route_params = control_plane_route.as_dict() route_params["next_hop_ip_address"] = \ nic.ip_configurations[0].private_ip_address utils.retry_on_error()(self.network_client.routes.create_or_update)( self.cluster_name, "%s-node-routetable" % self.cluster_name, control_plane_route.name, route_params).wait()
def _create_bootstrap_vm(self): self.logging.info("Setting up the bootstrap VM") cloud_config_file = os.path.join( os.getcwd(), "cluster-api/azure/bootstrap-vm-cloud-config.txt") with open(cloud_config_file) as f: cloud_config = f.read() vm_nic = self._create_bootstrap_vm_nic() vm_parameters = { "location": self.azure_location, "os_profile": { "computer_name": self.bootstrap_vm_name, "admin_username": "******", "custom_data": base64.b64encode(cloud_config.encode('ascii')).decode('ascii'), "linux_configuration": { "disable_password_authentication": True, "ssh": { "public_keys": [{ "key_data": os.environ["AZURE_SSH_PUBLIC_KEY"], "path": "/home/capi/.ssh/authorized_keys" }] } } }, "hardware_profile": { "vm_size": self.bootstrap_vm_size }, "storage_profile": { "image_reference": { "publisher": "Canonical", "offer": "UbuntuServer", "sku": "18_04-lts-gen2", "version": "latest" }, }, "network_profile": { "network_interfaces": [{ "id": vm_nic.id }] } } self.logging.info("Creating bootstrap VM") vm = utils.retry_on_error()( self.compute_client.virtual_machines.create_or_update)( self.cluster_name, self.bootstrap_vm_name, vm_parameters).result() vm = self._wait_for_bootstrap_vm() ip_config = self.network_client.network_interfaces.get( self.cluster_name, vm_nic.name).ip_configurations[0] self.bootstrap_vm_private_ip = ip_config.private_ip_address public_ip = self.network_client.public_ip_addresses.get( self.cluster_name, self.bootstrap_vm_public_ip_name) self.bootstrap_vm_public_ip = public_ip.ip_address self._wait_for_ready_bootstrap_vm() self.logging.info("Finished setting up the bootstrap VM") return vm
def _wait_for_windows_agents(self, check_nodes_ready=True, timeout=3600): self.logging.info("Waiting up to %.2f minutes for the Windows agents", timeout / 60.0) sleep_time = 5 start = time.time() while True: elapsed = time.time() - start if elapsed > timeout: raise Exception("The Windows agents didn't become ready " "within %.2f minutes" % (timeout / 60.0)) cmd = [ self.kubectl, "get", "machine", "--kubeconfig", self.kind_kubeconfig_path, "--output=custom-columns=NAME:.metadata.name", "--no-headers" ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) machines = output.decode().strip().split('\n') windows_machines = [ # This value is given in the capz cluster.yaml config, and # it's hardcoded since it's going to be part of the Windows # agents hostnames, and together with the unique suffix added # by the cluster-api Azure provider, the length must be <= 15 # characters. m for m in machines if m.startswith("capi-win-") ] if len(windows_machines) == 0: time.sleep(sleep_time) continue all_ready = True for windows_machine in windows_machines: cmd = [ self.kubectl, "get", "machine", "--kubeconfig", self.kind_kubeconfig_path, "--output=custom-columns=READY:.status.phase", "--no-headers", windows_machine ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) status_phase = output.decode().strip() if status_phase != "Running": all_ready = False continue if not check_nodes_ready: continue cmd = [ self.kubectl, "get", "machine", "--kubeconfig", self.kind_kubeconfig_path, "--output=custom-columns=NODE_NAME:.status.nodeRef.name", windows_machine, "--no-headers" ] output, _ = utils.retry_on_error()(utils.run_shell_cmd)( cmd, sensitive=True) node_name = output.decode("ascii").strip() all_ready = self._is_k8s_node_ready(node_name) if all_ready: self.logging.info("All the Windows agents are ready") break time.sleep(sleep_time)
def _run_remote_cmd(self, cmd, node_addresses): for node_address in node_addresses: utils.retry_on_error()(self.deployer.run_cmd_on_k8s_node)( cmd, node_address)
def _upload_to(self, local_path, remote_path, node_addresses): for node_address in node_addresses: utils.retry_on_error()(self.deployer.upload_to_k8s_node)( local_path, remote_path, node_address)
def on_install(self, _): retry_on_error()(apt_update)(fatal=True) retry_on_error()(apt_install)(packages=self.APT_PACKAGES, fatal=True) self.setup_simplesamlphp() self.setup_apache2()
from charmhelpers.fetch.archiveurl import ArchiveUrlFetchHandler from ops.charm import CharmBase from ops.model import ActiveStatus, BlockedStatus from ops.main import main from utils import render_configs, retry_on_error logger = logging.getLogger(__name__) try: from lxml import etree # NOQA:F401 except ImportError: logger.warning( 'Failed to import the lxml module. Re-installing the charm venv') retry_on_error()(apt_update)(fatal=True) retry_on_error()(apt_install)(packages=['python3-pip'], fatal=True) pip_cmd = [ 'pip3', 'install', '--upgrade', '--force-reinstall', '--target=venv', '--requirement=requirements.txt' ] retry_on_error()(subprocess.check_call)(pip_cmd) from lxml import etree # NOQA:F401 class TestSamlIdpCharm(CharmBase): BASE_DOWNLOAD_URL = ('https://github.com/simplesamlphp/simplesamlphp/' 'releases/download') DEST_DIR = '/var/simplesamlphp' APT_PACKAGES = [ 'php', 'php-xml', 'php-date', 'php-mbstring', 'php-curl', 'apache2',
def _create_capz_cluster(self): context = { "cluster_name": self.cluster_name, "cluster_network_subnet": self.cluster_network_subnet, "azure_location": self.azure_location, "azure_subscription_id": os.environ["AZURE_SUBSCRIPTION_ID"], "azure_tenant_id": os.environ["AZURE_TENANT_ID"], "azure_client_id": os.environ["AZURE_CLIENT_ID"], "azure_client_secret": os.environ["AZURE_CLIENT_SECRET"], "azure_ssh_public_key": os.environ["AZURE_SSH_PUBLIC_KEY"], "azure_ssh_public_key_b64": os.environ["AZURE_SSH_PUBLIC_KEY_B64"], "master_vm_size": self.master_vm_size, "win_minion_count": self.win_minion_count, "win_minion_size": self.win_minion_size, "win_minion_image_rg": self.win_minion_image_rg, "win_minion_image_gallery": self.win_minion_image_gallery, "win_minion_image_definition": self.win_minion_image_definition, "win_minion_image_version": self.win_minion_image_version, "ci_version": self.ci_version, "flannel_mode": self.flannel_mode, "container_runtime": self.container_runtime, "k8s_bins": "k8sbins" in self.bins_built, "sdn_cni_bins": "sdncnibins" in self.bins_built, "containerd_bins": "containerdbins" in self.bins_built, "containerd_shim_bins": "containerdshim" in self.bins_built, } self.logging.info("Create CAPZ cluster") template_file = os.path.join(os.getcwd(), "cluster-api/azure/cluster.yaml.j2") output_file = "/tmp/capz-cluster.yaml" utils.render_template(template_file, output_file, context) utils.retry_on_error()(utils.run_shell_cmd)([ self.kubectl, "apply", "--kubeconfig", self.kind_kubeconfig_path, "-f", output_file ]) self._wait_for_cluster() self._setup_bootstrap_vm() context["bootstrap_vm_address"] = self.bootstrap_vm_private_ip self.upload_to_bootstrap_vm("%s/" % self.ci_artifacts_dir) self.upload_to_bootstrap_vm( os.path.join(os.getcwd(), "cluster-api/scripts")) self.logging.info("Create CAPZ control-plane") template_file = os.path.join( os.getcwd(), "cluster-api/azure/control-plane.yaml.j2") output_file = "/tmp/control-plane.yaml" utils.render_template(template_file, output_file, context) utils.retry_on_error()(utils.run_shell_cmd)([ self.kubectl, "apply", "--kubeconfig", self.kind_kubeconfig_path, "-f", output_file ]) self.logging.info("Create CAPZ Windows agents") template_file = os.path.join( os.getcwd(), "cluster-api/azure/windows-agents.yaml.j2") output_file = "/tmp/windows-agents.yaml" utils.render_template(template_file, output_file, context) utils.retry_on_error()(utils.run_shell_cmd)([ self.kubectl, "apply", "--kubeconfig", self.kind_kubeconfig_path, "-f", output_file ])