def _create_bootstrap_vm_nic(self):
self.logging.info("Creating bootstrap VM NIC")
public_ip = self._create_bootstrap_vm_public_ip()
control_plane_subnet = utils.retry_on_error()(
self.network_client.subnets.get)(
self.cluster_name, "%s-vnet" % self.cluster_name,
"%s-controlplane-subnet" % self.cluster_name)
nsg = self._create_bootstrap_vm_secgroup()
nic_parameters = {
"location":
self.azure_location,
"network_security_group": {
"id": nsg.id
},
"ip_configurations": [{
"name": "%s-ipconfig" % self.bootstrap_vm_nic_name,
"subnet": {
"id": control_plane_subnet.id
},
"public_ip_address": {
"id": public_ip.id
}
}]
}
return utils.retry_on_error()(
self.network_client.network_interfaces.create_or_update)(
self.cluster_name, self.bootstrap_vm_nic_name,
nic_parameters).result()
def _add_flannel_cni(self):
template_file = os.path.join(
os.getcwd(), "cluster-api/flannel/kube-flannel.yaml.j2")
context = {
"cluster_network_subnet": self.deployer.cluster_network_subnet,
"flannel_mode": self.opts.flannel_mode
}
kube_flannel = "/tmp/kube-flannel.yaml"
utils.render_template(template_file, kube_flannel, context)
server_core_tag = "windowsservercore-%s" % (
self.opts.base_container_image_tag)
mode = "overlay"
if self.opts.flannel_mode == constants.FLANNEL_MODE_L2BRIDGE:
mode = "l2bridge"
context = {
"server_core_tag": server_core_tag,
"container_runtime": self.opts.container_runtime,
"mode": mode
}
kube_flannel_windows = "/tmp/kube-flannel-windows.yaml"
searchpath = os.path.join(os.getcwd(), "cluster-api/flannel")
utils.render_template("kube-flannel-windows.yaml.j2",
kube_flannel_windows, context, searchpath)
cmd = [self.kubectl, "apply", "-f", kube_flannel]
utils.retry_on_error()(utils.run_shell_cmd)(cmd)
cmd = [self.kubectl, "apply", "-f", kube_flannel_windows]
utils.retry_on_error()(utils.run_shell_cmd)(cmd)
if self.opts.flannel_mode == constants.FLANNEL_MODE_OVERLAY:
self._set_vxlan_devices_mtu()
def upload_to_bootstrap_vm(self, local_path):
self.logging.info("Uploading %s to bootstrap VM", local_path)
ssh_cmd = ("ssh -i %s -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=/dev/null" % os.environ["SSH_KEY"])
cmd = [
"rsync", "--chmod=D755,F644", "-r", "-e",
"\"%s\"" % ssh_cmd, local_path,
"capi@%s:/www/" % self.bootstrap_vm_public_ip
]
utils.retry_on_error()(utils.run_shell_cmd)(cmd)
def _wait_for_control_plane(self, timeout=2700):
self.logging.info(
"Waiting up to %.2f minutes for the control-plane to be ready.",
timeout / 60.0)
sleep_time = 5
start = time.time()
while True:
elapsed = time.time() - start
if elapsed > timeout:
raise Exception("The control-plane didn't become ready "
"within %.2f minutes" % (timeout / 60.0))
cmd = [
self.kubectl, "get", "machine", "--kubeconfig",
self.kind_kubeconfig_path,
"--output=custom-columns=NAME:.metadata.name", "--no-headers"
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
machines = output.decode().strip().split('\n')
control_plane_machines = [
m for m in machines
if m.startswith("%s-control-plane" % self.cluster_name)
]
if len(control_plane_machines) == 0:
time.sleep(sleep_time)
continue
all_ready = True
for control_plane_machine in control_plane_machines:
cmd = [
self.kubectl, "get", "machine", "--kubeconfig",
self.kind_kubeconfig_path,
"--output=custom-columns=READY:.status.phase",
"--no-headers", control_plane_machine
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
status_phase = output.decode().strip()
if status_phase != "Running":
all_ready = False
continue
if all_ready:
self.logging.info(
"The control plane provisioned in "
"%.2f minutes", (time.time() - start) / 60.0)
break
time.sleep(sleep_time)
def _set_vxlan_devices_mtu(self):
self.logging.info(
"Set the proper MTU for the k8s master vxlan devices")
ssh_key_path = (os.environ.get("SSH_KEY")
or os.path.join(os.environ.get("HOME"), ".ssh/id_rsa"))
utils.retry_on_error()(utils.run_shell_cmd)([
"ssh", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", ssh_key_path,
"capi@%s" % self.deployer.master_public_address,
"'sudo bash -s' < %s" % os.path.join(
os.getcwd(),
"cluster-api/scripts/set-vxlan-devices-mtu.sh")
])
def on_joined(self, event):
if not self.cluster.is_ag_ready:
logger.warning('The availability group is not ready. Defering '
'hacluster on_joined until AG is ready.')
event.defer()
return
logger.info('Installing Microsoft SQL Server HA components')
retry_on_error()(apt_install)(packages=self.APT_PACKAGES, fatal=True)
self.setup_pacemaker_mssql_login()
rel_data = {
'resources': {
'ag_cluster': 'ocf:mssql:ag'
},
'resource_params': {
'ag_cluster':
'params ag_name="{ag_name}" '
'meta failure-timeout=60s '
'op start timeout=60s '
'op stop timeout=60s '
'op promote timeout=60s '
'op demote timeout=10s '
'op monitor timeout=60s interval=10s '
'op monitor timeout=60s interval=11s role="Master" '
'op monitor timeout=60s interval=12s role="Slave" '
'op notify timeout=60s'.format(ag_name=self.cluster.AG_NAME)
},
'ms': {
'ms-ag_cluster':
'ag_cluster meta '
'master-max="1" master-node-max="1" '
'clone-max="3" clone-node-max="1" notify="true"'
}
}
update_hacluster_vip('mssql', rel_data)
group_name = VIP_GROUP_NAME.format(service='mssql')
rel_data.update({
'colocations': {
'vip_on_master':
'inf: {} ms-ag_cluster:Master'.format(group_name)
},
'orders': {
'ag_first':
'inf: ms-ag_cluster:promote {}:start'.format(group_name)
}
})
rel = self.model.get_relation(event.relation.name, event.relation.id)
for k, v in rel_data.items():
rel.data[self.unit]['json_{}'.format(k)] = json.dumps(
v, **JSON_ENCODE_OPTIONS)
def _is_k8s_node_ready(self, node_name=None):
if not node_name:
self.logging.warning("Empty node_name parameter")
return False
cmd = [
self.kubectl, "get", "--kubeconfig", self.capz_kubeconfig_path,
"node", "-o", "yaml", node_name
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(cmd,
sensitive=True)
node = yaml.safe_load(output.decode('ascii'))
if "status" not in node:
self.logging.info("Node %s didn't report status yet", node_name)
return False
ready_condition = [
c for c in node["status"]["conditions"] if c["type"] == "Ready"
]
if len(ready_condition) == 0:
self.logging.info("Node %s didn't report ready condition yet",
node_name)
return False
try:
is_ready = strtobool(ready_condition[0]["status"])
except ValueError:
is_ready = False
if not is_ready:
self.logging.info("Node %s is not ready yet", node_name)
return False
return True
def _validate_k8s_api_container_images(self):
self.logging.info("Validating K8s API container images")
output, _ = utils.retry_on_error()(utils.run_shell_cmd)([
self.kubectl, "get", "nodes", "-o", "yaml", "-l",
"kubernetes.io/os=linux"
])
nodes = yaml.safe_load(output.decode("ascii"))
images_tag = self.ci_version.replace("+", "_").strip("v")
name_regex = re.compile(r"^(k8s.gcr.io/kube-.*):v(.*)$")
for node in nodes["items"]:
non_ci_images_names = []
for image in node["status"]["images"]:
non_ci_images_names += [
name for name in image["names"]
if (name_regex.match(name)
and name_regex.match(name).group(2) != images_tag)]
if len(non_ci_images_names) > 0:
self.logging.error(
"Found the following non-CI images %s on the "
"node %s.", non_ci_images_names,
node["metadata"]["name"])
raise Exception("Found non-CI container images on "
"node %s" % node["metadata"]["name"])
def _add_kube_proxy_windows(self):
template_file = os.path.join(
os.getcwd(), "cluster-api/kube-proxy/kube-proxy-windows.yaml.j2")
server_core_tag = "windowsservercore-%s" % (
self.opts.base_container_image_tag)
context = {
"kubernetes_version": self.kubernetes_version,
"server_core_tag": server_core_tag,
"enable_win_dsr": str(self.opts.enable_win_dsr).lower(),
"flannel_mode": self.opts.flannel_mode
}
output_file = "/tmp/kube-proxy-windows.yaml"
utils.render_template(template_file, output_file, context)
cmd = [self.kubectl, "apply", "-f", output_file]
utils.retry_on_error()(utils.run_shell_cmd)(cmd)
def _create_bootstrap_vm_secgroup(self):
self.logging.info("Creating bootstrap VM security group")
secgroup_rules = [
net_models.SecurityRule(
protocol="Tcp",
priority="1000",
source_port_range="*",
source_address_prefix="0.0.0.0/0",
destination_port_range="22",
destination_address_prefix="0.0.0.0/0",
destination_address_prefixes=[],
destination_application_security_groups=[],
access=net_models.SecurityRuleAccess.allow,
direction=net_models.SecurityRuleDirection.inbound,
name="Allow_SSH")
]
secgroup_params = net_models.NetworkSecurityGroup(
name=self.bootstrap_vm_secgroup_name,
location=self.azure_location,
security_rules=secgroup_rules)
return utils.retry_on_error()(
self.network_client.network_security_groups.create_or_update)(
self.cluster_name, self.bootstrap_vm_secgroup_name,
secgroup_params).result()
def _wait_for_bootstrap_vm(self, timeout=900):
self.logging.info("Waiting up to %.2f minutes for VM %s to provision",
timeout / 60.0, self.bootstrap_vm_name)
sleep_time = 5
start = time.time()
while True:
elapsed = time.time() - start
if elapsed > timeout:
err_msg = "VM %s didn't provision within %.2f minutes" % (
self.bootstrap_vm_name, timeout / 60)
self.logging.error(err_msg)
raise Exception(err_msg)
vm = utils.retry_on_error()(
self.compute_client.virtual_machines.get)(
self.cluster_name, self.bootstrap_vm_name)
if vm.provisioning_state == "Succeeded":
break
if vm.provisioning_state not in ("Creating", "Updating"):
err_msg = 'VM "%s" entered invalid state: "%s"' % (
self.bootstrap_vm_name, vm.provisioning_state)
self.logging.error(err_msg)
raise Exception(err_msg)
time.sleep(sleep_time)
return vm
def _get_agents_private_addresses(self, operating_system):
output, _ = utils.retry_on_error()(utils.run_shell_cmd)([
self.kubectl, "get", "machine", "--kubeconfig",
self.kind_kubeconfig_path, "-o", "yaml"
])
addresses = []
nodes = yaml.safe_load(output)
for node in nodes['items']:
if (operating_system == "linux"
and node["metadata"]["name"].startswith("capi-win")):
continue
if (operating_system == "windows"
and not node["metadata"]["name"].startswith("capi-win")):
continue
try:
node_addresses = [
n['address'] for n in node['status']['addresses']
if n['type'] == 'InternalIP'
]
except Exception as ex:
self.logging.warning(
"Cannot find private address for node %s. Exception "
"details: %s. Skipping", node["metadata"]["name"], ex)
# pick the first node internal address
addresses.append(node_addresses[0])
return addresses
def master_public_port(self):
output, _ = utils.retry_on_error()(utils.run_shell_cmd)([
self.kubectl, "get", "cluster", "--kubeconfig",
self.kind_kubeconfig_path, self.cluster_name, "-o",
"custom-columns=MASTER_PORT:.spec.controlPlaneEndpoint.port",
"--no-headers"
])
return output.decode("ascii").strip()
def _wait_for_cluster(self, timeout=900):
self.logging.info(
"Waiting up to %.2f minutes for the cluster to provision.",
timeout / 60.0)
sleep_time = 5
start = time.time()
cluster_resource_name = "cluster.cluster.x-k8s.io/%s" % (
self.cluster_name)
while True:
elapsed = time.time() - start
if elapsed > timeout:
raise Exception("Cluster didn't provision within %.2f "
"minutes" % (timeout / 60.0))
cmd = [
self.kubectl, "get", "cluster", "--kubeconfig",
self.kind_kubeconfig_path, self.cluster_name, "--output=name"
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
names = output.decode().strip().split('\n')
found = [c for c in names if c == cluster_resource_name]
if len(found) == 0:
time.sleep(sleep_time)
continue
cmd = [
self.kubectl, "get", "cluster", "--kubeconfig",
self.kind_kubeconfig_path, self.cluster_name,
"--output=custom-columns=CLUSTER_STATUS:.status.phase",
"--no-headers"
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
cluster_status = output.decode().strip()
if cluster_status == "Provisioned":
self.logging.info("Cluster provisioned in %.2f minutes",
(time.time() - start) / 60.0)
break
time.sleep(sleep_time)
def _create_bootstrap_vm_public_ip(self):
self.logging.info("Creating bootstrap VM public IP")
public_ip_parameters = {
"location": self.azure_location,
"public_ip_address_version": "IPV4"
}
return utils.retry_on_error()(
self.network_client.public_ip_addresses.create_or_update)(
self.cluster_name, self.bootstrap_vm_public_ip_name,
public_ip_parameters).result()
def enable_ip_forwarding(self):
self.logging.info("Enabling IP forwarding for the cluster VMs")
vm_nics = utils.retry_on_error()(
self.network_client.network_interfaces.list)(self.cluster_name)
for nic in vm_nics:
if nic.name == self.bootstrap_vm_nic_name:
continue
if nic.enable_ip_forwarding:
self.logging.info("IP forwarding is already enabled on nic %s",
nic.name)
continue
self.logging.info("Enabling IP forwarding on nic %s", nic.name)
nic_parameters = nic.as_dict()
nic_parameters["enable_ip_forwarding"] = True
utils.retry_on_error()(
self.network_client.network_interfaces.create_or_update)(
self.cluster_name, nic.name, nic_parameters).result()
def _setup_capz_kubeconfig(self):
self.logging.info("Setting up CAPZ kubeconfig")
cmd = [
self.kubectl, "get", "--kubeconfig", self.kind_kubeconfig_path,
"secret/%s-kubeconfig" % self.cluster_name,
"--output=custom-columns=KUBECONFIG_B64:.data.value",
"--no-headers"
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(cmd)
with open(self.capz_kubeconfig_path, 'w') as f:
f.write(base64.b64decode(output).decode('ascii'))
def setup_simplesamlphp(self):
if os.path.exists(self.DEST_DIR):
os.rmdir(self.DEST_DIR)
version = self.config.get('simple-saml-php-version')
archive_handler = ArchiveUrlFetchHandler()
retry_on_error()(archive_handler.install)(
source='{0}/v{1}/simplesamlphp-{1}.tar.gz'.format(
self.BASE_DOWNLOAD_URL, version),
dest=os.path.dirname(self.DEST_DIR))
os.rename('{0}-{1}'.format(self.DEST_DIR, version), self.DEST_DIR)
key_file = '{0}/cert/server.pem'.format(self.DEST_DIR)
cert_file = '{0}/cert/server.crt'.format(self.DEST_DIR)
ssl.generate_selfsigned(keyfile=key_file,
certfile=cert_file,
keysize=2048,
cn=get_unit_hostname())
uid = pwd.getpwnam(self.APACHE_USER).pw_uid
gid = grp.getgrnam(self.APACHE_GROUP).gr_gid
os.chown(key_file, uid, gid)
os.chown(cert_file, uid, gid)
def _prepull_images(self, timeout=3600):
prepull_yaml_path = "/tmp/prepull-windows-images.yaml"
utils.download_file(self.opts.prepull_yaml, prepull_yaml_path)
self.logging.info("Starting Windows images pre-pull")
utils.retry_on_error()(utils.run_shell_cmd)(
[self.kubectl, "apply", "-f", prepull_yaml_path])
self.logging.info(
"Waiting up to %.2f minutes to pre-pull Windows container images",
timeout / 60.0)
sleep_time = 5
start = time.time()
cmd = [self.kubectl, "get", "-o", "yaml", "-f", prepull_yaml_path]
while True:
elapsed = time.time() - start
if elapsed > timeout:
raise Exception("Couldn't pre-pull Windows images within "
"%.2f minutes." % (timeout / 60.0))
output, _ = utils.retry_on_error()(
utils.run_shell_cmd)(cmd, sensitive=True)
prepull_daemonset = yaml.safe_load(output.decode("ascii"))
if (prepull_daemonset["status"]["numberReady"] ==
prepull_daemonset["status"]["desiredNumberScheduled"]):
break
time.sleep(sleep_time)
self.logging.info("Windows images pre-pulled in %.2f minutes",
(time.time() - start) / 60.0)
self.logging.info("Cleaning up")
utils.run_shell_cmd(
[self.kubectl, "delete", "--wait", "-f", prepull_yaml_path])
def _cleanup_bootstrap_vm(self):
self.logging.info("Cleaning up the bootstrap VM")
self.logging.info("Deleting bootstrap VM")
utils.retry_on_error()(self.compute_client.virtual_machines.delete)(
self.cluster_name, self.bootstrap_vm_name).wait()
self.logging.info("Deleting bootstrap VM NIC")
utils.retry_on_error()(self.network_client.network_interfaces.delete)(
self.cluster_name, self.bootstrap_vm_nic_name).wait()
self.logging.info("Deleting bootstrap VM public IP")
utils.retry_on_error()(self.network_client.public_ip_addresses.delete)(
self.cluster_name, self.bootstrap_vm_public_ip_name).wait()
self.logging.info("Deleting bootstrap VM security group")
utils.retry_on_error()(
self.network_client.network_security_groups.delete)(
self.cluster_name, self.bootstrap_vm_secgroup_name).wait()
def _validate_k8s_api_versions(self):
self.logging.info("Validating K8s API versions")
output, _ = utils.retry_on_error()(
utils.run_shell_cmd)([self.kubectl, "get", "nodes", "-o", "yaml"])
nodes = yaml.safe_load(output.decode("ascii"))
for node in nodes["items"]:
node_name = node["metadata"]["name"]
node_info = node["status"]["nodeInfo"]
if node_info["kubeletVersion"] != self.ci_version:
raise Exception(
"Wrong kubelet version on node %s. "
"Expected %s, but found %s" %
(node_name, self.ci_version, node_info["kubeletVersion"]))
if node_info["kubeProxyVersion"] != self.ci_version:
raise Exception(
"Wrong kube-proxy version on node %s. "
"Expected %s, but found %s" %
(node_name, self.ci_version, node_info["kubeletVersion"]))
def _build_k8s_artifacts(self):
k8s_path = utils.get_k8s_folder()
utils.clone_git_repo(
self.opts.k8s_repo, self.opts.k8s_branch, k8s_path)
self.logging.info("Building K8s Linux binaries")
cmd = [
'make', 'WHAT="cmd/kubectl cmd/kubelet cmd/kubeadm"',
'KUBE_BUILD_PLATFORMS="linux/amd64"'
]
utils.run_shell_cmd(cmd, k8s_path)
del os.environ["KUBECTL_PATH"]
self.logging.info("Building K8s Windows binaries")
cmd = [
'make',
'WHAT="cmd/kubectl cmd/kubelet cmd/kubeadm cmd/kube-proxy"',
'KUBE_BUILD_PLATFORMS="windows/amd64"'
]
utils.run_shell_cmd(cmd, k8s_path)
self.logging.info("Building K8s Linux DaemonSet container images")
cmd = ['make', 'quick-release-images']
env = {"KUBE_FASTBUILD": "true",
"KUBE_BUILD_CONFORMANCE": "n"}
utils.retry_on_error()(utils.run_shell_cmd)(cmd, k8s_path, env)
kubeadm_bin = os.path.join(constants.KUBERNETES_LINUX_BINS_LOCATION,
'kubeadm')
out, _ = utils.run_shell_cmd(
[kubeadm_bin, "version", "-o=short"], k8s_path)
self.ci_version = out.decode().strip()
self.deployer.ci_version = self.ci_version
ci_artifacts_linux_bin_dir = "%s/%s/bin/linux/amd64" % (
self.ci_artifacts_dir, self.ci_version)
ci_artifacts_windows_bin_dir = "%s/%s/bin/windows/amd64" % (
self.ci_artifacts_dir, self.ci_version)
ci_artifacts_images_dir = "%s/%s/images" % (
self.ci_artifacts_dir, self.ci_version)
os.makedirs(ci_artifacts_linux_bin_dir, exist_ok=True)
os.makedirs(ci_artifacts_windows_bin_dir, exist_ok=True)
os.makedirs(ci_artifacts_images_dir, exist_ok=True)
for bin_name in ["kubectl", "kubelet", "kubeadm"]:
linux_bin_path = "%s/%s/%s" % (
k8s_path, constants.KUBERNETES_LINUX_BINS_LOCATION, bin_name)
shutil.copy(linux_bin_path, ci_artifacts_linux_bin_dir)
for bin_name in ["kubectl", "kubelet", "kubeadm", "kube-proxy"]:
win_bin_path = "%s/%s/%s.exe" % (
k8s_path, constants.KUBERNETES_WINDOWS_BINS_LOCATION, bin_name)
shutil.copy(win_bin_path, ci_artifacts_windows_bin_dir)
images_names = [
"kube-apiserver.tar", "kube-controller-manager.tar",
"kube-proxy.tar", "kube-scheduler.tar"
]
for image_name in images_names:
image_path = "%s/%s/%s" % (
k8s_path, constants.KUBERNETES_IMAGES_LOCATION,
image_name)
shutil.copy(image_path, ci_artifacts_images_dir)
def _connect_control_plane_to_node_subnet(self):
self.logging.info(
"Adding second NIC from node subnet to control plane")
control_plane_vm = self._get_control_plane_vm()
if not control_plane_vm:
raise Exception("The control plane VM was not found")
if len(control_plane_vm.network_profile.network_interfaces) > 1:
self.logging.info(
"The second NIC is already attached to the control plane")
return
nic_name = "%s-control-plane-node-nic" % self.cluster_name
node_subnet = utils.retry_on_error()(self.network_client.subnets.get)(
self.cluster_name, "%s-vnet" % self.cluster_name,
"%s-node-subnet" % self.cluster_name)
nic_parameters = {
"location":
self.azure_location,
"ip_configurations": [{
"name": "%s-ipconfig" % nic_name,
"subnet": {
"id": node_subnet.id
}
}]
}
self.logging.info("Creating the second NIC")
nic = utils.retry_on_error()(
self.network_client.network_interfaces.create_or_update)(
self.cluster_name, nic_name, nic_parameters).result()
self.logging.info("Deallocating the control plane VM")
utils.retry_on_error()(
self.compute_client.virtual_machines.deallocate)(
self.cluster_name, control_plane_vm.name).wait()
self.logging.info("Attaching the second NIC to the control plane VM")
vm_parameters = control_plane_vm.as_dict()
vm_parameters["network_profile"]["network_interfaces"].append({
"id":
nic.id,
"primary":
False
})
utils.retry_on_error()(
self.compute_client.virtual_machines.create_or_update)(
self.cluster_name, control_plane_vm.name,
vm_parameters).wait()
self.logging.info("Starting the control plane VM")
utils.retry_on_error()(self.compute_client.virtual_machines.start)(
self.cluster_name, control_plane_vm.name).wait()
utils.wait_for_port_connectivity(self.master_public_address, 22)
self.logging.info("Updating the control plane route")
control_plane_route = self._get_control_plane_route()
if not control_plane_route:
raise Exception("The control plane route was not found")
route_params = control_plane_route.as_dict()
route_params["next_hop_ip_address"] = \
nic.ip_configurations[0].private_ip_address
utils.retry_on_error()(self.network_client.routes.create_or_update)(
self.cluster_name, "%s-node-routetable" % self.cluster_name,
control_plane_route.name, route_params).wait()
def _create_bootstrap_vm(self):
self.logging.info("Setting up the bootstrap VM")
cloud_config_file = os.path.join(
os.getcwd(), "cluster-api/azure/bootstrap-vm-cloud-config.txt")
with open(cloud_config_file) as f:
cloud_config = f.read()
vm_nic = self._create_bootstrap_vm_nic()
vm_parameters = {
"location": self.azure_location,
"os_profile": {
"computer_name":
self.bootstrap_vm_name,
"admin_username":
"******",
"custom_data":
base64.b64encode(cloud_config.encode('ascii')).decode('ascii'),
"linux_configuration": {
"disable_password_authentication": True,
"ssh": {
"public_keys": [{
"key_data":
os.environ["AZURE_SSH_PUBLIC_KEY"],
"path":
"/home/capi/.ssh/authorized_keys"
}]
}
}
},
"hardware_profile": {
"vm_size": self.bootstrap_vm_size
},
"storage_profile": {
"image_reference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
"sku": "18_04-lts-gen2",
"version": "latest"
},
},
"network_profile": {
"network_interfaces": [{
"id": vm_nic.id
}]
}
}
self.logging.info("Creating bootstrap VM")
vm = utils.retry_on_error()(
self.compute_client.virtual_machines.create_or_update)(
self.cluster_name, self.bootstrap_vm_name,
vm_parameters).result()
vm = self._wait_for_bootstrap_vm()
ip_config = self.network_client.network_interfaces.get(
self.cluster_name, vm_nic.name).ip_configurations[0]
self.bootstrap_vm_private_ip = ip_config.private_ip_address
public_ip = self.network_client.public_ip_addresses.get(
self.cluster_name, self.bootstrap_vm_public_ip_name)
self.bootstrap_vm_public_ip = public_ip.ip_address
self._wait_for_ready_bootstrap_vm()
self.logging.info("Finished setting up the bootstrap VM")
return vm
def _wait_for_windows_agents(self, check_nodes_ready=True, timeout=3600):
self.logging.info("Waiting up to %.2f minutes for the Windows agents",
timeout / 60.0)
sleep_time = 5
start = time.time()
while True:
elapsed = time.time() - start
if elapsed > timeout:
raise Exception("The Windows agents didn't become ready "
"within %.2f minutes" % (timeout / 60.0))
cmd = [
self.kubectl, "get", "machine", "--kubeconfig",
self.kind_kubeconfig_path,
"--output=custom-columns=NAME:.metadata.name", "--no-headers"
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
machines = output.decode().strip().split('\n')
windows_machines = [
# This value is given in the capz cluster.yaml config, and
# it's hardcoded since it's going to be part of the Windows
# agents hostnames, and together with the unique suffix added
# by the cluster-api Azure provider, the length must be <= 15
# characters.
m for m in machines if m.startswith("capi-win-")
]
if len(windows_machines) == 0:
time.sleep(sleep_time)
continue
all_ready = True
for windows_machine in windows_machines:
cmd = [
self.kubectl, "get", "machine", "--kubeconfig",
self.kind_kubeconfig_path,
"--output=custom-columns=READY:.status.phase",
"--no-headers", windows_machine
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
status_phase = output.decode().strip()
if status_phase != "Running":
all_ready = False
continue
if not check_nodes_ready:
continue
cmd = [
self.kubectl, "get", "machine", "--kubeconfig",
self.kind_kubeconfig_path,
"--output=custom-columns=NODE_NAME:.status.nodeRef.name",
windows_machine, "--no-headers"
]
output, _ = utils.retry_on_error()(utils.run_shell_cmd)(
cmd, sensitive=True)
node_name = output.decode("ascii").strip()
all_ready = self._is_k8s_node_ready(node_name)
if all_ready:
self.logging.info("All the Windows agents are ready")
break
time.sleep(sleep_time)
def _run_remote_cmd(self, cmd, node_addresses):
for node_address in node_addresses:
utils.retry_on_error()(self.deployer.run_cmd_on_k8s_node)(
cmd, node_address)
def _upload_to(self, local_path, remote_path, node_addresses):
for node_address in node_addresses:
utils.retry_on_error()(self.deployer.upload_to_k8s_node)(
local_path, remote_path, node_address)
def on_install(self, _):
retry_on_error()(apt_update)(fatal=True)
retry_on_error()(apt_install)(packages=self.APT_PACKAGES, fatal=True)
self.setup_simplesamlphp()
self.setup_apache2()
from charmhelpers.fetch.archiveurl import ArchiveUrlFetchHandler
from ops.charm import CharmBase
from ops.model import ActiveStatus, BlockedStatus
from ops.main import main
from utils import render_configs, retry_on_error
logger = logging.getLogger(__name__)
try:
from lxml import etree # NOQA:F401
except ImportError:
logger.warning(
'Failed to import the lxml module. Re-installing the charm venv')
retry_on_error()(apt_update)(fatal=True)
retry_on_error()(apt_install)(packages=['python3-pip'], fatal=True)
pip_cmd = [
'pip3', 'install', '--upgrade', '--force-reinstall', '--target=venv',
'--requirement=requirements.txt'
]
retry_on_error()(subprocess.check_call)(pip_cmd)
from lxml import etree # NOQA:F401
class TestSamlIdpCharm(CharmBase):
BASE_DOWNLOAD_URL = ('https://github.com/simplesamlphp/simplesamlphp/'
'releases/download')
DEST_DIR = '/var/simplesamlphp'
APT_PACKAGES = [
'php', 'php-xml', 'php-date', 'php-mbstring', 'php-curl', 'apache2',
def _create_capz_cluster(self):
context = {
"cluster_name": self.cluster_name,
"cluster_network_subnet": self.cluster_network_subnet,
"azure_location": self.azure_location,
"azure_subscription_id": os.environ["AZURE_SUBSCRIPTION_ID"],
"azure_tenant_id": os.environ["AZURE_TENANT_ID"],
"azure_client_id": os.environ["AZURE_CLIENT_ID"],
"azure_client_secret": os.environ["AZURE_CLIENT_SECRET"],
"azure_ssh_public_key": os.environ["AZURE_SSH_PUBLIC_KEY"],
"azure_ssh_public_key_b64": os.environ["AZURE_SSH_PUBLIC_KEY_B64"],
"master_vm_size": self.master_vm_size,
"win_minion_count": self.win_minion_count,
"win_minion_size": self.win_minion_size,
"win_minion_image_rg": self.win_minion_image_rg,
"win_minion_image_gallery": self.win_minion_image_gallery,
"win_minion_image_definition": self.win_minion_image_definition,
"win_minion_image_version": self.win_minion_image_version,
"ci_version": self.ci_version,
"flannel_mode": self.flannel_mode,
"container_runtime": self.container_runtime,
"k8s_bins": "k8sbins" in self.bins_built,
"sdn_cni_bins": "sdncnibins" in self.bins_built,
"containerd_bins": "containerdbins" in self.bins_built,
"containerd_shim_bins": "containerdshim" in self.bins_built,
}
self.logging.info("Create CAPZ cluster")
template_file = os.path.join(os.getcwd(),
"cluster-api/azure/cluster.yaml.j2")
output_file = "/tmp/capz-cluster.yaml"
utils.render_template(template_file, output_file, context)
utils.retry_on_error()(utils.run_shell_cmd)([
self.kubectl, "apply", "--kubeconfig", self.kind_kubeconfig_path,
"-f", output_file
])
self._wait_for_cluster()
self._setup_bootstrap_vm()
context["bootstrap_vm_address"] = self.bootstrap_vm_private_ip
self.upload_to_bootstrap_vm("%s/" % self.ci_artifacts_dir)
self.upload_to_bootstrap_vm(
os.path.join(os.getcwd(), "cluster-api/scripts"))
self.logging.info("Create CAPZ control-plane")
template_file = os.path.join(
os.getcwd(), "cluster-api/azure/control-plane.yaml.j2")
output_file = "/tmp/control-plane.yaml"
utils.render_template(template_file, output_file, context)
utils.retry_on_error()(utils.run_shell_cmd)([
self.kubectl, "apply", "--kubeconfig", self.kind_kubeconfig_path,
"-f", output_file
])
self.logging.info("Create CAPZ Windows agents")
template_file = os.path.join(
os.getcwd(), "cluster-api/azure/windows-agents.yaml.j2")
output_file = "/tmp/windows-agents.yaml"
utils.render_template(template_file, output_file, context)
utils.retry_on_error()(utils.run_shell_cmd)([
self.kubectl, "apply", "--kubeconfig", self.kind_kubeconfig_path,
"-f", output_file
])
