def get_home_ssh_key():
"""
:return:
"""
key = None
if os.path.exists(os.path.expanduser("~/.ssh/id_rsa.pub")):
publickeyfile = os.path.expanduser("~/.ssh/id_rsa.pub")
with open(publickeyfile, 'r') as ssh:
key = ssh.read().rstrip()
elif os.path.exists(os.path.expanduser("~/.ssh/id_dsa.pub")):
publickeyfile = os.path.expanduser("~/.ssh/id_dsa.pub")
with open(publickeyfile, 'r') as ssh:
key = ssh.read().rstrip()
elif os.path.exists(os.path.expanduser("~/.kcli/id_rsa.pub")):
publickeyfile = os.path.expanduser("~/.kcli/id_rsa.pub")
with open(publickeyfile, 'r') as ssh:
key = ssh.read().rstrip()
elif os.path.exists(os.path.expanduser("~/.kcli/id_dsa.pub")):
publickeyfile = os.path.expanduser("~/.kcli/id_rda.pub")
with open(publickeyfile, 'r') as ssh:
key = ssh.read().rstrip()
else:
warning(
"neither id_rsa or id_dsa public keys found in your .ssh or .kcli directory, you might have trouble "
"accessing the vm")
return key
def update_cpus(self, name, numcpus):
nova = self.nova
try:
vm = nova.servers.find(name=name)
except:
error("VM %s not found" % name)
return {'result': 'failure', 'reason': "VM %s not found" % name}
currentflavor = nova.flavors.get(vm.flavor['id'])
if currentflavor.vcpus >= numcpus:
warning("No need to resize")
return {'result': 'success'}
allflavors = [f for f in nova.flavors.list() if f != currentflavor]
flavors = [
flavor for flavor in allflavors
if flavor.ram >= currentflavor.ram and flavor.vcpus >= numcpus
]
if flavors:
flavor = flavors[0]
pprint("Using flavor %s" % flavor.name)
vm.resize(flavor.id)
resizetimeout = 40
resizeruntime = 0
vmstatus = ''
while vmstatus != 'VERIFY_RESIZE':
if resizeruntime >= resizetimeout:
error("Time out waiting for resize to finish")
return {
'result': 'failure',
'reason': "Time out waiting for resize to finish"
}
vm = nova.servers.find(name=name)
vmstatus = vm.status
sleep(2)
pprint("Waiting for vm %s to be in verify_resize" % name)
resizeruntime += 2
vm.confirm_resize()
return {'result': 'success'}
else:
error("Couldn't find matching flavor for this number of cpus")
return {
'result': 'failure',
'reason':
"Couldn't find matching flavor for this number of cpus"
}
def update_etc_hosts(cluster, domain, host_ip, ingress_ip=None):
if not os.path.exists("/i_am_a_container"):
hosts = open("/etc/hosts").readlines()
wronglines = [e for e in hosts if not e.startswith('#') and "api.%s.%s" % (cluster, domain) in e and
host_ip not in e]
if ingress_ip is not None:
o = "oauth-openshift.apps.%s.%s" % (cluster, domain)
wrongingresses = [e for e in hosts if not e.startswith('#') and o in e and ingress_ip not in e]
wronglines.extend(wrongingresses)
for wrong in wronglines:
warning("Cleaning wrong entry %s in /etc/hosts" % wrong)
call("sudo sed -i '/%s/d' /etc/hosts" % wrong.strip(), shell=True)
hosts = open("/etc/hosts").readlines()
correct = [e for e in hosts if not e.startswith('#') and "api.%s.%s" % (cluster, domain) in e and
host_ip in e]
if not correct:
entries = ["api.%s.%s" % (cluster, domain)]
ingress_entries = ["%s.%s.%s" % (x, cluster, domain) for x in ['console-openshift-console.apps',
'oauth-openshift.apps', 'prometheus-k8s-openshift-monitoring.apps']]
if ingress_ip is None:
entries.extend(ingress_entries)
entries = ' '.join(entries)
call("sudo sh -c 'echo %s %s >> /etc/hosts'" % (host_ip, entries), shell=True)
if ingress_ip is not None:
entries = ' '.join(ingress_entries)
call("sudo sh -c 'echo %s %s >> /etc/hosts'" % (ingress_ip, entries), shell=True)
else:
entries = ["api.%s.%s" % (cluster, domain)]
ingress_entries = ["%s.%s.%s" % (x, cluster, domain) for x in ['console-openshift-console.apps',
'oauth-openshift.apps',
'prometheus-k8s-openshift-monitoring.apps']]
if ingress_ip is None:
entries.extend(ingress_entries)
entries = ' '.join(entries)
call("sh -c 'echo %s %s >> /etc/hosts'" % (host_ip, entries), shell=True)
if os.path.exists('/etcdir/hosts'):
call("sh -c 'echo %s %s >> /etcdir/hosts'" % (host_ip, entries), shell=True)
if ingress_ip is not None:
entries = ' '.join(ingress_entries)
call("sudo sh -c 'echo %s %s >> /etcdir/hosts'" % (ingress_ip, entries), shell=True)
else:
warning("Make sure to have the following entry in your /etc/hosts")
warning("%s %s" % (host_ip, entries))
def create(self,
name,
virttype=None,
profile='',
flavor=None,
plan='kvirt',
cpumodel='Westmere',
cpuflags=[],
cpupinning=[],
numcpus=2,
memory=512,
guestid='guestrhel764',
pool='default',
image=None,
disks=[{
'size': 10
}],
disksize=10,
diskthin=True,
diskinterface='virtio',
nets=['default'],
iso=None,
vnc=False,
cloudinit=True,
reserveip=False,
reservedns=False,
reservehost=False,
start=True,
keys=None,
cmds=[],
ips=None,
netmasks=None,
gateway=None,
nested=True,
dns=None,
domain=None,
tunnel=False,
files=[],
enableroot=True,
alias=[],
overrides={},
tags=[],
storemetadata=False,
sharedfolders=[],
kernel=None,
initrd=None,
cmdline=None,
cpuhotplug=False,
memoryhotplug=False,
numamode=None,
numa=[],
pcidevices=[],
tpm=False,
placement=[],
autostart=False,
rng=False,
metadata={},
securitygroups=[]):
"""
:param name:
:param virttype:
:param profile:
:param flavor:
:param plan:
:param cpumodel:
:param cpuflags:
:param cpupinning:
:param numcpus:
:param memory:
:param guestid:
:param pool:
:param image:
:param disks:
:param disksize:
:param diskthin:
:param diskinterface:
:param nets:
:param iso:
:param vnc:
:param cloudinit:
:param reserveip:
:param reservedns:
:param reservehost:
:param start:
:param keys:
:param cmds:
:param ips:
:param netmasks:
:param gateway:
:param nested:
:param dns:
:param domain:
:param tunnel:
:param files:
:param enableroot:
:param alias:
:param overrides:
:param tags:
:param cpuhotplug:
:param memoryhotplug:
:param numamode:
:param numa:
:param pcidevices:
:param tpm:
:return:
"""
reservation_id = overrides.get('hardware_reservation_id')
if reservation_id is not None:
reservations = self.conn.list_hardware_reservations(self.project)
if not reservations:
return {
'result': 'failure',
'reason': "No reserved hardware found"
}
elif reservation_id != 'next-available':
matching_ids = [
r.id for r in reservations
if r.id == reservation_id or r.short_id == reservation_id
]
if not matching_ids:
return {
'result':
'failure',
'reason':
"Reserved hardware with id %s not found" %
reservation_id
}
else:
reservation_id = matching_ids[0]
ipxe_script_url = None
userdata = None
networkid = None
networkids = []
vlan = False
for index, network in enumerate(nets):
if index > 1:
warning("Ignoring net higher than %s" % index)
break
if isinstance(network, str):
networkname = network
elif isinstance(network, dict) and 'name' in network:
networkname = network['name']
if networkname != 'default':
networks = [
n for n in self.conn.list_vlans(self.project)
if n.id == networkname or (n.description is not None and
n.description == networkname)
]
if not networks:
return {
'result': 'failure',
'reason': "Network %s not found" % networkname
}
else:
vlan = True
networkid = networks[0].id
else:
networkid = None
networkids.append(networkid)
if image is not None and not common.needs_ignition(image):
if '_' not in image and image in [
'rhel8', 'rhel7', 'centos7', 'centos8'
]:
image = image[:-1] + '_' + image[-1:]
pprint("Using image %s" % image)
found = False
for img in self.conn.list_operating_systems():
if img.slug == image:
found = True
if not found:
msg = "image %s doesn't exist" % image
return {'result': 'failure', 'reason': msg}
elif image is None:
ipxe_script_url = overrides.get('ipxe_script_url')
if ipxe_script_url is None:
return {
'result': 'failure',
'reason': 'You need to define ipxe_script_url as parameter'
}
image = 'custom_ipxe'
else:
ignition_url = overrides.get('ignition_url')
if ignition_url is None:
if self.tunnelhost is not None:
ignition_url = "http://%s/%s.ign" % (self.tunnelhost, name)
else:
return {
'result': 'failure',
'reason':
'You need to define ignition_url as parameter'
}
url = IMAGES[image]
if 'rhcos' in image:
if 'commit_id' in overrides:
kernel, initrd, metal = common.get_commit_rhcos_metal(
overrides['commit_id'])
else:
kernel, initrd, metal = common.get_latest_rhcos_metal(url)
elif 'fcos' in image:
kernel, initrd, metal = common.get_latest_fcos_metal(url)
interface = 'eth0' if 'fcos' in image else 'ens3f0'
userdata = self._ipxe(kernel, initrd, metal, ignition_url,
interface)
version = common.ignition_version(image)
ignitiondir = '/tmp'
ipv6 = []
ignitiondata = common.ignition(name=name,
keys=keys,
cmds=cmds,
nets=nets,
gateway=gateway,
dns=dns,
domain=domain,
reserveip=reserveip,
files=files,
enableroot=enableroot,
overrides=overrides,
version=version,
plan=plan,
ipv6=ipv6,
image=image)
image = 'custom_ipxe'
with open('%s/%s.ign' % (ignitiondir, name), 'w') as ignitionfile:
ignitionfile.write(ignitiondata)
if self.tunnelhost is not None:
pprint("Copying ignition data to %s" % self.tunnelhost)
scpcmd = "scp -qP %s /tmp/%s.ign %s@%s:%s/%s.ign" % (
self.tunnelport, name, self.tunneluser, self.tunnelhost,
self.tunneldir, name)
os.system(scpcmd)
if flavor is None:
# if f[1] >= numcpus and f[2] >= memory:
minmemory = 512000
for f in self.conn.list_plans():
if not f.specs:
continue
flavorname = f.name
# skip this flavor until we know where it can be launched
if flavorname == 'c3.small.x86' or (
vlan
and flavorname in ['t1.small.x86', 'c1.small.x86']):
continue
flavorcpus = int(f.specs['cpus'][0]['count'])
flavormemory = int(f.specs['memory']['total'].replace(
'GB', '')) * 1024
if flavorcpus >= 1 and flavormemory >= memory and flavormemory < minmemory:
flavor = flavorname
minmemory = flavormemory
validfacilities = f.available_in
if flavor is None:
return {
'result': 'failure',
'reason': 'Couldnt find flavor matching requirements'
}
pprint("Using flavor %s" % flavor)
else:
flavors = [f for f in self.conn.list_plans() if f.slug == flavor]
if not flavors:
return {
'result': 'failure',
'reason': 'Flavors %s not found' % flavor
}
else:
validfacilities = flavors[0].available_in
features = ['tpm'] if tpm else []
if cloudinit and userdata is None:
userdata = common.cloudinit(name=name,
keys=keys,
cmds=cmds,
nets=nets,
gateway=gateway,
dns=dns,
domain=domain,
reserveip=reserveip,
files=files,
enableroot=enableroot,
overrides=overrides,
fqdn=True,
storemetadata=storemetadata)[0]
validfacilities = [
os.path.basename(e['href']) for e in validfacilities
]
validfacilities = [
f.code for f in self.conn.list_facilities()
if f.id in validfacilities
]
if not validfacilities:
return {
'result': 'failure',
'reason': 'no valid facility found for flavor %s' % flavor
}
facility = overrides.get('facility')
if facility is not None:
matchingfacilities = [
f for f in self.conn.list_facilities() if f.slug == facility
]
if not matchingfacilities:
return {
'result': 'failure',
'reason': 'Facility %s not found' % facility
}
if facility not in validfacilities:
return {
'result':
'failure',
'reason':
'Facility %s not allowed. You should choose between %s' %
(facility, ','.join(validfacilities))
}
elif self.facility is not None:
if self.facility not in validfacilities:
return {
'result':
'failure',
'reason':
'Facility %s not allowed. You should choose between %s' %
(self.facility, ','.join(validfacilities))
}
facility = self.facility
else:
facility = validfacilities[0]
tags = ['project_%s' % self.project]
if userdata is not None and 'ignition' in userdata:
tags.append("kernel_%s" % os.path.basename(kernel))
for entry in [field for field in metadata if field in METADATA_FIELDS]:
tags.append("%s_%s" % (entry, metadata[entry]))
# ip_addresses = [{"address_family": 4, "public": True}, {"address_family": 6, "public": False}]
data = {
'project_id': self.project,
'hostname': name,
'plan': flavor,
'facility': facility,
'operating_system': image,
'userdata': userdata,
'features': features,
'tags': tags
}
if ipxe_script_url is not None:
data['ipxe_script_url'] = ipxe_script_url
if reservation_id is not None:
data['hardware_reservation_id'] = reservation_id
try:
device = self.conn.create_device(**data)
except Exception as e:
return {'result': 'failure', 'reason': e}
for networkid in networkids:
if networkid is None:
continue
elif 'cluster' in overrides and name.startswith(
"%s-" % overrides['cluster']):
warning(
"Not applying custom vlan to speed process for openshift..."
)
warning("This will be applied manually later...")
continue
status = 'provisioning'
while status != 'active':
status = self.info(name).get('status')
pprint("Waiting 5s for %s to be active..." % name)
sleep(5)
device_port_id = device["network_ports"][2]["id"]
self.conn.disbond_ports(device_port_id, False)
self.conn.assign_port(device_port_id, networkid)
break
return {'result': 'success'}
def create_network(self,
name,
cidr=None,
dhcp=True,
nat=True,
domain=None,
plan='kvirt',
overrides={}):
if nat:
externalnets = [
n for n in self.neutron.list_networks()['networks']
if n['router:external']
]
externalnet_id = externalnets[0]['id'] if externalnets else None
routers = [
router for router in self.neutron.list_routers()['routers']
if router['name'] == 'kvirt'
]
router_id = routers[0]['id'] if routers else None
try:
IPNetwork(cidr)
except:
return {'result': 'failure', 'reason': "Invalid Cidr %s" % cidr}
neutron = self.neutron
network_id = None
networks = {
net['name']: net['id']
for net in neutron.list_networks()['networks']
}
if name not in networks:
network = {'name': name, 'admin_state_up': True}
if 'port_security_enabled' in overrides:
network['port_security_enabled'] = bool(
overrides['port_security_enabled'])
network = neutron.create_network({'network': network})
network_id = network['network']['id']
tenant_id = network['network']['tenant_id']
else:
warning("Network already there. Creating subnet")
if cidr is not None:
if network_id is None:
network_id = networks[name]
cidrs = [
s['cidr'] for s in neutron.list_subnets()['subnets']
if s['network_id'] == network_id
]
if cidr not in cidrs:
subnet = {
'name': cidr,
'network_id': network_id,
'ip_version': 4,
"cidr": cidr,
'enable_dhcp': dhcp
}
if domain is not None:
subnet['dns_nameservers'] = [domain]
subnet = neutron.create_subnet({'subnet': subnet})
subnet_id = subnet['subnet']['id']
tenant_id = subnet['subnet']['tenant_id']
else:
warning("Subnet already there. Leaving")
return {'result': 'success'}
if nat:
if externalnet_id is not None:
if router_id is None:
router = {'name': 'kvirt', 'tenant_id': tenant_id}
# router['external_gateway_info'] = {"network_id": externalnet_id, "enable_snat": True}
router = neutron.create_router({'router': router})
router_id = router['router']['id']
router_dict = {"network_id": externalnet_id}
neutron.add_gateway_router(router_id, router_dict)
neutron.add_interface_router(router_id,
{'subnet_id': subnet_id})
return {'result': 'success'}
def create(config, plandir, cluster, overrides):
platform = config.type
data = {'kubetype': 'k3s', 'sdn': 'flannel'}
data.update(overrides)
data['cluster'] = overrides.get(
'cluster', cluster if cluster is not None else 'testk')
plan = cluster if cluster is not None else data['cluster']
data['kube'] = data['cluster']
masters = data.get('masters', 1)
network = data.get('network', 'default')
sdn = None if 'sdn' in overrides and overrides[
'sdn'] is None else data.get('sdn')
token = data.get('token', 'supersecret')
api_ip = data.get('api_ip')
if masters > 1:
if platform in cloudplatforms:
domain = data.get('domain', 'karmalabs.com')
api_ip = "%s-master.%s" % (cluster, domain)
elif api_ip is None:
if network == 'default' and platform == 'kvm':
warning("Using 192.168.122.253 as api_ip")
data['api_ip'] = "192.168.122.253"
api_ip = "192.168.122.253"
elif platform == 'kubevirt':
selector = {'kcli/plan': plan, 'kcli/role': 'master'}
api_ip = config.k.create_service("%s-api" % cluster,
config.k.namespace,
selector,
_type="LoadBalancer",
ports=[6443])
if api_ip is None:
sys.exit(1)
else:
pprint("Using api_ip %s" % api_ip)
data['api_ip'] = api_ip
else:
error("You need to define api_ip in your parameters file")
sys.exit(1)
if data.get('virtual_router_id') is None:
data['virtual_router_id'] = hash(data['cluster']) % 254 + 1
pprint("Using keepalived virtual_router_id %s" %
data['virtual_router_id'])
data['basedir'] = '/workdir' if container_mode() else '.'
install_k3s_args = []
for arg in data:
if arg.startswith('install_k3s'):
install_k3s_args.append("%s=%s" % (arg.upper(), data[arg]))
cluster = data.get('cluster')
clusterdir = os.path.expanduser("~/.kcli/clusters/%s" % cluster)
firstmaster = "%s-master-0" % cluster
if os.path.exists(clusterdir):
error("Please remove existing directory %s first..." % clusterdir)
sys.exit(1)
if find_executable('kubectl') is None:
get_kubectl()
if not os.path.exists(clusterdir):
os.makedirs(clusterdir)
os.mkdir("%s/auth" % clusterdir)
with open("%s/kcli_parameters.yml" % clusterdir, 'w') as p:
installparam = overrides.copy()
installparam['api_ip'] = api_ip
installparam['plan'] = plan
installparam['kubetype'] = 'k3s'
yaml.safe_dump(installparam,
p,
default_flow_style=False,
encoding='utf-8',
allow_unicode=True)
if os.path.exists("manifests") and os.path.isdir("manifests"):
data['files'] = [{
"path": "/root/manifests",
"currentdir": True,
"origin": "manifests"
}]
k = config.k
bootstrap_overrides = data.copy()
bootstrap_install_k3s_args = install_k3s_args.copy()
if sdn is None or sdn != 'flannel':
bootstrap_install_k3s_args.append(
"INSTALL_K3S_EXEC='--flannel-backend=none'")
bootstrap_install_k3s_args = ' '.join(bootstrap_install_k3s_args)
bootstrap_overrides['install_k3s_args'] = bootstrap_install_k3s_args
result = config.plan(plan,
inputfile='%s/bootstrap.yml' % plandir,
overrides=bootstrap_overrides)
if result['result'] != "success":
sys.exit(1)
nodes_overrides = data.copy()
nodes_install_k3s_args = install_k3s_args.copy()
if sdn is None or sdn != 'flannel':
nodes_install_k3s_args.append(
"INSTALL_K3S_EXEC='--flannel-backend=none'")
nodes_install_k3s_args = ' '.join(nodes_install_k3s_args)
nodes_overrides['install_k3s_args'] = nodes_install_k3s_args
if masters > 1:
pprint("Deploying extra masters")
threaded = data.get('threaded', False) or data.get(
'masters_threaded', False)
config.plan(plan,
inputfile='%s/masters.yml' % plandir,
overrides=nodes_overrides,
threaded=threaded)
with open("%s/join.sh" % clusterdir, 'w') as f:
if api_ip is None:
api_ip = k.info(firstmaster)['ip']
joincmd = f"curl -sfL https://get.k3s.io | K3S_URL=https://{api_ip}:6443 K3S_TOKEN={token} "
joincmd += ' '.join(install_k3s_args)
extra_args = data['extra_worker_args'] if data.get(
'extra_worker_args', []) else data.get('extra_args', [])
extra_args = ' '.join(extra_args)
f.write("%s sh -s - agent %s \n" % (joincmd, extra_args))
workers = data.get('workers', 0)
if workers > 0:
pprint("Deploying workers")
if 'name' in data:
del data['name']
os.chdir(os.path.expanduser("~/.kcli"))
threaded = data.get('threaded', False) or data.get(
'workers_threaded', False)
config.plan(plan,
inputfile='%s/workers.yml' % plandir,
overrides=data,
threaded=threaded)
success("K3s cluster %s deployed!!!" % cluster)
info2("export KUBECONFIG=$HOME/.kcli/clusters/%s/auth/kubeconfig" %
cluster)
info2("export PATH=$PWD:$PATH")
def create(config, plandir, cluster, overrides):
platform = config.type
k = config.k
data = {'kubetype': 'generic', 'xip': False, 'domain': 'karmalabs.com'}
data.update(overrides)
if 'keys' not in overrides and not os.path.exists(os.path.expanduser("~/.ssh/id_rsa.pub"))\
and not os.path.exists(os.path.expanduser("~/.ssh/id_dsa.pub"))\
and not os.path.exists(os.path.expanduser("~/.kcli/id_rsa.pub"))\
and not os.path.exists(os.path.expanduser("~/.kcli/id_dsa.pub")):
error(
"No usable public key found, which is required for the deployment")
os._exit(1)
data['cluster'] = overrides.get(
'cluster', cluster if cluster is not None else 'testk')
plan = cluster if cluster is not None else data['cluster']
data['kube'] = data['cluster']
masters = data.get('masters', 1)
if masters == 0:
error("Invalid number of masters")
os._exit(1)
network = data.get('network', 'default')
xip = data['xip']
api_ip = data.get('api_ip')
if platform in cloudplatforms:
domain = data.get('domain', 'karmalabs.com')
api_ip = "%s-master.%s" % (cluster, domain)
elif api_ip is None:
if network == 'default' and platform == 'kvm':
warning("Using 192.168.122.253 as api_ip")
data['api_ip'] = "192.168.122.253"
api_ip = "192.168.122.253"
elif platform == 'kubevirt':
selector = {'kcli/plan': plan, 'kcli/role': 'master'}
api_ip = config.k.create_service("%s-api" % cluster,
config.k.namespace,
selector,
_type="LoadBalancer",
ports=[6443])
if api_ip is None:
os._exit(1)
else:
pprint("Using api_ip %s" % api_ip)
data['api_ip'] = api_ip
else:
error("You need to define api_ip in your parameters file")
os._exit(1)
if xip and platform not in cloudplatforms:
data['domain'] = "%s.xip.io" % api_ip
if data.get('virtual_router_id') is None:
data['virtual_router_id'] = hash(data['cluster']) % 254 + 1
pprint("Using keepalived virtual_router_id %s" % data['virtual_router_id'])
version = data.get('version')
if version is not None and not str(version).startswith('1.'):
error("Invalid version %s" % version)
os._exit(1)
data['basedir'] = '/workdir' if os.path.exists(
"/i_am_a_container") else '.'
cluster = data.get('cluster')
image = data.get('image', 'centos7')
data['ubuntu'] = True if 'ubuntu' in image.lower() or [
entry for entry in UBUNTUS if entry in image
] else False
clusterdir = os.path.expanduser("~/.kcli/clusters/%s" % cluster)
firstmaster = "%s-master-0" % cluster
if os.path.exists(clusterdir):
error("Please remove existing directory %s first..." % clusterdir)
sys.exit(1)
if find_executable('kubectl') is None:
get_kubectl()
if not os.path.exists(clusterdir):
os.makedirs(clusterdir)
os.mkdir("%s/auth" % clusterdir)
with open("%s/kcli_parameters.yml" % clusterdir, 'w') as p:
installparam = overrides.copy()
installparam['api_ip'] = api_ip
installparam['virtual_router_id'] = data['virtual_router_id']
installparam['plan'] = plan
yaml.safe_dump(installparam,
p,
default_flow_style=False,
encoding='utf-8',
allow_unicode=True)
result = config.plan(plan,
inputfile='%s/masters.yml' % plandir,
overrides=data)
if result['result'] != "success":
os._exit(1)
source, destination = "/root/join.sh", "%s/join.sh" % clusterdir
firstmasterip, firstmastervmport = _ssh_credentials(k, firstmaster)[1:]
scpcmd = scp(firstmaster,
ip=firstmasterip,
user='******',
source=source,
destination=destination,
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
download=True,
insecure=True,
vmport=firstmastervmport)
os.system(scpcmd)
source, destination = "/etc/kubernetes/admin.conf", "%s/auth/kubeconfig" % clusterdir
scpcmd = scp(firstmaster,
ip=firstmasterip,
user='******',
source=source,
destination=destination,
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
download=True,
insecure=True,
vmport=firstmastervmport)
os.system(scpcmd)
workers = data.get('workers', 0)
if workers > 0:
pprint("Deploying workers")
if 'name' in data:
del data['name']
os.chdir(os.path.expanduser("~/.kcli"))
config.plan(plan, inputfile='%s/workers.yml' % plandir, overrides=data)
success("Kubernetes cluster %s deployed!!!" % cluster)
masters = data.get('masters', 1)
info2("export KUBECONFIG=$HOME/.kcli/clusters/%s/auth/kubeconfig" %
cluster)
info2("export PATH=$PWD:$PATH")
prefile = 'pre_ubuntu.sh' if data['ubuntu'] else 'pre_el.sh'
predata = config.process_inputfile(plan,
"%s/%s" % (plandir, prefile),
overrides=data)
with open("%s/pre.sh" % clusterdir, 'w') as f:
f.write(predata)
os.environ['KUBECONFIG'] = "%s/auth/kubeconfig" % clusterdir
apps = data.get('apps', [])
if apps:
os.environ["PATH"] += ":%s" % os.getcwd()
for app in apps:
appdir = "%s/apps/%s" % (plandir, app)
if not os.path.exists(appdir):
warning("Skipping unsupported app %s" % app)
else:
pprint("Adding app %s" % app)
if '%s_version' % app not in overrides:
data['%s_version' % app] = 'latest'
kube_create_app(config, appdir, overrides=data)
def create(config, plandir, cluster, overrides):
platform = config.type
data = {'kubetype': 'k3s'}
data.update(overrides)
data['cluster'] = overrides.get('cluster', cluster if cluster is not None else 'testk')
plan = cluster if cluster is not None else data['cluster']
data['kube'] = data['cluster']
masters = data.get('masters', 1)
network = data.get('network', 'default')
token = data.get('token', 'supersecret')
api_ip = data.get('api_ip')
if masters > 1:
if platform in cloudplatforms:
domain = data.get('domain', 'karmalabs.com')
api_ip = "%s-master.%s" % (cluster, domain)
elif api_ip is None:
if network == 'default' and platform == 'kvm':
warning("Using 192.168.122.253 as api_ip")
data['api_ip'] = "192.168.122.253"
api_ip = "192.168.122.253"
elif platform == 'kubevirt':
selector = {'kcli/plan': plan, 'kcli/role': 'master'}
api_ip = config.k.create_service("%s-api" % cluster, config.k.namespace, selector,
_type="LoadBalancer", ports=[6443])
if api_ip is None:
os._exit(1)
else:
pprint("Using api_ip %s" % api_ip)
data['api_ip'] = api_ip
else:
error("You need to define api_ip in your parameters file")
os._exit(1)
version = data.get('version', 'stable')
if version not in ['stable', 'latest', 'testing']:
error("Invalid version %s" % version)
os._exit(1)
data['basedir'] = '/workdir' if os.path.exists("/i_am_a_container") else '.'
cluster = data.get('cluster')
clusterdir = os.path.expanduser("~/.kcli/clusters/%s" % cluster)
firstmaster = "%s-master-0" % cluster
if os.path.exists(clusterdir):
error("Please remove existing directory %s first..." % clusterdir)
sys.exit(1)
if find_executable('kubectl') is None:
get_kubectl()
if not os.path.exists(clusterdir):
os.makedirs(clusterdir)
os.mkdir("%s/auth" % clusterdir)
with open("%s/kcli_parameters.yml" % clusterdir, 'w') as p:
installparam = overrides.copy()
installparam['api_ip'] = api_ip
installparam['plan'] = plan
yaml.safe_dump(installparam, p, default_flow_style=False, encoding='utf-8', allow_unicode=True)
if os.path.exists("manifests") and os.path.isdir("manifests"):
data['files'] = [{"path": "/root/manifests", "currentdir": True, "origin": "manifests"}]
k = config.k
result = config.plan(plan, inputfile='%s/bootstrap.yml' % plandir, overrides=data)
if result['result'] != "success":
os._exit(1)
if masters > 1:
pprint("Deploying extra masters")
if 'name' in data:
del data['name']
config.plan(plan, inputfile='%s/masters.yml' % plandir, overrides=data)
firstmasterip, firstmastervmport = _ssh_credentials(k, firstmaster)[1:]
with open("%s/join.sh" % clusterdir, 'w') as f:
if api_ip is None:
api_ip = k.info(firstmaster)['ip']
joincmd = "curl -sfL https://get.k3s.io | K3S_URL=https://%s:6443 K3S_TOKEN=%s" % (api_ip, token)
if data['sdn'] != "flannel":
joincmd += " INSTALL_K3S_EXEC='--no-flannel'"
f.write("%s sh -\n" % joincmd)
source, destination = "/root/kubeconfig", "%s/auth/kubeconfig" % clusterdir
scpcmd = scp(firstmaster, ip=firstmasterip, user='******', source=source, destination=destination,
tunnel=config.tunnel, tunnelhost=config.tunnelhost, tunnelport=config.tunnelport,
tunneluser=config.tunneluser, download=True, insecure=True, vmport=firstmastervmport)
os.system(scpcmd)
workers = data.get('workers', 0)
if workers > 0:
pprint("Deploying workers")
if 'name' in data:
del data['name']
os.chdir(os.path.expanduser("~/.kcli"))
config.plan(plan, inputfile='%s/workers.yml' % plandir, overrides=data)
success("K3s cluster %s deployed!!!" % cluster)
info2("export KUBECONFIG=$HOME/.kcli/clusters/%s/auth/kubeconfig" % cluster)
info2("export PATH=$PWD:$PATH")
def create(config, plandir, cluster, overrides):
log_level = 'debug' if config.debug else 'info'
os.environ["PATH"] += ":%s" % os.getcwd()
k = config.k
platform = config.type
arch = k.get_capabilities()['arch'] if config.type == 'kvm' else 'x86_64'
arch_tag = 'arm64' if arch in ['aarch64', 'arm64'] else 'latest'
overrides['arch_tag'] = arch_tag
if 'KUBECONFIG' not in os.environ:
error("Missing KUBECONFIG...")
sys.exit(1)
elif not os.path.isabs(os.environ['KUBECONFIG']):
os.environ['KUBECONFIG'] = "%s/%s" % (os.getcwd(),
os.environ['KUBECONFIG'])
data = {
'kubetype': 'hypershift',
'domain': 'karmalabs.com',
'network': 'default',
'workers': 2,
'async': False,
'tag': OPENSHIFT_TAG,
'version': 'nightly',
'namespace': 'mynamespace',
'pub_key': os.path.expanduser('~/.ssh/id_rsa.pub'),
'pull_secret': 'openshift_pull.json'
}
data.update(overrides)
if 'cluster' in overrides:
clustervalue = overrides.get('cluster')
elif cluster is not None:
clustervalue = cluster
else:
clustervalue = 'testk'
data['cluster'] = clustervalue
data['kube'] = data['cluster']
pprint("Deploying cluster %s" % clustervalue)
plan = cluster if cluster is not None else clustervalue
domain = data.get('domain')
version = data.get('version')
tag = data.get('tag')
if str(tag) == '4.1':
tag = '4.10'
data['tag'] = tag
data['basedir'] = '/workdir' if container_mode() else '.'
api_ip = os.popen(
"oc get node -o wide | grep master | head -1 | awk '{print $6}'").read(
).strip()
data['api_ip'] = api_ip
cluster = data.get('cluster')
namespace = data.get('namespace')
clusterdir = os.path.expanduser("~/.kcli/clusters/%s" % cluster)
if os.path.exists(clusterdir):
error("Please remove existing directory %s first..." % clusterdir)
sys.exit(1)
if find_executable('oc') is None:
get_oc()
pub_key = data.get('pub_key')
pull_secret = pwd_path(data.get('pull_secret'))
pull_secret = os.path.expanduser(pull_secret)
if not os.path.exists(pull_secret):
error("Missing pull secret file %s" % pull_secret)
sys.exit(1)
data['pull_secret'] = re.sub(r"\s", "", open(pull_secret).read())
if not os.path.exists(pub_key):
if os.path.exists(os.path.expanduser('~/.kcli/id_rsa.pub')):
pub_key = os.path.expanduser('~/.kcli/id_rsa.pub')
else:
error("Missing public key file %s" % pub_key)
sys.exit(1)
data['pub_key'] = open(pub_key).read().strip()
ingress_ip = data.get('ingress_ip')
cidr = '192.168.122.0/24'
if config.type in virtplatforms:
if ingress_ip is None:
network = data.get('network')
networkinfo = k.info_network(network)
if config.type == 'kvm' and networkinfo['type'] == 'routed':
cidr = networkinfo['cidr']
ingress_index = 3 if ':' in cidr else -4
ingress_ip = str(ip_network(cidr)[ingress_index])
warning("Using %s as ingress_ip" % ingress_ip)
overrides['ingress_ip'] = ingress_ip
else:
error("You need to define ingress_ip in your parameters file")
sys.exit(1)
if data.get('virtual_router_id') is None:
overrides['virtual_router_id'] = hash(cluster) % 254 + 1
pprint("Using keepalived virtual_router_id %s" %
overrides['virtual_router_id'])
pprint(f"Using {ingress_ip} for ingress vip....")
ipv6 = True if ':' in cidr else False
data['ipv6'] = ipv6
assetdata = data.copy()
if version == 'cluster':
release_image = os.popen(
"oc get clusterversion version -o jsonpath={'.status.history[-1].image'}"
).read()
version = os.popen(
"oc get clusterversion version -o jsonpath={'.status.history[-1].version'}"
).read()
minor = version[:3].replace('.', '')
image = f'rhcos{minor}'
data['image'] = image
else:
if os.path.exists('openshift-install'):
pprint("Removing old openshift-install")
os.remove('openshift-install')
if find_executable('openshift-install') is None:
if version == 'ci':
run = get_ci_installer(pull_secret, tag=tag)
elif version == 'nightly':
run = get_downstream_installer(nightly=True,
tag=tag,
pull_secret=pull_secret)
else:
run = get_downstream_installer(tag=tag,
pull_secret=pull_secret)
if run != 0:
error("Couldn't download openshift-install")
sys.exit(run)
pprint(
"Move downloaded openshift-install somewhere in your PATH if you want to reuse it"
)
else:
warning("Using existing openshift-install found in your PATH")
INSTALLER_VERSION = get_installer_version()
pprint("Using installer version %s" % INSTALLER_VERSION)
release_image = os.popen(
"openshift-install version | grep 'release image' | cut -f3 -d' '"
).read().strip()
image = data.get('image')
if image is None:
image_type = 'openstack' if data.get(
'kvm_openstack',
False) and config.type == 'kvm' else config.type
region = config.k.region if config.type == 'aws' else None
image_url = get_installer_rhcos(_type=image_type,
region=region,
arch=arch)
if platform in ['aws', 'gcp']:
image = image_url
else:
image = os.path.basename(os.path.splitext(image_url)[0])
if platform == 'ibm':
image = image.replace('.', '-').replace('_', '-').lower()
images = [v for v in k.volumes() if image in v]
if not images:
result = config.handle_host(
pool=config.pool,
image=image,
download=True,
update_profile=False,
url=image_url,
size=data.get('kubevirt_disk_size'))
if result['result'] != 'success':
sys.exit(1)
pprint("Using image %s" % image)
data['image'] = image
else:
pprint("Checking if image %s is available" % image)
images = [v for v in k.volumes() if image in v]
if not images:
error(
"Missing %s. Indicate correct image in your parameters file..."
% image)
sys.exit(1)
assetdata['release_image'] = release_image
if not os.path.exists(clusterdir):
os.makedirs(clusterdir)
os.mkdir("%s/auth" % clusterdir)
with open("%s/kcli_parameters.yml" % clusterdir, 'w') as p:
installparam = overrides.copy()
installparam['plan'] = plan
installparam['kubetype'] = 'hypershift'
installparam['api_ip'] = api_ip
installparam['ingress_ip'] = ingress_ip
installparam['image'] = image
installparam['ipv6'] = ipv6
yaml.safe_dump(installparam,
p,
default_flow_style=False,
encoding='utf-8',
allow_unicode=True)
assetdata['cidr'] = cidr
pprint("Creating control plane assets")
cmcmd = f"oc create ns {namespace} -o yaml --dry-run=client | oc apply -f -"
call(cmcmd, shell=True)
assetfile = config.process_inputfile(cluster,
f"{plandir}/assets.yaml",
overrides=assetdata)
with open(f"{clusterdir}/assets.yaml", 'w') as f:
f.write(assetfile)
cmcmd = f"oc create -f {clusterdir}/assets.yaml"
call(cmcmd, shell=True)
assetdata['clusterdir'] = clusterdir
ignitionscript = config.process_inputfile(cluster,
f"{plandir}/ignition.sh",
overrides=assetdata)
with open(f"{clusterdir}/ignition.sh", 'w') as f:
f.write(ignitionscript)
# TODO: monitor when ignition server is around
pprint("Waiting 2mn for ignition server to be usable")
time.sleep(120)
call(f'bash {clusterdir}/ignition.sh', shell=True)
pprint("Deploying workers")
if 'name' in data:
del data['name']
if data.get('baremetal', False):
result = config.plan(plan,
inputfile=f'{plandir}/kcli_plan.yml',
overrides=data,
onlyassets=True)
iso_data = result['assets'][0]
with open('iso.ign', 'w') as f:
f.write(iso_data)
ignitionfile = f'{cluster}-worker.ign'
with open(ignitionfile, 'w') as f:
f.write(iso_data)
iso_pool = data['pool'] or config.pool
generate_rhcos_iso(k, f"{cluster}-worker", iso_pool, installer=True)
else:
worker_threaded = data.get('threaded', False) or data.get(
'workers_threaded', False)
config.plan(plan,
inputfile=f'{plandir}/kcli_plan.yml',
overrides=data,
threaded=worker_threaded)
if data.get('ignore_hosts', False):
warning("Not updating /etc/hosts as per your request")
else:
update_etc_hosts(cluster, domain, api_ip, ingress_ip)
kubeconfigpath = f'{clusterdir}/auth/kubeconfig'
kubeconfig = os.popen(
f"oc extract -n {namespace} secret/{cluster}-admin-kubeconfig --to=-"
).read()
with open(kubeconfigpath, 'w') as f:
f.write(kubeconfig)
kubeadminpath = f'{clusterdir}/auth/kubeadmin-password'
kubeadmin = os.popen(
f"oc extract -n {namespace} secret/{cluster}-kubeadmin-password --to=-"
).read()
with open(kubeadminpath, 'w') as f:
f.write(kubeadmin)
autoapproverpath = f'{clusterdir}/autoapprovercron.yml'
autoapprover = config.process_inputfile(cluster,
f"{plandir}/autoapprovercron.yml",
overrides=data)
with open(autoapproverpath, 'w') as f:
f.write(autoapprover)
call(f"oc apply -f {autoapproverpath}", shell=True)
async_install = data.get('async')
if async_install or find_executable('openshift-install') is None:
success(f"Kubernetes cluster {cluster} deployed!!!")
info2(
f"export KUBECONFIG=$HOME/.kcli/clusters/{cluster}/auth/kubeconfig"
)
info2("export PATH=$PWD:$PATH")
else:
installcommand = 'openshift-install --dir=%s --log-level=%s wait-for install-complete' % (
clusterdir, log_level)
installcommand += " || %s" % installcommand
pprint(
"Launching install-complete step. It will be retried one extra time in case of timeouts"
)
call(installcommand, shell=True)
os.environ['KUBECONFIG'] = f"{clusterdir}/auth/kubeconfig"
apps = overrides.get('apps', [])
process_apps(config, clusterdir, apps, overrides)
def create(config, plandir, cluster, overrides):
k = config.k
bootstrap_helper_ip = None
client = config.client
platform = config.type
pprint("Deploying on client %s" % client)
data = {
'helper_image': 'CentOS-7-x86_64-GenericCloud.qcow2',
'domain': 'karmalabs.com',
'network': 'default',
'masters': 1,
'workers': 0,
'tag': DEFAULT_TAG,
'ipv6': False,
'pub_key': os.path.expanduser('~/.ssh/id_rsa.pub'),
'pull_secret': 'openshift_pull.json',
'version': 'nightly',
'macosx': False,
'upstream': False,
'fips': False,
'apps': [],
'minimal': False,
'dualstack': False,
'sno': False,
'sno_baremetal': False,
'sno_disk': 'vda'
}
data.update(overrides)
if 'cluster' in overrides:
clustervalue = overrides.get('cluster')
elif cluster is not None:
clustervalue = cluster
else:
clustervalue = 'testk'
data['cluster'] = clustervalue
domain = data.get('domain')
pprint("Deploying cluster %s" % clustervalue)
plan = cluster if cluster is not None else clustervalue
overrides['kubetype'] = 'openshift'
apps = overrides.get('apps', [])
if ('localstorage' in apps or 'ocs' in apps) and 'extra_disks' not in overrides\
and 'extra_master_disks' not in overrides and 'extra_worker_disks' not in overrides:
warning("Storage apps require extra disks to be set")
overrides['kube'] = data['cluster']
installparam = overrides.copy()
sno = data.get('sno', False)
ignore_hosts = data.get('ignore_hosts', False)
if sno:
sno_baremetal = data.get('sno_baremetal', False)
sno_disk = data.get('sno_disk', 'vda')
if sno_baremetal and 'vda' in sno_disk:
error("You need to define correct sno_disk for baremetal")
os._exit(1)
# tag = 'registry.svc.ci.openshift.org/sno-dev/openshift-bip:0.5.0'
masters = 1
workers = 0
data['mdns'] = False
data['kubetype'] = 'openshift'
data['kube'] = data['cluster']
masters = data.get('masters', 1)
if masters == 0:
error("Invalid number of masters")
os._exit(1)
network = data.get('network')
ipv6 = data['ipv6']
disconnected_deploy = data.get('disconnected_deploy', False)
disconnected_reuse = data.get('disconnected_reuse', False)
disconnected_url = data.get('disconnected_url')
disconnected_user = data.get('disconnected_user')
disconnected_password = data.get('disconnected_password')
disconnected_prefix = data.get('disconnected_prefix', 'ocp4')
dualstack = data.get('dualstack')
upstream = data.get('upstream')
version = data.get('version')
tag = data.get('tag')
if os.path.exists('openshift-install'):
pprint("Removing old openshift-install")
os.remove('openshift-install')
minimal = data.get('minimal')
if version not in ['ci', 'nightly']:
pprint("Using stable version")
else:
pprint("Using %s version" % version)
cluster = data.get('cluster')
helper_image = data.get('helper_image')
image = data.get('image')
api_ip = data.get('api_ip')
if platform in virtplatforms and not sno and api_ip is None:
if network == 'default' and platform == 'kvm':
warning("Using 192.168.122.253 as api_ip")
overrides['api_ip'] = "192.168.122.253"
api_ip = "192.168.122.253"
else:
error("You need to define api_ip in your parameters file")
os._exit(1)
if not sno and ':' in api_ip:
ipv6 = True
if ipv6:
data['network_type'] = 'OVNKubernetes'
data['ipv6'] = True
overrides['ipv6'] = True
if not disconnected_deploy and disconnected_url is None:
warning(
"Forcing disconnected_deploy to True as no disconnected_url was provided"
)
data['disconnected_deploy'] = True
disconnected_deploy = True
ingress_ip = data.get('ingress_ip')
if ingress_ip is not None and api_ip is not None and ingress_ip == api_ip:
ingress_ip = None
overrides['ingress_ip'] = None
public_api_ip = data.get('public_api_ip')
network = data.get('network')
if platform == 'packet':
if network == 'default':
error("You need to indicate a specific vlan network")
os._exit(1)
else:
facilities = [
n['domain'] for n in k.list_networks().values()
if str(n['cidr']) == network
]
if not facilities:
error("Vlan network %s not found in any facility" % network)
os._exit(1)
elif k.facility not in facilities:
error("Vlan network %s not found in facility %s" %
(network, k.facility))
os._exit(1)
masters = data.get('masters')
workers = data.get('workers')
tag = data.get('tag')
pub_key = data.get('pub_key')
pull_secret = pwd_path(data.get(
'pull_secret')) if not upstream else "%s/fake_pull.json" % plandir
pull_secret = os.path.expanduser(pull_secret)
macosx = data.get('macosx')
if macosx and not os.path.exists('/i_am_a_container'):
macosx = False
if platform == 'openstack' and (api_ip is None or public_api_ip is None):
error(
"You need to define both api_ip and public_api_ip in your parameters file"
)
os._exit(1)
if not os.path.exists(pull_secret):
error("Missing pull secret file %s" % pull_secret)
sys.exit(1)
if not os.path.exists(pub_key):
if os.path.exists(os.path.expanduser('~/.kcli/id_rsa.pub')):
pub_key = os.path.expanduser('~/.kcli/id_rsa.pub')
else:
error("Missing public key file %s" % pub_key)
sys.exit(1)
clusterdir = os.path.expanduser("~/.kcli/clusters/%s" % cluster)
if os.path.exists(clusterdir):
if [v for v in config.k.list() if v.get('plan', 'kvirt') == cluster]:
error("Please remove existing directory %s first..." % clusterdir)
sys.exit(1)
else:
pprint("Removing directory %s" % clusterdir)
rmtree(clusterdir)
os.environ['KUBECONFIG'] = "%s/auth/kubeconfig" % clusterdir
if find_executable('oc') is None:
get_oc(macosx=macosx)
if version == 'ci':
if '/' not in str(tag):
basetag = 'ocp' if not upstream else 'origin'
tag = 'registry.ci.openshift.org/%s/release:%s' % (basetag, tag)
os.environ['OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE'] = tag
pprint("Setting OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE to %s" % tag)
if find_executable('openshift-install') is None:
if version == 'ci':
run = get_ci_installer(pull_secret, tag=tag, upstream=upstream)
elif version == 'nightly':
run = get_downstream_installer(nightly=True, tag=tag)
elif upstream:
run = get_upstream_installer(tag=tag)
else:
run = get_downstream_installer(tag=tag)
if run != 0:
error("Couldn't download openshift-install")
os._exit(run)
pprint(
"Move downloaded openshift-install somewhere in your path if you want to reuse it"
)
os.environ["PATH"] += ":%s" % os.getcwd()
if disconnected_url is not None:
if '/' not in str(tag):
tag = '%s/%s/release:%s' % (disconnected_url, disconnected_prefix,
tag)
os.environ['OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE'] = tag
pprint("Setting OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE to %s" % tag)
INSTALLER_VERSION = get_installer_version()
COMMIT_ID = os.popen('openshift-install version').readlines()[1].replace(
'built from commit', '').strip()
if platform == 'packet' and not upstream:
overrides['commit_id'] = COMMIT_ID
pprint("Using installer version %s" % INSTALLER_VERSION)
OPENSHIFT_VERSION = INSTALLER_VERSION[0:3].replace('.', '')
curl_header = "Accept: application/vnd.coreos.ignition+json; version=3.1.0"
if upstream:
curl_header = "User-Agent: Ignition/2.3.0"
elif OPENSHIFT_VERSION.isdigit() and int(OPENSHIFT_VERSION) < 46:
curl_header = "User-Agent: Ignition/0.35.0"
overrides['curl_header'] = curl_header
if sno:
pass
elif image is None:
if upstream:
fcos_base = 'stable' if version == 'stable' else 'testing'
fcos_url = "https://builds.coreos.fedoraproject.org/streams/%s.json" % fcos_base
image_url = get_latest_fcos(fcos_url, _type=config.type)
else:
image_url = get_commit_rhcos(COMMIT_ID, _type=config.type)
image = os.path.basename(os.path.splitext(image_url)[0])
images = [v for v in k.volumes() if image in v]
if not images:
result = config.handle_host(pool=config.pool,
image=image,
download=True,
update_profile=False,
url=image_url)
if result['result'] != 'success':
os._exit(1)
else:
pprint("Using image %s" % image)
elif platform != 'packet':
pprint("Checking if image %s is available" % image)
images = [v for v in k.volumes() if image in v]
if not images:
error(
"Missing %s. Indicate correct image in your parameters file..."
% image)
os._exit(1)
else:
error(
"Missing image in your parameters file. This is required for packet"
)
os._exit(1)
overrides['image'] = image
overrides['cluster'] = cluster
if not os.path.exists(clusterdir):
os.makedirs(clusterdir)
with open("%s/kcli_parameters.yml" % clusterdir, 'w') as p:
installparam['plan'] = plan
yaml.safe_dump(installparam,
p,
default_flow_style=False,
encoding='utf-8',
allow_unicode=True)
data['pub_key'] = open(pub_key).read().strip()
if platform in virtplatforms and disconnected_deploy:
if platform == 'kvm' and network in [
n for n in k.list_networks()
if k.list_networks()[n]['type'] == 'routed'
]:
data['disconnected_dns'] = True
disconnected_vm = "%s-disconnecter" % cluster
pprint("Deploying disconnected vm %s" % disconnected_vm)
data['pull_secret'] = re.sub(r"\s", "", open(pull_secret).read())
disconnected_plan = "%s-reuse" % plan if disconnected_reuse else plan
if version == 'ci' and 'disconnected_origin' not in overrides:
warning("Forcing disconnected_origin to registry.ci.openshift.org")
data['disconnected_origin'] = "registry.ci.openshift.org"
result = config.plan(disconnected_plan,
inputfile='%s/disconnected.yml' % plandir,
overrides=data)
if result['result'] != 'success':
os._exit(1)
disconnected_ip, disconnected_vmport = _ssh_credentials(
k, disconnected_vm)[1:]
cacmd = "cat /opt/registry/certs/domain.crt"
cacmd = ssh(disconnected_vm,
ip=disconnected_ip,
user='******',
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
insecure=True,
cmd=cacmd,
vmport=disconnected_vmport)
disconnected_ca = os.popen(cacmd).read().strip()
if data['ca'] is not None:
data['ca'] += disconnected_ca
else:
data['ca'] = disconnected_ca
urlcmd = "cat /root/url.txt"
urlcmd = ssh(disconnected_vm,
ip=disconnected_ip,
user='******',
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
insecure=True,
cmd=urlcmd,
vmport=disconnected_vmport)
disconnected_url = os.popen(urlcmd).read().strip()
overrides['disconnected_url'] = disconnected_url
data['disconnected_url'] = disconnected_url
if disconnected_user is None:
disconnected_user = '******'
if disconnected_password is None:
disconnected_password = '******'
versioncmd = "cat /root/version.txt"
versioncmd = ssh(disconnected_vm,
ip=disconnected_ip,
user='******',
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
insecure=True,
cmd=versioncmd,
vmport=disconnected_vmport)
disconnected_version = os.popen(versioncmd).read().strip()
os.environ[
'OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE'] = disconnected_version
pprint("Setting OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE to %s" %
disconnected_version)
if disconnected_url is not None and disconnected_user is not None and disconnected_password is not None:
key = "%s:%s" % (disconnected_user, disconnected_password)
key = str(b64encode(key.encode('utf-8')), 'utf-8')
auths = {
'auths': {
disconnected_url: {
'auth': key,
'email': '*****@*****.**'
}
}
}
data['pull_secret'] = json.dumps(auths)
else:
data['pull_secret'] = re.sub(r"\s", "", open(pull_secret).read())
installconfig = config.process_inputfile(cluster,
"%s/install-config.yaml" %
plandir,
overrides=data)
with open("%s/install-config.yaml" % clusterdir, 'w') as f:
f.write(installconfig)
with open("%s/install-config.yaml.bck" % clusterdir, 'w') as f:
f.write(installconfig)
autoapprover = config.process_inputfile(cluster,
"%s/autoapprovercron.yml" %
plandir,
overrides=data)
with open("%s/autoapprovercron.yml" % clusterdir, 'w') as f:
f.write(autoapprover)
run = call('openshift-install --dir=%s create manifests' % clusterdir,
shell=True)
if run != 0:
error("Leaving environment for debugging purposes")
error("You can delete it with kcli delete kube --yes %s" % cluster)
os._exit(run)
if minimal:
warning("Deploying cvo overrides to provide a minimal install")
with open("%s/cvo-overrides.yaml" % plandir) as f:
cvo_override = f.read()
with open("%s/manifests/cvo-overrides.yaml" % clusterdir, "a") as f:
f.write(cvo_override)
if ipv6:
for role in ['master', 'worker']:
blacklist = config.process_inputfile(cluster,
"%s/99-blacklist-ipi.yaml" %
plandir,
overrides={'role': role})
with open(
"%s/openshift/99-blacklist-ipi-%s.yaml" %
(clusterdir, role), 'w') as f:
f.write(blacklist)
for f in glob("%s/customisation/*.yaml" % plandir):
if '99-ingress-controller.yaml' in f:
ingressrole = 'master' if workers == 0 else 'worker'
replicas = masters if workers == 0 else workers
ingressconfig = config.process_inputfile(cluster,
f,
overrides={
'replicas': replicas,
'role': ingressrole
})
with open("%s/openshift/99-ingress-controller.yaml" % clusterdir,
'w') as f:
f.write(ingressconfig)
else:
copy2(f, "%s/openshift" % clusterdir)
manifestsdir = pwd_path("manifests")
if os.path.exists(manifestsdir) and os.path.isdir(manifestsdir):
for f in glob("%s/*.yaml" % manifestsdir):
copy2(f, "%s/openshift" % clusterdir)
if 'network_type' in data and data['network_type'] == 'Calico':
for asset in calicoassets:
fetch(asset, manifestsdir)
if 'network_type' in data and data['network_type'] == 'Contrail':
pprint("Fetching contrail assets")
for asset in contrail_manifests:
fetch(asset, "%s/manifests" % clusterdir)
for asset in contrail_openshifts:
fetch(asset, "%s/openshift" % clusterdir)
contrail_registry = data.get('contrail_registry', "hub.juniper.net")
contrail_user = data.get('contrail_user')
contrail_password = data.get('contrail_password')
if contrail_user is None:
error("Missing contrail_user")
os._exit(1)
if contrail_password is None:
error("Missing contrail_password")
os._exit(1)
contrail_creds = "%s:%s" % (contrail_user, contrail_password)
contrail_auth = b64encode(contrail_creds.encode()).decode("UTF-8")
contrail_auth = {
"auths": {
contrail_registry: {
"username": contrail_user,
"password": contrail_password,
"auth": contrail_auth
}
}
}
contrail_auth = json.dumps(contrail_auth)
contrail_data = {
'contrail_auth': b64encode(contrail_auth.encode()).decode("UTF-8")
}
contrail_secret = config.process_inputfile(
cluster,
"%s/contrail_registry_secret.j2" % plandir,
overrides=contrail_data)
with open(
"%s/manifests/00-contrail-02-registry-secret.yaml" %
clusterdir, 'w') as f:
f.write(contrail_secret)
if dualstack:
copy2("%s/dualstack.yml" % plandir, "%s/openshift" % clusterdir)
if sno:
sno_name = "%s-sno" % cluster
sno_dns = data.get('sno_dns', True)
run = call(
'openshift-install --dir=%s create single-node-ignition-config' %
clusterdir,
shell=True)
if run != 0:
error("Hit issue.Leaving")
os._exit(run)
os.rename("%s/bootstrap-in-place-for-live-iso.ign" % clusterdir,
"./%s.ign" % sno_name)
with open("iso.ign", 'w') as f:
if sno_dns:
_files = [{
"path": "/root/sno-finish.service",
"origin": "%s/sno-finish.service" % plandir
}, {
"path": "/usr/local/bin/sno-finish.sh",
"origin": "%s/sno-finish.sh" % plandir,
"mode": 700
}, {
"path": "/root/coredns.yml",
"origin": "%s/staticpods/coredns.yml" % plandir
}, {
"path": "/root/Corefile",
"origin": "%s/Corefile" % plandir
}, {
"path": "/root/99-forcedns",
"origin": "%s/99-forcedns" % plandir
}]
iso_overrides = {'files': _files}
else:
iso_overrides = {}
iso_overrides.update(data)
result = config.create_vm(sno_name,
'rhcos46',
overrides=iso_overrides,
onlyassets=True)
pprint("Writing iso.ign to current dir")
f.write(result['data'])
if config.type != 'kvm':
pprint("Additional workflow not available on %s" % config.type)
pprint("Embed iso.ign in rhcos live iso")
os._exit(0)
else:
generate_rhcos_iso(k, cluster, data['pool'])
if not sno_baremetal:
pprint("Deploying sno")
result = config.plan(plan,
inputfile='%s/sno.yml' % plandir,
overrides=data)
if result['result'] != 'success':
os._exit(1)
if ignore_hosts:
warning("Not updating /etc/hosts as per your request")
else:
while api_ip is None:
api_ip = k.info(sno_name).get('ip')
pprint("Waiting 5s to retrieve sno ip...")
sleep(5)
update_etc_hosts(cluster, domain, api_ip)
installcommand = 'openshift-install --dir=%s --log-level=debug wait-for install-complete' % clusterdir
installcommand += " || %s" % installcommand
pprint(
"Launching install-complete step. It will be retried one extra time in case of timeouts"
)
call(installcommand, shell=True)
else:
warning(
"You might need to create manual entries in /etc/hosts to reach the sno installation"
)
os._exit(0)
call('openshift-install --dir=%s create ignition-configs' % clusterdir,
shell=True)
for role in ['master', 'worker']:
ori = "%s/%s.ign" % (clusterdir, role)
copy2(ori, "%s.ori" % ori)
if masters < 3:
version_match = re.match("4.([0-9]*).*", INSTALLER_VERSION)
COS_VERSION = "4%s" % version_match.group(
1) if version_match is not None else '45'
if not upstream and int(COS_VERSION) > 43:
bootstrap_patch = open('%s/bootstrap_patch.sh' % plandir).read()
bootstrap_service = open('%s/bootstrap_patch.service' %
plandir).read()
patch_bootstrap("%s/bootstrap.ign" % clusterdir, bootstrap_patch,
bootstrap_service)
staticdata = gather_dhcp(data, platform)
if staticdata:
pprint("Deploying helper dhcp node" % image)
staticdata.update({
'network': network,
'dhcp_image': helper_image,
'prefix': cluster,
domain: '%s.%s' % (cluster, domain)
})
result = config.plan(plan,
inputfile='%s/dhcp.yml' % plandir,
overrides=staticdata)
if result['result'] != 'success':
os._exit(1)
if platform in virtplatforms:
if data.get('virtual_router_id') is None:
overrides['virtual_router_id'] = hash(cluster) % 254 + 1
pprint("Using keepalived virtual_router_id %s" %
overrides['virtual_router_id'])
pprint("Using %s for api vip...." % api_ip)
host_ip = api_ip if platform != "openstack" else public_api_ip
if ignore_hosts:
warning("Ignoring /etc/hosts")
else:
update_etc_hosts(cluster, domain, host_ip, ingress_ip)
if platform in [
'openstack', 'vsphere'
] or (platform == 'packet' and config.k.tunnelhost is None):
# bootstrap ignition is too big in those platforms so we deploy a temporary web server to serve it
helper_overrides = {}
if helper_image is None:
images = [
v for v in k.volumes()
if 'centos' in v.lower() or 'fedora' in v.lower()
]
if images:
image = os.path.basename(images[0])
else:
helper_image = "CentOS-7-x86_64-GenericCloud.qcow2"
pprint("Downloading centos helper image")
result = config.handle_host(pool=config.pool,
image="centos7",
download=True,
update_profile=False)
pprint("Using helper image %s" % helper_image)
else:
images = [v for v in k.volumes() if helper_image in v]
if not images:
error(
"Missing image %s. Indicate correct helper image in your parameters file"
% helper_image)
os._exit(1)
if platform == 'openstack':
helper_overrides['flavor'] = "m1.medium"
helper_overrides['nets'] = [network]
helper_overrides['plan'] = cluster
bootstrap_helper_name = "%s-bootstrap-helper" % cluster
cmds = ["iptables -F", "yum -y install httpd", "setenforce 0"]
if platform == 'packet':
config.k.tunnelhost = bootstrap_helper_ip
cmds.append(
"sed -i 's/apache/root/' /etc/httpd/conf/httpd.conf")
cmds.append("systemctl enable --now httpd")
helper_overrides['cmds'] = cmds
config.create_vm("%s-bootstrap-helper" % cluster,
helper_image,
overrides=helper_overrides,
wait=True)
bootstrap_helper_ip, bootstrap_helper_vmport = _ssh_credentials(
bootstrap_helper_name)[1:]
source, destination = "%s/bootstrap.ign" % clusterdir, "/var/www/html/bootstrap"
scpcmd = scp(bootstrap_helper_name,
ip=bootstrap_helper_ip,
user='******',
source=source,
destination=destination,
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
download=False,
insecure=True,
vmport=bootstrap_helper_vmport)
os.system(scpcmd)
cmd = "chown apache.apache /var/www/html/bootstrap"
sshcmd = ssh(bootstrap_helper_name,
ip=bootstrap_helper_ip,
user='******',
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
insecure=True,
cmd=cmd,
vmport=bootstrap_helper_vmport)
os.system(sshcmd)
sedcmd = 'sed "s@https://api-int.%s.%s:22623/config/master@http://%s/bootstrap@" ' % (
cluster, domain, bootstrap_helper_ip)
sedcmd += '%s/master.ign' % clusterdir
sedcmd += ' > %s/bootstrap.ign' % clusterdir
call(sedcmd, shell=True)
sedcmd = 'sed "s@https://api-int.%s.%s:22623/config/master@http://%s/worker@" ' % (
cluster, domain, bootstrap_helper_ip)
sedcmd += '%s/master.ign' % clusterdir
sedcmd += ' > %s/worker.ign' % clusterdir
call(sedcmd, shell=True)
new_api_ip = api_ip if not ipv6 else "[%s]" % api_ip
sedcmd = 'sed -i "s@https://api-int.%s.%s:22623/config@http://%s:22624/config@"' % (
cluster, domain, new_api_ip)
sedcmd += ' %s/master.ign %s/worker.ign' % (clusterdir, clusterdir)
call(sedcmd, shell=True)
if platform in cloudplatforms:
bootstrap_helper_name = "%s-bootstrap-helper" % cluster
helper_overrides = {
'reservedns': True,
'domain': '%s.%s' % (cluster, domain),
'tags': [tag],
'plan': cluster,
'nets': [network]
}
config.create_vm("%s-bootstrap-helper" % cluster,
helper_image,
overrides=helper_overrides)
status = ""
while status != "running":
status = k.info(bootstrap_helper_name).get('status')
pprint("Waiting 5s for bootstrap helper node to be running...")
sleep(5)
sleep(5)
bootstrap_helper_ip, bootstrap_helper_vmport = _ssh_credentials(
k, bootstrap_helper_name)[1:]
cmd = "iptables -F ; yum -y install httpd ; systemctl start httpd"
sshcmd = ssh(bootstrap_helper_name,
ip=bootstrap_helper_ip,
user='******',
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
insecure=True,
cmd=cmd,
vmport=bootstrap_helper_vmport)
os.system(sshcmd)
source, destination = "%s/bootstrap.ign" % clusterdir, "/var/www/html/bootstrap"
scpcmd = scp(bootstrap_helper_name,
ip=bootstrap_helper_ip,
user='******',
source=source,
destination=destination,
tunnel=config.tunnel,
tunnelhost=config.tunnelhost,
tunnelport=config.tunnelport,
tunneluser=config.tunneluser,
download=False,
insecure=True,
vmport=bootstrap_helper_vmport)
os.system(scpcmd)
sedcmd = 'sed "s@https://api-int.%s.%s:22623/config/master@' % (
cluster, domain)
sedcmd += 'http://%s-bootstrap-helper.%s.%s/bootstrap@ "' % (
cluster, cluster, domain)
sedcmd += '%s/master.ign' % clusterdir
sedcmd += ' > %s/bootstrap.ign' % clusterdir
call(sedcmd, shell=True)
if platform in virtplatforms:
pprint("Deploying bootstrap")
result = config.plan(plan,
inputfile='%s/bootstrap.yml' % plandir,
overrides=overrides)
if result['result'] != 'success':
os._exit(1)
pprint("Deploying masters")
result = config.plan(plan,
inputfile='%s/masters.yml' % plandir,
overrides=overrides)
if result['result'] != 'success':
os._exit(1)
if platform == 'packet':
allnodes = ["%s-bootstrap" % cluster] + [
"%s-master-%s" % (cluster, num) for num in range(masters)
]
for node in allnodes:
try:
k.add_nic(node, network)
except Exception as e:
error("Hit %s. Continuing still" % str(e))
continue
bootstrapcommand = 'openshift-install --dir=%s wait-for bootstrap-complete' % clusterdir
bootstrapcommand += ' || %s' % bootstrapcommand
run = call(bootstrapcommand, shell=True)
if run != 0:
error("Leaving environment for debugging purposes")
error("You can delete it with kcli delete cluster --yes %s" %
cluster)
os._exit(run)
todelete = [] if 'network_type' in data and data[
'network_type'] == 'Contrail' else ["%s-bootstrap" % cluster]
if platform in ['openstack', 'vsphere', 'packet']:
todelete.append("%s-bootstrap-helper" % cluster)
else:
pprint("Deploying bootstrap")
result = config.plan(plan,
inputfile='%s/cloud_bootstrap.yml' % plandir,
overrides=overrides)
if result['result'] != 'success':
os._exit(1)
pprint("Deploying masters")
result = config.plan(plan,
inputfile='%s/cloud_masters.yml' % plandir,
overrides=overrides)
if result['result'] != 'success':
os._exit(1)
call('openshift-install --dir=%s wait-for bootstrap-complete || exit 1'
% clusterdir,
shell=True)
todelete = [] if 'network_type' in data and data[
'network_type'] == 'Contrail' else ["%s-bootstrap" % cluster]
todelete.append("%s-bootstrap-helper" % cluster)
if platform in virtplatforms:
if workers > 0:
pprint("Deploying workers")
if 'name' in overrides:
del overrides['name']
if platform in virtplatforms:
result = config.plan(plan,
inputfile='%s/workers.yml' % plandir,
overrides=overrides)
elif platform in cloudplatforms:
result = config.plan(plan,
inputfile='%s/cloud_workers.yml' %
plandir,
overrides=overrides)
if result['result'] != 'success':
os._exit(1)
if platform == 'packet':
allnodes = [
"%s-worker-%s" % (cluster, num) for num in range(workers)
]
for node in allnodes:
k.add_nic(node, network)
if 'network_type' in data and data['network_type'] == 'Contrail':
pprint("Waiting 7mn on install to be stable")
sleep(420)
call(
"oc adm taint nodes -l node-role.kubernetes.io/master node-role.kubernetes.io/master:NoSchedule-",
shell=True)
pprint("Deploying certs autoapprover cronjob")
autoapprovercmd = 'oc create -f %s/autoapprovercron.yml' % clusterdir
call(autoapprovercmd, shell=True)
if not minimal:
installcommand = 'openshift-install --dir=%s wait-for install-complete' % clusterdir
installcommand += " || %s" % installcommand
pprint(
"Launching install-complete step. It will be retried one extra time in case of timeouts"
)
call(installcommand, shell=True)
else:
kubeconf = os.environ['KUBECONFIG']
kubepassword = open("%s/auth/auth/kubeadmin-password" %
clusterdir).read()
success("Minimal Cluster ready to be used")
success("INFO Install Complete")
info2(
"To access the cluster as the system:admin user when running 'oc', run export KUBECONFIG=%s"
% kubeconf)
info2(
"Access the Openshift web-console here: https://console-openshift-console.apps.%s.%s"
% (cluster, domain))
info2("Login to the console with user: kubeadmin, password: %s" %
kubepassword)
for vm in todelete:
pprint("Deleting %s" % vm)
k.delete(vm)
os.environ['KUBECONFIG'] = "%s/auth/kubeconfig" % clusterdir
if apps:
overrides['openshift_version'] = INSTALLER_VERSION[0:3]
for app in apps:
appdir = "%s/apps/%s" % (plandir, app)
if not os.path.exists(appdir):
warning("Skipping unsupported app %s" % app)
else:
pprint("Adding app %s" % app)
kube_create_app(config, appdir, overrides=overrides)
def scale(config, plandir, cluster, overrides):
plan = cluster if cluster is not None else 'testk'
client = config.client
platform = config.type
k = config.k
data = {}
pprint("Scaling on client %s" % client)
cluster = overrides.get('cluster', 'testk')
clusterdir = os.path.expanduser("~/.kcli/clusters/%s" % cluster)
if not os.path.exists(clusterdir):
error("Cluster directory %s not found..." % clusterdir)
sys.exit(1)
if os.path.exists("%s/kcli_parameters.yml" % clusterdir):
with open("%s/kcli_parameters.yml" % clusterdir, 'r') as install:
installparam = yaml.safe_load(install)
data.update(installparam)
plan = installparam.get('plan', plan)
data.update(overrides)
api_ip = data.get('api_ip')
if platform in virtplatforms:
if api_ip is None:
network = data.get('network')
if network == 'default' and platform == 'kvm':
warning("Using 192.168.122.253 as api_ip")
data['api_ip'] = "192.168.122.253"
elif platform == 'kubevirt':
selector = {'kcli/plan': plan, 'kcli/role': 'master'}
api_ip = config.k.create_service("%s-api" % cluster,
config.k.namespace,
selector,
_type="LoadBalancer",
ports=[6443, 22623, 22624])
if api_ip is None:
os._exit(1)
else:
pprint("Using api_ip %s" % api_ip)
data['api_ip'] = api_ip
else:
error("You need to define api_ip in your parameters file")
os._exit(1)
if data.get('virtual_router_id') is None:
data['virtual_router_id'] = hash(cluster) % 254 + 1
pprint("Using keepalived virtual_router_id %s" %
data['virtual_router_id'])
if platform == 'packet':
network = data.get('network')
if network is None:
error("You need to indicate a specific vlan network")
os._exit(1)
image = overrides.get('image')
if image is None:
cluster_image = k.info("%s-master-0" % cluster).get('image')
if cluster_image is None:
error("Missing image...")
sys.exit(1)
else:
pprint("Using image %s" % cluster_image)
image = cluster_image
data['image'] = image
for role in ['masters', 'workers']:
overrides = data.copy()
if overrides.get(role, 0) == 0:
continue
if platform in virtplatforms:
os.chdir(os.path.expanduser("~/.kcli"))
result = config.plan(plan,
inputfile='%s/%s.yml' % (plandir, role),
overrides=overrides)
elif platform in cloudplatforms:
result = config.plan(plan,
inputfile='%s/cloud_%s.yml' % (plandir, role),
overrides=overrides)
if result['result'] != 'success':
os._exit(1)
elif platform == 'packet' and 'newvms' in result and result['newvms']:
for node in result['newvms']:
k.add_nic(node, network)
