def validate(self, args, host):
if "gerrit" in args.glue["roles"] and "gerrit" not in host["roles"]:
fail("Cgit needs to be deployed on the same host as Gerrit.")
elif "install-server" in args.glue["roles"] and \
"install-server" not in host["roles"]:
fail("Cgit needs to be deployed on the same host as "
"the install-server.")
def configure(self, args, host):
if (bool(args.sfconfig["network"]["tls_cert_file"]) != bool(
args.sfconfig["network"]["tls_key_file"])
or bool(args.sfconfig["network"]["tls_cert_file"]) != bool(
args.sfconfig["network"]["tls_chain_file"])):
fail("tls_cert_file, tls_key_file and tls_chain_file "
"all need to be set")
if args.sfconfig["network"]["tls_cert_file"]:
# Check file exists
for k in ("tls_cert_file", "tls_chain_file", "tls_key_file"):
if not os.path.isfile(args.sfconfig["network"][k]):
fail("%s: doesn't not exists" %
args.sfconfig["network"][k])
# Check key is secured
if os.stat(
args.sfconfig["network"]["tls_key_file"]).st_mode & 0o7077:
fail("%s: insecure file mode, set to 0400" %
args.sfconfig["network"]["tls_key_file"])
if os.stat(
os.path.dirname(args.sfconfig["network"]
["tls_key_file"])).st_mode & 0o7077:
fail("%s: insecure dir mode, set to 0700" %
os.path.dirname(args.sfconfig["network"]["tls_key_file"]))
# Use user-provided certificate for the gateway
args.glue["gateway_crt"] = open(
args.sfconfig["network"]["tls_cert_file"]).read()
args.glue["gateway_chain"] = open(
args.sfconfig["network"]["tls_chain_file"]).read()
args.glue["gateway_key"] = open(
args.sfconfig["network"]["tls_key_file"]).read()
else:
self.get_or_generate_cert(args, "gateway", args.sfconfig["fqdn"])
args.glue["gateway_topmenu_logo_data"] = encode_image(
"/etc/software-factory/logo-topmenu.png")
args.glue["gateway_favicon_data"] = encode_image(
"/etc/software-factory/logo-favicon.ico")
args.glue["gateway_splash_image_data"] = encode_image(
"/etc/software-factory/logo-splash.png")
self.get_or_generate_ssh_key(args, "zuul_gatewayserver_rsa")
args.glue["pagesuser_authorized_keys"] = []
args.glue["pagesuser_authorized_keys"].append(
args.glue["zuul_gatewayserver_rsa_pub"])
if "koji_host" in args.sfconfig["network"] and \
args.sfconfig["network"]["koji_host"]:
args.glue["koji_host"] = args.sfconfig["network"]["koji_host"]
def validate(self, args, host):
if bool(args.sfconfig['config-locations']['config-repo']) != \
bool(args.sfconfig['config-locations']['jobs-repo']):
fail("Both config-repo and jobs-repo needs to be set")
if args.sfconfig["tenant-deployment"] and (bool(
args.sfconfig["tenant-deployment"]["name"]) != bool(
args.sfconfig["tenant-deployment"]["master-sf"])):
fail("Both tenant name and master-sf url need to be set")
if args.sfconfig["tenant-deployment"] and ("zuul" in args.glue["roles"]
or "nodepool"
in args.glue["roles"]):
fail("Zuul and Nodepool can't be running in a tenant deployment")
def configure(self, args, host):
args.glue["nodepool_providers"] = args.sfconfig.get(
"nodepool", {}).get("providers", [])
args.glue["nodepool_dib_reg_passwords"] = args.sfconfig.get(
"nodepool", {}).get("dib_reg_passwords", [])
args.glue["nodepool_clouds_file"] = args.sfconfig.get(
"nodepool", {}).get("clouds_file", None)
if args.glue["nodepool_clouds_file"]:
if not os.path.isfile(args.glue["nodepool_clouds_file"]):
fail("%s: does not exists" % args.glue["nodepool_clouds_file"])
if args.glue["nodepool_providers"] and args.glue[
"nodepool_clouds_file"]:
fail("Both clouds_file and providers can not be set "
"at the same time")
args.glue["nodepool_kube_file"] = args.sfconfig.get(
"nodepool", {}).get("kube_file", None)
if args.glue["nodepool_kube_file"]:
if not os.path.isfile(args.glue["nodepool_kube_file"]):
fail("%s: does not exists" % args.glue["nodepool_kube_file"])
self.get_or_generate_ssh_key(args, "nodepool_rsa")
self.get_or_generate_ssh_key(args, "zuul_rsa")
args.glue["nodepool_internal_url"] = "http://%s:%s" % (
args.glue["nodepool_launcher_host"],
args.defaults["nodepool_webapp_port"])
# nodepool_openshift_providers is only used to hold managed clusters
args.glue["nodepool_openshift_providers"] = []
for host in args.sfarch['inventory']:
if 'hypervisor-openshift' in host['roles']:
args.glue["nodepool_openshift_providers"].append({
"url":
"https://%s:8443" % host['hostname'],
"hostname":
host['hostname'],
"context":
"local-%s" % host['hostname'].replace('.', '-'),
"max_servers":
host.get('max-servers', 10),
"insecure_skip_tls_verify":
True,
})
def configure(self, args, host):
self.get_or_generate_CA(args)
self.get_or_generate_ssh_key(args, "service_rsa")
self.get_or_generate_ssh_key(args, "zuul_worker_rsa")
# When install-server is hosted on the gateway, we can use fqdn instead
args.glue["install_server_hostname"] = args.glue["install_server_host"]
if args.glue["install_server_host"] == args.glue["gateway_host"]:
args.glue["install_server_hostname"] = args.sfconfig["fqdn"]
args.glue["sf_version"] = get_sf_version()
args.glue["sf_previous_version"] = get_previous_version()
if args.upgrade:
print("Going to upgrade from %s to %s" %
(args.glue["sf_previous_version"], args.glue["sf_version"]))
if bool(args.sfconfig['config-locations']['config-repo']):
args.glue["remote_config_repositories"] = True
else:
args.glue["remote_config_repositories"] = False
args.glue["sync_strategy"] = 'push'
args.glue["sync_strategy"] = args.sfconfig['config-locations'][
'strategy'].get('sync', 'push')
if args.glue["sync_strategy"] not in ('push', 'patch', 'review'):
fail("Only push or patch or review sync strategy is supported.")
args.glue["resources_connections"] = {'__force_dict__': True}
if bool(args.sfconfig["tenant-deployment"]):
args.glue["tenant_name"] = args.sfconfig["tenant-deployment"][
"name"]
# Import master sf connections
self.read_master_sf_resources(args, host)
self.resolve_config_location(args, host)
self.resolve_zuul_jobs_location(args, host)
if bool(args.sfconfig["tenant-deployment"]):
# This is a tenant deployment, do extra configuration
args.glue["tenant_status_page_url"] = "%s/zuul/status.html" % (
args.glue["gateway_url"])
args.glue["tenant_zuul_api"] = "%s/zuul/api" % (
args.glue["gateway_url"])
args.glue["tenant_deployment"] = True
args.glue["config_key_exists"] = False
self.resolve_tenant_informations(args, host)
else:
if "zuul" not in args.glue["roles"]:
fail("Zuul service is required in non tenant-deployment mode")
# This is the master deployment, set default configuration
args.glue["tenant_name"] = args.sfconfig["default-tenant-name"]
args.glue[
"tenant_status_page_url"] = "%s/zuul/t/%s/status.html" % (
args.glue["gateway_url"], args.glue["tenant_name"])
args.glue["tenant_zuul_api"] = "%s/zuul/api/tenant/%s" % (
args.glue["gateway_url"], args.glue["tenant_name"])
args.glue["tenant_deployment"] = False
# Master tenant deployment always has config key ready to be used
args.glue["config_key_exists"] = True
# Convert sfconfig.yaml connections into group vars
zuul_config = args.sfconfig.get("zuul", {})
args.glue.setdefault("zuul_ssh_known_hosts", [])
args.glue.setdefault("zuul_gerrit_connections", [])
args.glue.setdefault("zuul_github_connections", [])
args.glue.setdefault("zuul_git_connections", [])
# Add local gerrit if available
if "gerrit" in args.glue["roles"]:
args.glue["gerrit_pub_url"] = "%s/r/" % args.glue["gateway_url"]
puburl = args.glue["gerrit_pub_url"]
if puburl[-1] == "/":
puburl = puburl[:-1]
if args.glue["tenant_deployment"]:
tenant_name = args.glue["tenant_name"]
else:
tenant_name = 'gerrit'
args.glue["zuul_gerrit_connections"].append({
'name':
tenant_name,
'port':
29418,
'hostname':
args.glue["gerrit_host"],
'canonical_hostname':
args.sfconfig["fqdn"],
'puburl':
puburl,
'username':
'******'
})
if not args.sfconfig["network"]["disable_external_resources"]:
for extra_gerrit in zuul_config.get("gerrit_connections", []):
if extra_gerrit.get("port", 29418) == 22:
host_packed = extra_gerrit["hostname"]
else:
host_packed = "[%s]:%s" % (extra_gerrit["hostname"],
extra_gerrit.get("port", 29418))
args.glue["zuul_ssh_known_hosts"].append({
"host_packed":
host_packed,
"host":
extra_gerrit["hostname"],
"port":
extra_gerrit.get("port", 29418)
})
args.glue["zuul_gerrit_connections"].append(extra_gerrit)
for github_connection in zuul_config.get("github_connections", []):
if github_connection.get("port", 22) == 22:
host_packed = github_connection.get(
"hostname", "github.com")
else:
host_packed = "[%s]:%s" % (github_connection["hostname"],
github_connection["port"])
args.glue["zuul_ssh_known_hosts"].append({
"host_packed":
host_packed,
"host":
github_connection.get("hostname", "github.com"),
"port":
github_connection.get("port", 22)
})
args.glue["zuul_github_connections"].append(github_connection)
gh_app_key = github_connection.get('app_key')
if gh_app_key and os.path.isfile(gh_app_key):
github_connection['app_key'] = open(gh_app_key).read()
for git_connection in zuul_config.get("git_connections", []):
args.glue["zuul_git_connections"].append(git_connection)
# Set auto-configurations of pipelines list
args.glue.setdefault("zuul_gerrit_connections_pipelines", [])
args.glue.setdefault("zuul_github_connections_pipelines", [])
args.glue["zuul_gate_pipeline"] = False
for gerrit_connection in args.glue["zuul_gerrit_connections"]:
if not gerrit_connection.get("default_pipelines", True) or \
gerrit_connection.get("report_only", False):
continue
args.glue["zuul_gerrit_connections_pipelines"].append(
gerrit_connection)
args.glue["zuul_gate_pipeline"] = True
for github_connection in args.glue["zuul_github_connections"]:
if not github_connection.get("default_pipelines", True):
continue
args.glue["zuul_github_connections_pipelines"].append(
github_connection)
if github_connection.get("app_name"):
args.glue["zuul_gate_pipeline"] = True
def resolve_tenant_informations(self, args, host):
# TODO: make sfconfig.yaml zuul setting usable without sf-zuul role
args.glue["zuul_default_retry_attempts"] = args.sfconfig["zuul"].get(
"default_retry_attempts", 3)
args.glue["zuul_periodic_pipeline_mail_rcpt"] = args.sfconfig["zuul"][
"periodic_pipeline_mail_rcpt"]
# Fetch zuul public key from master sf - Needed for:
# * For registering the master's zuul pub on the tenant's Gerrit
# * For exposing the master's zuul pub key on the tenant /var/www/keys
zuul_key_path = "%s/ssh_keys/master_zuul_rsa.pub" % args.lib
try:
zuul_key = open(zuul_key_path).read()
except Exception:
zuul_key = None
if not zuul_key or "ssh-rsa" not in zuul_key:
key_url = "%s/keys/zuul_rsa.pub" % args.glue["master_sf_url"]
try:
req = request.urlopen(key_url)
zuul_key = req.read().decode("utf-8")
open(zuul_key_path, "w").write(zuul_key)
except Exception:
fail("Couldn't get zuul public key at %s" % key_url)
args.glue["zuul_rsa_pub"] = zuul_key
# Look for master zuul-jobs project name and connection name
try:
# TODO: add special attribute to zuul-jobs to better identify it
# or add support for tenant custom zuul-jobs location...
zj = self.master_resource.get("resources", {}).get(
"projects", {}).get("internal",
{}).get("source-repositories")[-1]
zuul_name = list(zj.keys())[0]
zuul_conn = zj[zuul_name]["connection"]
args.glue["zuul_jobs_connection_name"] = zuul_conn
args.glue["zuul_jobs_project_name"] = zuul_name
args.glue["zuul_upstream_zuul_jobs"] = True
except Exception:
fail("Couldn't find zuul-jobs location in master sf")
# Look for tenant default connection name and type to define
# zuul connections on the master sf
# Check if tenant config key already exists in master sf
# If not then it means the tenant have not been defined yet on the
# master sf. If yes, fetch it. The key will be used to initialize
# Zuul's secrets (like logserver ssh key) into the tenant's sf
# config repository.
key_path = "/var/lib/software-factory/bootstrap-data/certs/config.pub"
if (os.path.exists(key_path)
and "PUBLIC KEY" in open(key_path).read()):
args.glue["config_key_exists"] = True
else:
try:
req = request.urlopen(
"%s/zuul/api/tenant/%s/key/%s.pub" %
(args.glue["master_sf_url"], args.glue["tenant_name"],
args.glue["config_project_name"]))
key_data = req.read().decode("utf-8")
if "PUBLIC KEY" in key_data:
open(key_path, "w").write(key_data)
args.glue["config_key_exists"] = True
else:
raise RuntimeError("Invalid public key --[%s]--" %
key_data)
except Exception:
print("Tenant isn't registered in master deployment, "
"add tenant config and restart sfconfig")
# Fetch main install-server tenant-update secret to trigger zuul reload
secret_path = "/var/lib/software-factory/bootstrap-data/certs/" \
"tenant-update-secret.yaml"
if args.glue["config_key_exists"]:
if (os.path.exists(secret_path)
and "pkcs" in open(secret_path).read()):
args.glue["tenant_update_secret"] = open(secret_path).read()
else:
try:
req = request.urlopen(
"%s/.config/tenant-update_%s_secret.yaml" %
(args.glue["master_sf_url"], args.glue["tenant_name"]))
secret_data = req.read().decode("utf-8")
if "pkcs" in secret_data:
open(secret_path, "w").write(secret_data)
args.glue["tenant_update_secret"] = secret_data
else:
raise RuntimeError("Invalid secret --[%s]--" %
secret_data)
except Exception:
fail("Ooops, tenant-update secret hasn't been generated")
def read_master_sf_resources(self, args, host):
"""The goal of this method is fetch resources from master deployment
and configure a zuul-less deployment"""
args.glue["master_sf_fqdn"] = parse.urlparse(
args.sfconfig["tenant-deployment"]["master-sf"]).hostname
args.glue["master_sf_url"] = (parse.urlparse(
args.sfconfig["tenant-deployment"]["master-sf"]).scheme + '://' +
args.glue["master_sf_fqdn"])
# Check master SF connectivity
try:
req = request.urlopen("%s/manage/v2/resources" %
args.glue["master_sf_url"])
self.master_resource = json.loads(req.read().decode('utf-8'))
args.glue["resources_connections"] = self.master_resource.get(
'resources', {}).get('connections', {})
args.glue["resources_connections"]['__force_dict__'] = True
except Exception:
fail("Couldn't contact master-sf: %s" %
(args.glue["master_sf_url"]))
# Import master sf connection
tenant_connections = set()
tenant_conf = self.master_resource.get("resources", {}).get(
"tenants", {}).get(args.glue["tenant_name"], {})
tenant_def_conn = tenant_conf.get("default-connection")
if tenant_def_conn:
tenant_connections.add(tenant_def_conn)
if tenant_conf.get("allowed-reporters"):
tenant_connections.update(tenant_conf["allowed-reporters"])
if tenant_conf.get("allowed-triggers"):
tenant_connections.update(tenant_conf["allowed-triggers"])
args.glue["zuul_gate"] = False
for name, values in self.master_resource.get("resources", {}).get(
"connections", {}).items():
if name not in tenant_connections:
continue
if values.get("type") == "gerrit":
hostname = parse.urlparse(values.get("base-url")).hostname
args.sfconfig["zuul"]["gerrit_connections"].append({
"name":
name,
"hostname":
hostname,
"username":
"******"
})
args.glue["zuul_gate"] = True
elif values.get("type") == "github":
args.sfconfig["zuul"]["github_connections"].append({
"name":
name,
"app_name":
values.get("github-app-name", ""),
"label_name":
values.get("github-label", "")
})
if values.get("github-app-name"):
args.glue["zuul_gate"] = True
def resolve_config_location(self, args, host):
"""The goal of this method is to discover the config and sf-jobs
location and their associated connection
"""
if not args.glue["remote_config_repositories"]:
# When config repositories are local, default names are config
# and sf-jobs
conf_name = "config"
jobs_name = "sf-jobs"
if "gerrit" in args.glue["roles"]:
# If gerrit is enabled, use it's connection and default push
# location
conn_name = "gerrit"
url = "%s/r/config" % args.glue["gateway_url"]
conf_loc = "git+ssh://gerrit/config"
jobs_loc = "git+ssh://gerrit/sf-jobs"
else:
if "cgit" in args.glue["roles"]:
conn_name = "local-cgit"
url = "%s/cgit/config" % args.glue["gateway_url"]
else:
conn_name = "local-git"
url = "/var/lib/software-factory/git/config.git"
# Inject zuul git connection
self.ensure_git_connection(conn_name, args, host)
# If gerrit is not enabled, push location is a local path
conf_loc = "/var/lib/software-factory/git/config.git"
jobs_loc = "/var/lib/software-factory/git/sf-jobs.git"
else:
# When config repositories are remote, we need to look for the
# zuul connection name.
sync_user = args.sfconfig['config-locations']['strategy']['user']
zuul_config = args.sfconfig['zuul']
for repo in ('config-repo', 'jobs-repo'):
location = args.sfconfig['config-locations'].get(repo)
if repo == 'config-repo':
# We only need url location of the config repo,
# it's needed for config-update job
url = location
found = False
# First we look for a matching github connections
for conn in zuul_config['github_connections']:
host = conn.get('hostname', 'github.com')
if host not in location:
continue
# Project name is the last two components
project_name = "/".join(location.split('/')[-2:])
if args.glue["sync_strategy"] != 'push':
_loc = location
else:
_loc = "ssh://%s@%s/%s" % (sync_user, host,
project_name)
if repo == "config-repo":
conn_name = conn["name"]
conf_name = project_name
conf_loc = _loc
elif conn_name != conn["name"]:
fail("Config and jobs needs to share "
"the same connection")
else:
jobs_name = project_name
jobs_loc = _loc
found = True
break
if not found:
# If location is matching github connections, look for
# available gerrit connections.
for conn in zuul_config['gerrit_connections']:
puburl = conn.get('puburl')
if puburl not in location:
continue
# Project name is the remaining part after the puburl
project_name = location[len(puburl):].lstrip('/')
if args.glue["sync_strategy"] != 'push':
_loc = location
else:
_loc = "ssh://%s@%s/%s" % (
sync_user, conn["hostname"], project_name)
if repo == "config-repo":
conn_name = conn["name"]
conf_name = project_name
conf_loc = _loc
elif conn_name != conn["name"]:
fail("Config and jobs needs to share "
"the same connection")
else:
jobs_name = project_name
jobs_loc = _loc
found = True
break
if not found:
# We couldn't find a zuul connection for remote locations
fail("%s: No zuul connection configured" % location)
# Theses variable will be used in resources and zuul templates
args.glue["config_connection_name"] = conn_name
args.glue["config_public_location"] = url
args.glue["config_project_name"] = conf_name
args.glue["config_location"] = conf_loc
args.glue["sf_jobs_project_name"] = jobs_name
args.glue["sf_jobs_location"] = jobs_loc
def process(args):
# scalable_roles are the roles that can be instantiate multiple time
# this indicate that we don't need $role.$fqdn aliases
# TODO: remove this logic when $role.$fqdn are no longer used
args.glue["scalable_roles"] = [
"zuul",
"zuul-merger",
"zuul-executor",
"hypervisor-runc",
]
# roles is a dictwith roles name as key and host list as value
args.glue["roles"] = {}
# hosts_files is a dict with host ip as key and hostname list as value
args.glue["hosts_file"] = {}
for host in args.sfarch["inventory"]:
if "install-server" in host["roles"]:
host["ip"] = pread(["ip", "route", "get", "8.8.8.8"]).split()[6]
elif "ip" not in host:
fail("%s: host '%s' needs an ip" % (args.arch, host))
if "public_url" not in host:
if "gateway" in host["roles"]:
host["public_url"] = "https://%s" % args.sfconfig["fqdn"]
else:
host["public_url"] = "http://%s" % host["ip"]
else:
host["public_url"] = host["public_url"].rstrip("/")
# TODO: remove this aliases logic when $role.$fqdn are no longer used
aliases = set()
if 'name' in host:
aliases.add(host['name'])
for role in host["roles"]:
# Add host to role list
args.glue["roles"].setdefault(role, []).append(host)
# Add extra aliases for specific roles
if role == "gateway":
aliases.add(args.sfconfig["fqdn"])
elif role == "cauth":
aliases.add("auth.%s" % args.sfconfig["fqdn"])
elif role not in args.glue["scalable_roles"]:
# Add role name virtual name (as cname)
aliases.add("%s.%s" % (role, args.sfconfig["fqdn"]))
aliases.add(role)
args.glue["hosts_file"][host["ip"]] = [host["hostname"]] + \
list(aliases)
# Check roles
for requirement in required_roles:
if requirement not in args.glue["roles"]:
fail("%s role is missing" % requirement)
if len(args.glue["roles"][requirement]) > 1:
fail("Only one instance of %s is required" % requirement)
if len(args.sfarch["inventory"]) > 1 and ("cgit" not in args.glue["roles"]
and "gerrit"
not in args.glue["roles"]):
fail("Cgit or Gerrit component is required for distributed deployment")
# Add install-server hostname for easy access
args.glue["install_server"] = args.glue["roles"]["install-server"][0][
"hostname"]