def generate(host):
"""Args is the layers we should probe."""
domain = lib.get_domain(host)
layers = lib.get_layers(domain)
if domain == 'EVENT':
mount = '%s-mgmt' % lib.get_current_event()
else:
mount = 'mgmt'
info = {}
info['layers'] = {}
# Only configure layers that we should probe
for layer in layers:
config = {
'port': 161,
}
secret = lib.read_secret('%s/%s' % (mount, 'snmpv3:%s' % layer))
if secret:
try:
config.update(parse_v3(secret))
except KeyError:
# F-up, invalid layer config
continue
else:
secret = lib.read_secret('%s/%s' % (mount, 'snmpv2:%s' % layer))
if not secret:
# F-up, no layer config
continue
config.update(parse_v2(secret))
info['layers'][layer] = config
return {'snmpexporter': info}
def generate_borg_passphrase(host):
login_data = lib.read_secret(login_path(host)) or {}
if login_data.get('borg_passphrase', None) is not None:
return
# Generate new passphrase
login_data['borg_passphrase'] = base64.b64encode(os.urandom(32))
lib.save_secret(login_path(host), **login_data)
def generate(host, *args):
current_event = lib.get_current_event()
info = {}
info['kubernetes::install'] = {}
if 'colo' in args:
info['colo_k8s'] = {}
else:
variant = args[1]
if 'worker' in args:
# find api server that maches the variant
apiserver = ""
for h, o in lib.get_nodes_with_package("kubernetes").items():
if "control" in o and variant in o:
apiserver = h
if apiserver == "":
raise Exception("k8s apiserver missing in ipplan")
secretpath = '{}-services/kube-{}:token'.format(
current_event, variant)
secrets = lib.read_secret(secretpath)
if not secrets:
raise Exception(
"Kubeadm token secret missing. Is master deployed?")
info['kubernetes::worker'] = {
'variant': variant,
'apiserver': apiserver,
'cert_hash': secrets['cert_hash'],
'token': secrets['token']
}
if 'control' in args:
servicenet = lib.match_networks_name("EVENT@" + variant.upper() +
".*K8S-SVC")
podnet = lib.match_networks_name("EVENT@" + variant.upper() +
".*K8S-POD")
if len(servicenet) == 0 or len(podnet) == 0:
raise Exception("service- and/or podnet not found in ipplan")
info['kubernetes::master'] = {
'variant': variant,
'podnet': podnet[0],
'servicenet': servicenet[0],
'current_event': current_event
}
return info
def sign_host_key(host):
db = pypuppetdb.connect()
hosts = {x.node: x for x in db.facts('ssh')}
if host not in hosts:
return ''
ssh_keys = hosts[host].value
# TODO(bluecmd): Only ecdsa for now
ssh_key = ssh_keys['ecdsa']['key']
# See if this key is already signed and saved
login_data = lib.read_secret(login_path(host)) or {}
if (login_data.get('sshecdsakey', None) == ssh_key
and 'sshecdsakey_signed' in login_data):
return login_data['sshecdsakey_signed']
res = lib.save_secret('ssh/sign/server',
cert_type='host',
public_key=ssh_key)
# Save key
login_data['sshecdsakey'] = ssh_key
login_data['sshecdsakey_signed'] = res['signed_key']
lib.save_secret(login_path(host), **login_data)
return res['signed_key']
def generate(host, *args):
info = {}
# get admins from ldap for usage in factorio as admins
info['admins'] = sorted(grp.getgrnam('factorio-admin-access').gr_mem)
# see if we have a group password for factorio, otherwise create it
if lib.get_domain(host) == 'EVENT':
event = lib.get_current_event()
secretpath = '{}-mgmt/factorio:{}'.format(event, host)
else:
secretpath = 'secrets/factorio:{}'.format(host)
secrets = lib.read_secret(secretpath)
if not secrets:
password = '******'
res = lib.save_secret(secretpath, password=password)
secrets = {'password': password}
info['password'] = secrets['password']
info['world_name'] = args[0]
return {'factorio': info}
def generate(host, *args):
"""Generate ldaplogin information.
Args:
*args: list(str): list of groups allowed to log in in addition to the
default users.
special keywords:
- git: restrict non-sudo users to git-shell
- otp: allow use of otp
- [0-9]+: listen on given port
"""
my_domain = lib.get_domain(host)
ldap_servers = lib.get_nodes_with_package('ldap', my_domain)
ldap_replicas = sorted(k for k, v in ldap_servers.iteritems()
if 'master' not in v)
# We want to be able to mutate the list
args = list(args)
if not ldap_replicas:
# If we don't have any LDAP servers, don't include this module
return {}
# We're only using the lowest two parts of the FQDN for identity.
# LDAP doesn't allow period in group names, replace with slash.
ident = '-'.join(host.split('.')[0:2])
info = {}
# 'otp' is a special keyword to enable otp authentication on the host
if 'otp' in args:
args.remove('otp')
info['use_otp'] = True
# find numbers in args, use it for ports
# 22 used by default, 2022 used for jumpgates
info['ssh_ports'] = set([22, 2022])
for arg in args:
if re.match('^[0-9]+$', arg):
info['ssh_ports'].add(int(arg))
args.remove(arg)
info['ssh_ports'] = list(info['ssh_ports'])
# For sudo users we have two groups:
# For event: services-event
# For colo: services-colo
# (.. and the ident-sudo group)
info['sudo'] = [ident + '-sudo-access']
services_group = 'services-colo-team'
if my_domain == 'EVENT':
services_group = 'services-team'
info['sudo'].append(services_group)
groups = []
for arg in args:
if arg.startswith('sudo'):
_, group = arg.split(':', 2)
info['sudo'].append(group)
else:
groups.append(arg)
# 'git' is a special keyword to enable restricting users to git-shell
if 'git' in args:
args.remove('git')
info['gitshell'] = ','.join(['!' + x for x in info['sudo']]) + ',*'
# Who should be allowed to log in to this system?
info['logon'] = [
(ident + '-access', 'ALL EXCEPT LOCAL'),
]
# Allow sudo users to logon everywhere
for group in info['sudo']:
info['logon'].append((group, 'ALL'))
# Add all explicit groups
for group in groups:
# Allow local logins as well to allow scripts to auth users
formatted = group.format(event=lib.get_current_event())
info['logon'].append((formatted, 'ALL'))
# LDAP settings
info['ldap'] = {'servers': [], 'servers_ip': {}}
for server in ldap_replicas:
info['ldap']['servers'].append(server)
(server_ipv4, server_ipv6), = lib.resolve_nodes_to_ip([server
]).values()
info['ldap']['servers_ip'][server] = (server_ipv4, server_ipv6)
info['ldap']['base'] = 'dc=tech,dc=dreamhack,dc=se'
info['ldap']['mount'] = '/ldap'
info['ca'] = lib.read_secret('ssh/config/ca')['public_key']
info['host_cert'] = sign_host_key(host)
info['panic_users'] = sorted(grp.getgrnam(services_group).gr_mem)
generate_borg_passphrase(host)
return {'ldaplogin': info}
def generate_backend(host, local_services):
scrape_configs = []
scrape_configs.extend(local_services)
domain = lib.get_domain(host)
basic_auth = lib.read_secret('services/monitoring:login')
# Find services that wants to be monitored
manifest = yaml.load(file(MANIFEST_PATH).read())
for package, spec in manifest['packages'].iteritems():
if spec is None or 'monitor' not in spec:
continue
urls = (spec['monitor']['url']
if isinstance(spec['monitor']['url'], dict) else {
None: spec['monitor']['url']
})
for url_id, url_str in urls.iteritems():
url = urlparse.urlparse(url_str)
targets = []
for target in sorted(
lib.get_nodes_with_package(package, domain).keys()):
targets.append(target if url.port is None else '%s:%d' %
(target, url.port))
scrape_config = {
'job_name': package + ('-%s' % url_id if url_id else ''),
'metrics_path': url.path,
'scheme': url.scheme,
'static_configs': [{
'targets': sorted(targets)
}],
}
if 'interval' in spec['monitor']:
scrape_config['scrape_interval'] = spec['monitor']['interval']
if 'labels' in spec['monitor']:
scrape_config['static_configs'][0]['labels'] = spec['monitor'][
'labels']
# Only allow authentication over https
if spec['monitor'].get('auth', False) and url.scheme == 'https':
scrape_config['basic_auth'] = basic_auth
scrape_configs.append(scrape_config)
# Layer specific monitoring
layers = lib.get_layers(domain)
snmp_nodes = {}
ssh_nodes = {}
for layer in layers:
hosts = lib.get_nodes_with_layer(layer, domain)
snmp_mute = lib.get_nodes_with_layer(layer, domain, 'no-snmp')
ssh_mute = lib.get_nodes_with_layer(layer, domain, 'no-ssh')
snmp_nodes[layer] = list(set(hosts) - set(snmp_mute))
ssh_nodes[layer] = [x + ':22' for x in set(hosts) - set(ssh_mute)]
# SNMP
for layer in layers:
# TODO(bluecmd): Use options for this
if layer == 'access':
host = 'snmp2.event.dreamhack.se'
else:
host = 'snmp1.event.dreamhack.se'
snmp = blackbox('snmp_%s' % layer,
host,
snmp_nodes[layer], {'layer': [layer]},
labels={'layer': layer})
snmp['scrape_interval'] = '30s'
snmp['scrape_timeout'] = '30s'
scrape_configs.append(snmp)
# SSH
for layer in layers:
for host in ['jumpgate1', 'jumpgate2', 'rancid']:
fqdn = host + '.event.dreamhack.se:9115'
ssh = blackbox('ssh_%s_%s' % (layer, host),
fqdn,
ssh_nodes[layer], {'module': ['ssh_banner']},
labels={'layer': layer})
ssh['scrape_interval'] = '30s'
ssh['scrape_timeout'] = '30s'
scrape_configs.append(ssh)
# Add external service-discovery
external = {
'job_name': 'external',
'file_sd_configs': [{
'files': ['/etc/prometheus/external/*.yaml'],
}],
}
scrape_configs.append(external)
if host.endswith('event.dreamhack.se'):
# Event should scrape puppet.tech.dreamhack.se to get information about
# puppet runs
puppet = {
'job_name': 'puppet_runs',
'metrics_path': '/metrics',
'scrape_interval': '60s',
'scrape_timeout': '55s',
'static_configs': [{
'targets': ['puppet.tech.dreamhack.se:9100'],
}],
}
scrape_configs.append(puppet)
vcenter = {
'job_name': 'vmware_vcenter',
'metrics_path': '/metrics',
'scrape_interval': '60s',
'scrape_timeout': '55s',
'static_configs': [{
'targets': ['provision.event.dreamhack.se:9272'],
}],
}
scrape_configs.append(vcenter)
# Make sure that all metrics have a host label.
# This rule uses the existing host label if there is one,
# stripping of the port (which shouldn't be part of the host label anyway)
# *or* if that label does not exist it uses the instance label
# (again stripping of the port)
relabel = {
'regex': r':?([^:]*):?.*',
'separator': ':',
'replacement': '${1}',
'source_labels': ['host', 'instance'],
'target_label': 'host',
}
mrc = 'metric_relabel_configs'
for scrape in scrape_configs:
if mrc in scrape:
scrape[mrc].append(relabel)
else:
scrape[mrc] = [relabel]
return {'scrape_configs': scrape_configs}
def generate(host, *args):
root_ca = lib.read_secret('ca-pki/cert/ca')
return {'system': {'ca': root_ca['certificate']}}