def get_raw_configuration():
# env
for env_var in ZENTRAL_CONF_ENV_VARS:
raw_cfg = os.environ.get(env_var)
if raw_cfg:
if env_var.startswith("B64GZIP"):
try:
raw_cfg = gzip.decompress(base64.b64decode(raw_cfg))
except Exception:
raise ImproperlyConfigured(
"Could not read base64gzipped zentral conf")
logger.info("Got raw configuration from environment")
return raw_cfg, "ENV"
# file
conf_dir = get_conf_dir()
if conf_dir:
for filename in ("base.json", "base.yaml", "base.yml"):
filepath = os.path.join(conf_dir, filename)
if os.path.exists(filepath):
try:
with open(filepath, "r") as f:
raw_cfg = f.read()
except Exception:
logger.exception(
"Could not read configuration file {}".format(
filepath))
else:
logger.info(
"Got raw configuration from file {}".format(filepath))
return raw_cfg, filepath
raise ImproperlyConfigured("Could not get raw configuration")
def register_compliance_check_class(compliance_check_class):
try:
model = compliance_check_class.get_model()
except AttributeError:
raise ImproperlyConfigured('Not a valid compliance check class')
if model in compliance_check_classes:
raise ImproperlyConfigured(f'Compliance check class "{model}" already registered')
compliance_check_classes[model] = compliance_check_class
logger.debug('Compliance check class "%s" registered', model)
def __init__(self, config_d):
super(EventStore, self).__init__(config_d)
# priority
priority = config_d.get("priority")
if not priority:
priority = self.DEFAULT_PRIORITY
try:
priority = SysLogHandler.priority_names[priority.lower()]
except (TypeError, KeyError):
raise ImproperlyConfigured(
"Unknown syslog priority {}".format(priority))
# facility
facility = config_d.get("facility")
if not facility:
facility = self.DEFAULT_FACILITY
try:
facility = SysLogHandler.facility_names[facility.lower()]
except (TypeError, KeyError):
raise ImproperlyConfigured(
"Unknown syslog facility {}".format(facility))
self.priority = ("<%d>" % ((facility << 3) | priority)).encode("utf-8")
# prepend ecc cookie string
self.prepend_ecc = bool(config_d.get("prepend_ecc", False))
# protocol
protocol = config_d.get("protocol")
if not protocol:
protocol = self.DEFAULT_PROTOCOL
protocol = protocol.lower()
if protocol == "udp":
self.socket_protocol = socket.SOCK_DGRAM
elif protocol == "tcp":
self.socket_protocol = socket.SOCK_STREAM
else:
raise ImproperlyConfigured(
"Unknown syslog protocol {}".format(protocol))
host = config_d.get("host", self.DEFAULT_HOST)
unix_socket = config_d.get("unix_socket")
if host and unix_socket:
raise ImproperlyConfigured(
"Can't have both host and unix_socket for a syslog store")
elif host:
# port
try:
port = int(config_d.get("port", self.DEFAULT_PORT))
except TypeError:
raise ImproperlyConfigured(
"Unknown syslog port {}".format(port))
self.socket_family = socket.AF_INET
self.address = (host, port)
else:
self.socket_family = socket.AF_UNIX
self.address = unix_socket
def __init__(self, config_d, test=False):
super(EventStore, self).__init__(config_d)
# es kwargs
kwargs = {}
# es kwargs > hosts
hosts = []
configured_hosts = config_d.get("hosts", config_d.get("servers"))
for host in configured_hosts:
if not isinstance(host, dict):
o = urllib.parse.urlparse(host)
host = {k: v for k, v in (('host', o.hostname),
('port', o.port),
('url_prefix', o.path)) if v}
if o.scheme == "https" or o.port == 443:
if o.port is None:
host['port'] = 443
host['use_ssl'] = True
hosts.append(host)
kwargs['hosts'] = hosts
# es kwargs > verify_certs
if any(host.get("use_ssl") for host in hosts):
kwargs['verify_certs'] = True
# es kwargs > http_auth (for AWS)
aws_auth = config_d.get("aws_auth")
if aws_auth:
kwargs["connection_class"] = RequestsHttpConnection
try:
from requests_aws4auth import AWS4Auth
kwargs['http_auth'] = AWS4Auth(aws_auth['access_id'],
aws_auth['secret_key'],
aws_auth['region'],
'es')
except ImportError:
raise ImproperlyConfigured("Missing requests_aws4auth pip dependency "
"for ES AWS credentials")
except KeyError:
raise ImproperlyConfigured("access_id, secret_key or region missing "
"in aws_auth config")
self._es = Elasticsearch(**kwargs)
self.use_mapping_types = None
self.index = config_d['index']
self.read_index = config_d.get('read_index', self.index)
self.kibana_base_url = config_d.get('kibana_base_url', None)
self.kibana_index_pattern_uuid = config_d.get('kibana_index_pattern_uuid')
self.index_settings = {
"index.mapping.total_fields.limit": config_d.get("index.mapping.total_fields.limit", 2000),
"number_of_shards": config_d.get("number_of_shards", 1),
"number_of_replicas": config_d.get("number_of_replicas", 0)
}
self.test = test
self.version = None
def find_conf_file(conf_dir, basename, required=True):
filepaths = [os.path.join(conf_dir, "{}{}".format(basename, ext)) for ext in [".json", ".yml"]]
found_files = [f for f in filepaths if os.path.exists(f)]
if not found_files:
if required:
raise ImproperlyConfigured('{} is required'.format(' or '.join(filepaths)))
else:
return None
elif len(found_files) == 2:
raise ImproperlyConfigured('{} both present: conflict'.format(' and '.join(filepaths)))
else:
return found_files[0]
def register_incident_class(incident_cls):
incident_type = incident_cls.incident_type
if incident_type in incident_types:
raise ImproperlyConfigured(
f'Incident type {incident_type} already registered')
logger.debug('Incident type "%s" registered', incident_type)
incident_types[incident_type] = incident_cls
def check_request_secret(self, request, *args, **kwargs):
req_sec = self.get_request_secret(request, *args, **kwargs)
if self.verify_module is None:
raise ImproperlyConfigured("self.verify_module is null")
data = verify_secret(req_sec, self.verify_module)
self.machine_serial_number = data.get('machine_serial_number', None)
self.business_unit = data.get('business_unit', None)
def register_probe_class(probe_cls):
probe_model = probe_cls.get_model()
if probe_model in probe_classes:
raise ImproperlyConfigured(
"Probe class '{}' already registered".format(probe_cls))
logger.debug('Probe class "%s" registered', probe_cls)
probe_classes[probe_model] = probe_cls
def get_conf_dir():
conf_dir = os.environ.get(ZENTRAL_CONF_DIR_ENV_VAR)
if not conf_dir:
conf_dir = os.path.realpath(
os.path.join(os.path.dirname(__file__), "../../conf"))
if os.path.exists(conf_dir):
return conf_dir
else:
raise ImproperlyConfigured('Conf dir could not be found.')
def get_configuration():
raw_cfg, src = get_raw_configuration()
for loader, kwargs in ((json.loads, {}), (yaml.load, {
"Loader": yaml.SafeLoader
})):
try:
return loader(raw_cfg, **kwargs)
except (ValueError, yaml.YAMLError):
pass
raise ImproperlyConfigured(
"Could not parse raw configuration from {}".format(src))
def __init__(self, config_g):
super().__init__(config_g)
error_msgs = []
self.es_hosts = config_g["es_hosts"]
if not self.es_hosts:
error_msgs.append("Missing es_hosts")
if not isinstance(self.es_hosts, list):
error_msgs.append("es_hosts must be a list")
if error_msgs:
raise ImproperlyConfigured("{} in {}".format(
", ".join(error_msgs), self.name))
def load_config_file(filepath):
root, ext = os.path.splitext(filepath)
if not ext:
raise ImproperlyConfigured("File {} without extension".format(filepath))
if ext == '.json':
fileopener = json.load
filetype = "JSON"
fileopener_kwargs = {}
elif ext == '.yml':
fileopener = yaml.load
filetype = "YAML"
fileopener_kwargs = {"Loader": yaml.SafeLoader}
else:
raise ImproperlyConfigured("Unknown extension {} of file {}".format(ext, filepath))
try:
with open(filepath) as f:
return fileopener(f, **fileopener_kwargs)
except (ValueError, yaml.YAMLError):
raise ImproperlyConfigured("{} error in file {}".format(filetype, filepath)) from None
except Exception as e:
raise ImproperlyConfigured(str(e)) from None
def get_api_secret(settings):
err_msg = None
try:
secret = settings['api']['secret']
except KeyError:
err_msg = "Missing api.secret key in conf"
else:
if not isinstance(secret, str):
err_msg = "api.secret must be a str"
elif len(secret) < API_SECRET_MIN_LENGTH:
warnings.warn("Your api.secret has less than {} characters.".format(API_SECRET_MIN_LENGTH))
if err_msg:
raise ImproperlyConfigured(err_msg)
return secret
def register_event_type(event_cls):
"""
Register event class for an event type.
event_type must be unique in the zentral configuration.
"""
event_type = event_cls.event_type
if event_type in event_types:
raise ImproperlyConfigured(
'Event type {} already registered'.format(event_type))
logger.debug('Event type "%s" registered', event_type)
event_types[event_type] = event_cls
for tag in event_cls.tags:
event_tags.setdefault(tag, []).append(event_cls)
def __init__(self, config_d, test=False):
super(EventStore, self).__init__(config_d)
# es kwargs
kwargs = {}
# es kwargs > hosts
hosts = []
configured_hosts = config_d.get("hosts", config_d.get("servers"))
for host in configured_hosts:
if not isinstance(host, dict):
o = urllib.parse.urlparse(host)
host = {
k: v
for k, v in (('host', o.hostname), ('port', o.port),
('url_prefix', o.path)) if v
}
if o.scheme == "https" or o.port == 443:
host['use_ssl'] = True
hosts.append(host)
kwargs['hosts'] = hosts
# es kwargs > verify_certs
if any(host.get("use_ssl") for host in hosts):
kwargs['verify_certs'] = True
# es kwargs > http_auth (for AWS)
aws_auth = config_d.get("aws_auth")
if aws_auth:
kwargs["connection_class"] = RequestsHttpConnection
try:
kwargs['http_auth'] = AWS4Auth(aws_auth['access_id'],
aws_auth['secret_key'],
aws_auth['region'], 'es')
except KeyError:
raise ImproperlyConfigured(
"access_id, secret_key or region missing "
"in aws_auth config")
self._es = Elasticsearch(**kwargs)
self.use_mapping_types = None
self.index = config_d['index']
self.read_index = config_d.get('read_index', self.index)
self.kibana_base_url = config_d.get('kibana_base_url', None)
self.INDEX_CONF["settings"]["number_of_shards"] = config_d.get(
"number_of_shards", 1)
self.test = test