def push_to_vault(self, exported_path, exported_kv, target_path):
"""
Push exported kv to Vault
:param exported_path: export root path
:type exported_path: str
:param target_path: push kv to this path
:type target_path: str
:param exported_kv: Exported KV store
:type exported_kv: dict
"""
self.logger.debug("Pushing exported kv to Vault")
vault_client = VaultClient(self.base_logger,
dry=self.parsed_args.dry_run,
vault_addr=os.environ["VAULT_TARGET_ADDR"],
skip_tls=self.parsed_args.skip_tls)
vault_client.authenticate(os.environ["VAULT_TARGET_TOKEN"])
for secret in exported_kv:
secret_target_path = self.__list_to_string(
target_path.split('/') +
secret.split('/')[len(exported_path.split('/')):],
separator="/")
self.logger.debug("Exporting secret: " + secret + " to " +
secret_target_path)
vault_client.write(secret_target_path,
exported_kv[secret],
hide_all=True)
def delete_from_vault(self, kv_to_delete):
"""
Delete all secrets at and under specified path
:param kv_to_delete: list of all secrets paths to delete
:type kv_to_delete: list
"""
self.logger.debug("Deleting secrets from " + os.environ["VAULT_ADDR"])
vault_client = VaultClient(self.base_logger,
dry=self.parsed_args.dry_run,
skip_tls=self.parsed_args.skip_tls)
vault_client.authenticate()
for secret in kv_to_delete:
self.logger.debug("Deleting " + secret)
vault_client.delete(secret)
def connect_to_vault(self, vault_addr, vault_token):
"""
Connect to a Vault instance
:param vault_addr: Vault URL
:type vault_addr: str
:param vault_token: Vault token
:type vault_token: str
:return: VaultClient
"""
self.logger.debug("Connecting to Vault instance '%s'" % vault_addr)
vault_client = VaultClient(self.base_logger,
dry=self.kwargs.dry_run,
vault_addr=vault_addr,
skip_tls=self.kwargs.skip_tls)
vault_client.authenticate(vault_token)
return vault_client
def read_from_vault(self, path_to_read):
"""
Read secret tree from Vault
:param path_to_read: secret path to read and return
:type path_to_read: str
:return dict(dict)
"""
self.logger.debug("Reading kv tree")
vault_client = VaultClient(self.base_logger,
dry=self.parsed_args.dry_run,
skip_tls=self.parsed_args.skip_tls)
vault_client.authenticate()
kv_full = {}
kv_list = vault_client.get_secrets_tree(path_to_read)
self.logger.debug("Secrets found: " + str(kv_list))
for kv in kv_list:
kv_full[kv] = vault_client.read_secret(kv)
return kv_full
class VaultManagerPolicies:
logger = None
subparser = None
kwargs = None
module_name = None
vault_client = None
base_logger = None
policies_folder = None
def __init__(self, base_logger=None):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
if base_logger:
self.logger = logging.getLogger(
base_logger + "." + self.__class__.__name__)
else:
self.logger = logging.getLogger()
self.logger.debug("Initializing VaultManagerPolicies")
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = \
subparsers.add_parser(self.module_name,
help=self.module_name + ' management')
self.subparser.add_argument(
"--pull", action='store_true',
help="Pull distant policies from Vault"
)
self.subparser.add_argument(
"--push", action='store_true', help="Push local policies to Vault"
)
self.subparser.set_defaults(module_name=self.module_name)
def check_args_integrity(self):
"""
Checking provided arguments integrity
"""
self.logger.debug("Checking arguments integrity")
if all([self.kwargs.pull, self.kwargs.push]):
self.logger.critical("push and pull args cannot "
"be specified at the same time")
return False
elif not any([self.kwargs.pull, self.kwargs.push]):
self.logger.critical("You must specify pull or push")
return False
return True
def policies_pull(self):
"""
Pull policies from vault
"""
self.logger.info("Pulling Policies from Vault")
self.logger.debug("Pulling policies")
distant_policies = self.vault_client.policy_list()
self.logger.info("Distant policies found:" + str(distant_policies))
for policy in distant_policies:
# policy name will always be 'type_name_policy'
splitted = policy.split("_")
if len(splitted) != 3 or splitted[2] != "policy":
self.logger.warning("Policy " + policy +
" does not match policy name pattern "
"and will not be pulled")
continue
# create the parent folder policy if doest not exists (user, etc...)
policy_folder = os.path.join(self.policies_folder, splitted[0])
if not os.path.isdir(policy_folder):
self.logger.debug("Folder " + policy_folder +
" doest not exists, creating...")
os.makedirs(policy_folder)
# create the policy file
policy_path = os.path.join(policy_folder, splitted[1] + ".hcl")
with open(policy_path, 'w+') as fd:
fd.write(self.vault_client.policy_get(policy))
self.logger.info("Policy " + policy_path + " saved")
self.logger.info("Policies fetched in policies folder")
def policies_push(self):
"""
Push all policies from policies folder to Vault
"""
self.logger.info("Pushing Policies to Vault")
self.logger.debug("Push all policies")
distant_policies = self.vault_client.policy_list()
local_policies = []
# Building local policies list
for policy_file in glob.iglob(os.path.join(self.policies_folder,
"*/**/*.hcl"), recursive=True):
prefix = os.path.relpath(policy_file, self.policies_folder)
policy_name = prefix.replace("/", "_")
self.logger.debug("Local policy %s - name: %s found"
% (policy_file, policy_name))
with open(policy_file, 'r') as fd:
local_policies.append(
{"name": policy_name.replace(".hcl", "_policy"),
"content": fd.read()})
# Removing distant policies which doesn't exists locally
for distant_policy in distant_policies:
if distant_policy not in [pol["name"] for pol in local_policies]:
self.logger.info("Removing distant policy " + distant_policy)
self.vault_client.policy_delete(distant_policy)
# Push local policies
for policy in local_policies:
self.vault_client.policy_set(policy_name=policy["name"],
policy_content=policy["content"])
if policy["name"] in distant_policies:
self.logger.info("Policy %s has been updated" % policy["name"])
else:
self.logger.info("Policy %s has been created" % policy["name"])
self.logger.info("Policies pushed to Vault")
def run(self, kwargs):
"""
Module entry point
:param kwargs: Arguments parsed
:type kwargs: dict
"""
# Convert kwargs to an Object with kwargs dict as class vars
self.kwargs = namedtuple("KwArgs", kwargs.keys())(*kwargs.values())
self.logger.debug("Module " + self.module_name + " started")
if not self.check_args_integrity():
self.subparser.print_help()
return False
missing_args = utils.keys_exists_in_dict(
self.logger, dict(self.kwargs._asdict()),
[{"key": "vault_addr", "exc": [None, '']},
{"key": "vault_token", "exc": [None, False]},
{"key": "vault_config", "exc": [None, False, '']}]
)
if len(missing_args):
raise ValueError(
"Following arguments are missing %s\n" % [
k['key'].replace("_", "-") for k in missing_args]
)
self.logger.debug("Vault config folder: %s" % self.kwargs.vault_config)
self.policies_folder = os.path.join(
self.kwargs.vault_config, "policies"
)
if not os.path.isdir(self.policies_folder):
os.mkdir(self.policies_folder)
self.vault_client = VaultClient(
self.base_logger,
vault_addr=self.kwargs.vault_addr,
dry=self.kwargs.dry_run,
skip_tls=self.kwargs.skip_tls
)
self.vault_client.authenticate()
if self.kwargs.pull:
self.policies_pull()
if self.kwargs.push:
self.policies_push()
class VaultManagerAuth:
"""
Authentication module
"""
logger = None
base_logger = None
subparser = None
parsed_args = None
arg_parser = None
module_name = None
conf = None
vault_client = None
local_auth_methods = None
distant_auth_methods = None
def __init__(self, base_logger, subparsers):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
self.logger = logging.getLogger(base_logger + "." + self.__class__.__name__)
self.logger.debug("Initializing VaultManagerAuth")
self.initialize_subparser(subparsers)
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = subparsers.add_parser(
self.module_name,
help=self.module_name + ' management'
)
self.subparser.add_argument("--push", action='store_true',
help="Push auth methods to Vault")
self.subparser.set_defaults(module_name=self.module_name)
def check_env_vars(self):
"""
Check if all needed env vars are set
:return: bool
"""
self.logger.debug("Checking env variables")
needed_env_vars = ["VAULT_ADDR", "VAULT_TOKEN", "VAULT_CONFIG"]
if not all(env_var in os.environ for env_var in needed_env_vars):
self.logger.critical("The following env vars must be set")
self.logger.critical(str(needed_env_vars))
return False
self.logger.debug("All env vars are set")
if not os.path.isdir(os.environ["VAULT_CONFIG"]):
self.logger.critical(
os.environ["VAULT_CONFIG"] + " is not a valid folder")
return False
self.logger.info("Vault address: " + os.environ["VAULT_ADDR"])
self.logger.info("Vault config folder: " + os.environ["VAULT_CONFIG"])
return True
def read_configuration(self):
"""
Read configuration file
"""
self.logger.debug("Reading configuration")
with open(os.path.join(os.environ["VAULT_CONFIG"], "auth-methods.yml"),
'r') as fd:
try:
self.conf = yaml.load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load conf file: " + str(e))
return False
self.logger.debug("Read conf: " + str(self.conf))
return True
def get_distant_auth_methods(self):
"""
Fetch distant auth methods
"""
self.logger.debug("Fetching distant auth methods")
self.distant_auth_methods = []
raw = self.vault_client.auth_list()
for auth_method in raw:
self.distant_auth_methods.append(
VaultAuthMethod(
type=raw[auth_method]["type"],
path=(raw[auth_method]["path"] if 'path' in raw[auth_method] else auth_method),
description=raw[auth_method]["description"],
tuning=OrderedDict(sorted(raw[auth_method]["config"].items()))
)
)
self.logger.debug("Distant auth methods found")
for elem in self.distant_auth_methods:
self.logger.debug(elem)
def get_local_auth_methods(self):
"""
Fetch local auth methods
"""
self.logger.debug("Fetching local auth methods")
self.local_auth_methods = []
for auth_method in self.conf["auth-methods"]:
auth_config = None
if "auth_config" in auth_method:
auth_config = OrderedDict(sorted(auth_method["auth_config"].items()))
self.local_auth_methods.append(
VaultAuthMethod(
type=auth_method["type"],
path=auth_method["path"],
description=auth_method["description"],
tuning=OrderedDict(sorted(auth_method["tuning"].items())),
auth_config=auth_config
)
)
self.logger.debug("Local auth methods found")
for elem in self.local_auth_methods:
self.logger.debug(elem)
def disable_distant_auth_methods(self):
"""
Disable auth methods not found in conf
"""
self.logger.debug("Disabling auth methods")
for auth_method in self.distant_auth_methods:
if auth_method not in self.local_auth_methods:
self.logger.info("Disabling: " + str(auth_method))
self.vault_client.auth_disable(auth_method.path)
def enable_distant_auth_methods(self):
"""
Enable auth methods found in conf
"""
self.logger.debug("Enabling auth methods")
for auth_method in self.local_auth_methods:
if auth_method not in self.distant_auth_methods:
self.logger.info("Enabling: " + str(auth_method))
self.vault_client.auth_enable(
auth_type=auth_method.type,
path=auth_method.path,
description=auth_method.description
)
def tune_auth_method(self, local_auth_method, distant_auth_method):
"""
Tune a auth method
:param local_auth_method: Local auth method
:type local_auth_method: VaultAuthMethod
:param distant_auth_method: Distant auth method
:type distant_auth_method: VaultAuthMethod
"""
self.logger.debug("Local tuning for: " + local_auth_method.path)
self.logger.debug("Description: " + local_auth_method.description)
self.logger.debug("Hash: " + local_auth_method.get_tuning_hash())
self.logger.debug("Tuning: " + str(local_auth_method.tuning))
self.logger.debug("Distant tuning for: " + distant_auth_method.path)
self.logger.debug("Description: " + distant_auth_method.description)
self.logger.debug("Hash: " + distant_auth_method.get_tuning_hash())
self.logger.debug("Tuning: " + str(distant_auth_method.tuning))
self.vault_client.auth_tune(
local_auth_method.path,
default_lease_ttl=local_auth_method.tuning["default_lease_ttl"],
max_lease_ttl=local_auth_method.tuning["max_lease_ttl"],
description=local_auth_method.description
)
def find_auth_methods_to_tune(self):
"""
Identify auth methods where a tuning is needed
"""
self.logger.debug("Tuning auth methods")
for distant_auth in self.distant_auth_methods:
for local_auth in self.local_auth_methods:
if distant_auth == local_auth:
distant_tuning_hash = distant_auth.get_tuning_hash()
local_tuning_hash = local_auth.get_tuning_hash()
self.logger.debug("Hashs for %s/" % distant_auth.path)
self.logger.debug("Local: " + local_tuning_hash)
self.logger.debug("Distant: " + distant_tuning_hash)
if distant_tuning_hash != local_tuning_hash:
self.logger.info("The auth method " + local_auth.path +
" will be tuned")
self.tune_auth_method(local_auth, distant_auth)
def run(self, arg_parser, parsed_args):
"""
Module entry point
:param parsed_args: Arguments parsed fir this module
:type parsed_args: argparse.ArgumentParser.parse_args()
:param arg_parser: Argument parser
:type arg_parser: argparse.ArgumentParser
"""
self.parsed_args = parsed_args
self.arg_parser = arg_parser
self.logger.debug("Module " + self.module_name + " started")
if self.parsed_args.push:
self.logger.info("Pushing auth methods to Vault")
if not self.check_env_vars():
return False
self.read_configuration()
self.vault_client = VaultClient(
self.base_logger,
dry=self.parsed_args.dry_run,
skip_tls=self.parsed_args.skip_tls
)
self.vault_client.authenticate()
self.get_distant_auth_methods()
self.get_local_auth_methods()
for auth_method in self.local_auth_methods:
if auth_method in self.distant_auth_methods:
self.logger.debug("Auth method remaining unchanged " +
str(auth_method))
self.disable_distant_auth_methods()
self.enable_distant_auth_methods()
self.get_distant_auth_methods()
self.logger.info("Auth methods successfully pushed to Vault")
self.logger.info("Tuning auth methods")
self.find_auth_methods_to_tune()
self.logger.info("Auth methods successfully tuned")
self.logger.info("Setting up auth method specific configuration")
for auth_method in self.local_auth_methods:
auth_method_module = None
if auth_method.auth_config:
if auth_method.type == "ldap":
auth_method_module = AuthMethodLDAP(
self.base_logger,
auth_method.path,
auth_method.auth_config,
self.vault_client
)
elif auth_method.type == "approle":
auth_method_module = AuthMethodAppRole(
self.base_logger,
auth_method.path,
auth_method.auth_config,
self.vault_client
)
if auth_method_module:
auth_method_module.auth_method_configuration()
else:
self.logger.debug("No specific auth method configuration")
self.logger.info("Auth method specific configuration OK")
class VaultManagerLDAP:
"""
LDAP Module
"""
logger = None
subparser = None
kwargs = None
module_name = None
base_logger = None
conf = None
ldap_conf = None
vault_client = None
ldap_users = None
ldap_kubernetes_groups = None
policies_folder = None
user_policies_folder = None
group_policies_folder = None
kubernetes_policies_folder = None
group_policies_to_create = None
kubernetes_policies_to_create = None
user_policies_to_create = None
def __init__(self, base_logger=None):
"""
:param base_logger: main class name
:type base_logger: string
"""
self.base_logger = base_logger
if base_logger:
self.logger = logging.getLogger(base_logger + "." +
self.__class__.__name__)
else:
self.logger = logging.getLogger()
self.logger.debug("Initializing VaultManagerLDAP")
def connect_to_vault(self, vault_addr, vault_token):
"""
Connect to a Vault instance
:param vault_addr: Vault URL
:type vault_addr: str
:param vault_token: Vault token
:type vault_token: str
:return: VaultClient
"""
self.logger.debug("Connecting to Vault instance '%s'" % vault_addr)
vault_client = VaultClient(self.base_logger,
dry=self.kwargs.dry_run,
vault_addr=vault_addr,
skip_tls=self.kwargs.skip_tls)
vault_client.authenticate(vault_token)
return vault_client
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = \
subparsers.add_parser(self.module_name,
help=self.module_name + ' management')
self.subparser.add_argument("--list-groups",
action='store_true',
help="List LDAP groups")
self.subparser.add_argument(
"--create-policies",
action='store_true',
help="Create policies from LDAP groups and users")
self.subparser.add_argument(
"--manage-ldap-groups",
nargs='?',
metavar="LDAP_mount_point",
help="""Create LDAP groups in Vault with associated
policies at specified mount point""")
self.subparser.add_argument(
"--manage-ldap-users",
nargs='?',
metavar="LDAP_mount_point",
help="""Create LDAP users in Vault with associated
policies and groups at specified mount point""")
self.subparser.add_argument(
"--create-groups-secrets",
nargs='?',
metavar="groups_secrets_folder",
help="Create a folder for each group in <groups_secrets_folder>")
self.subparser.add_argument(
"--create-users-secrets",
nargs='?',
metavar="users_secrets_folder",
help="Create a folder for each user in <users_secrets_folder>")
self.subparser.set_defaults(module_name=self.module_name)
def get_subparser(self):
"""
Module subparser getter
:return: argparse.ArgumentParser.add_subparsers().add_parser()
"""
return self.subparser
def check_args_integrity(self):
"""
Checking provided arguments integrity
"""
self.logger.debug("Checking arguments integrity")
args_false_count = [
self.kwargs.create_policies, self.kwargs.manage_ldap_groups,
self.kwargs.manage_ldap_users, self.kwargs.list_groups,
self.kwargs.create_groups_secrets, self.kwargs.create_users_secrets
].count(False)
args_none_count = [
self.kwargs.create_policies, self.kwargs.manage_ldap_groups,
self.kwargs.manage_ldap_users, self.kwargs.list_groups,
self.kwargs.create_groups_secrets, self.kwargs.create_users_secrets
].count(None)
no_args_count = args_false_count + args_none_count
if no_args_count in [6, 7]:
self.logger.critical("you must specify a command")
return False
return True
def read_configuration(self):
"""
Read the policies configuration file
"""
self.logger.debug("Reading configuration")
with open(os.path.join(self.policies_folder, "policies.yml"),
'r') as fd:
try:
self.conf = yaml.safe_load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load conf file: " + str(e))
return False
self.logger.debug("Read conf: " + str(self.conf))
return True
def read_ldap_configuration(self):
"""
Read the LDAP configuration file
"""
self.logger.debug("Reading LDAP configuration file")
with open(os.path.join(self.kwargs.vault_config, "ldap.yml"),
'r') as fd:
try:
self.ldap_conf = yaml.safe_load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load LDAP conf file: %s" %
str(e))
return False
self.logger.debug("Read LDAP conf: " + str(self.conf))
return True
def get_ldap_data(self):
"""
Fetch users and groups from LDAP
"""
self.logger.info("Reading LDAP data")
# base_logger, server, user, password, group_dn, user_dn
try:
if re.search("^VAULT{{.*}}$", self.ldap_conf["ldap"]["password"]):
ldap_password = self.vault_client.read_string_with_secret(
self.ldap_conf["ldap"]["password"])
elif re.search("^ENV{{.*}}$", self.ldap_conf["ldap"]["password"]):
ldap_password = self.vault_client.read_string_with_env(
self.ldap_conf["ldap"]["password"])
else:
ldap_password = self.ldap_conf["ldap"]["password"]
except TypeError as e:
raise Exception("LDAP password does not exists in env at %s" %
str(self.ldap_conf["ldap"]["password"]))
ldap_reader = LDAPReader(self.base_logger,
self.ldap_conf["ldap"]["server"],
self.ldap_conf["ldap"]["username"],
ldap_password,
self.ldap_conf["ldap"]["kubernetes_group_dn"],
self.ldap_conf["ldap"]["group_dn"],
self.ldap_conf["ldap"]["user_dn"])
if not ldap_reader.connect_to_ldap():
return False
self.ldap_users = ldap_reader.get_all_users(
ldap_reader.get_all_groups())
self.ldap_kubernetes_groups = ldap_reader.get_kubernetes_groups()
self.logger.debug("Users found: " + str(self.ldap_users))
ldap_reader.disconnect_from_ldap()
return True
def create_groups_policies(self):
"""
Create a policy for each group
"""
self.logger.info("Creating groups policies")
ldap_groups = list(
sorted(
set([
group for user in self.ldap_users
for group in self.ldap_users[user]
])))
for read_group in self.conf["groups"]["groups_to_add"]:
if read_group not in ldap_groups:
self.logger.warning("Group " + read_group +
" in conf file 't been found in LDAP "
"groups. Default conf file. "
"The default group policy will be created "
"anyway.")
with open(
os.path.join(self.policies_folder,
self.conf["general"]["group"]["default_policy"]),
'r') as fd:
default_policy = fd.read()
for group in self.conf["groups"]["groups_to_add"]:
policy_file = os.path.join(self.group_policies_folder,
group + ".hcl")
self.group_policies_to_create.append(policy_file)
if os.path.isfile(policy_file):
self.logger.info("Policy for group " + group +
" already exists and will not be overwritten")
else:
with open(policy_file, 'w+') as fd:
fd.write(default_policy.replace("{{GROUP_NAME}}", group))
self.logger.info("Default policy for " + group +
" written")
def create_users_policies(self):
"""
Create policies for each LDAP user
"""
self.logger.info("Creating user policies")
with open(
os.path.join(self.policies_folder,
self.conf["general"]["user"]["default_policy"]),
'r') as fd:
default_policy = fd.read()
for user in self.ldap_users:
if len(
set(self.conf["groups"]["groups_to_add"]).intersection(
self.ldap_users[user])):
policy_file = os.path.join(self.user_policies_folder,
user + ".hcl")
self.user_policies_to_create.append(policy_file)
if os.path.isfile(policy_file):
self.logger.info(
"Policy for user " + user +
" already exists and will not be overwritten")
else:
with open(policy_file, 'w+') as fd:
fd.write(default_policy.replace("{{USER_NAME}}", user))
self.logger.info("Policy for user " + user +
" created")
def create_kubernetes_policies(self):
"""
Create policies to allow kubernetes service-accounts to read secrets
"""
self.logger.debug("creating kubernetes policies for service_accounts")
with open(
os.path.join(
self.policies_folder,
self.conf["general"]["kubernetes"]["default_policy"]),
'r') as fd:
default_policy = fd.read()
template = Template(default_policy)
for env in ["qa", "preprod", "prod"]:
for group in self.ldap_kubernetes_groups:
policy_file = os.path.join(self.kubernetes_policies_folder,
env, group + ".hcl")
self.kubernetes_policies_to_create.append(policy_file)
if os.path.isfile(policy_file):
self.logger.info(
"Policy for kubernetes group " + group + " in env " +
env + " already exists and will not be overwritten")
else:
with open(policy_file, 'w+') as fd:
fd.write(template.render(GROUP=group, ENV=env))
self.logger.info("Policy for kubernetes group " +
group + "in env " + env + " created")
def deleting_previous_policies(self):
"""
Deleting policies of non existing LDAP users
"""
self.logger.debug(
"Deleting policies of previously existing LDAP users")
for file in os.listdir(self.group_policies_folder):
policy_path = os.path.join(self.group_policies_folder, file)
if policy_path not in self.group_policies_to_create:
self.logger.info("Deleting group policy: " + policy_path)
os.remove(policy_path)
for file in os.listdir(self.user_policies_folder):
policy_path = os.path.join(self.user_policies_folder, file)
if policy_path not in self.user_policies_to_create:
self.logger.info("Deleting user policy: " + policy_path)
os.remove(policy_path)
def ldap_list_groups(self):
"""
Method running the list-groups function of LDAP module
Display LDAP groups
"""
self.logger.debug("LDAP list-groups starting")
self.logger.debug("Displaying LDAP groups")
groups = []
for user in self.ldap_users:
for group in self.ldap_users[user]:
if group not in groups:
groups.append(group)
self.logger.info(str(sorted(groups)))
def ldap_create_policies(self):
"""
Method running the create-policies function of LDAP module
"""
self.logger.debug("LDAP create-policies starting")
self.logger.info("Creating LDAP policies")
self.create_groups_policies()
self.create_users_policies()
self.create_kubernetes_policies()
self.deleting_previous_policies()
def ldap_manage_ldap_groups(self):
"""
Method running the manage-ldap-groups function of LDAP module
Manage groups in Vault LDAP configuration
"""
self.logger.debug("LDAP manage-ldap-groups starting")
self.logger.info("Managing groups in Vault LDAP '%s' config" %
self.kwargs.manage_ldap_groups)
self.logger.debug("Managing groups to Vault LDAP configuration")
raw_vault_ldap_groups = self.vault_client.list('/auth/ldap/groups')
existing_groups = []
if len(raw_vault_ldap_groups):
existing_groups = raw_vault_ldap_groups["keys"]
for group in self.conf["groups"]["groups_to_add"]:
if group in existing_groups:
existing_groups.remove(group)
policies = ["group_" + group + "_policy"]
if "root" in self.conf["general"]["group"] and \
group in self.conf["general"]["group"]["root"]:
policies.append("root")
self.logger.info("Adding polices %s to group %s" %
(str(policies), group))
self.vault_client.write(
"/auth/ldap/groups/" + group, {
"policies":
utils.list_to_string(self.logger, policies, separator="")
})
self.logger.debug("Removing groups %s from Vault LDAP conf" %
str(existing_groups))
for group in existing_groups:
self.logger.info("Removing group %s from Vault LDAP conf" % group)
self.vault_client.delete('/auth/ldap/groups/' + group)
def ldap_manage_ldap_users(self):
"""
Method running the manage-ldap-users function of LDAP module
Manage users in Vault LDAP configuration
"""
self.logger.debug("LDAP manage-ldap-users starting")
self.logger.info("Managing users in Vault LDAP '%s' config" %
self.kwargs.manage_ldap_users)
self.logger.debug("Managing users to Vault LDAP configuration")
raw_vault_ldap_users = self.vault_client.list('/auth/ldap/users')
self.logger.debug("Users found: " + str(raw_vault_ldap_users))
existing_users = []
if len(raw_vault_ldap_users):
existing_users = raw_vault_ldap_users["keys"]
for user in self.ldap_users:
groups_of_user = list(
set(self.conf["groups"]["groups_to_add"]).intersection(
self.ldap_users[user]))
if not len(groups_of_user):
continue
if user in existing_users:
existing_users.remove(user)
policies = ["user_" + user + "_policy"]
if "root" in self.conf["general"]["group"] and \
user in self.conf["general"]["user"]["root"]:
policies.append("root")
self.logger.info("Adding polices %s to user %s" %
(str(policies), user))
self.logger.info("Adding groups %s to user %s" %
(str(groups_of_user), user))
self.vault_client.write(
"/auth/ldap/users/" + user, {
"policies":
utils.list_to_string(self.logger, policies, separator=""),
"groups":
utils.list_to_string(
self.logger, groups_of_user, separator="")
})
self.logger.debug("Removing users %s from Vault LDAP conf" %
str(existing_users))
for user in existing_users:
self.logger.info("Removing user %s from Vault LDAP conf" % user)
self.vault_client.delete('/auth/ldap/users/' + user)
self.logger.info("Creating k8s secrets paths for each user")
self.create_kubernetes_policies()
def find_ldap_group(self, user, group_regex):
"""
Find a group matching a regex
"""
ft = []
for group in self.ldap_users[user]:
match = re.match(group_regex, group)
if match:
ft.extend([g for g in match.groups() if g is not None])
if len(ft) == 0:
return ""
return ",".join(ft)
def ldap_create_groups_secrets(self):
"""
Method running the create-groups-secrets function of LDAP module
Create a secret folder for each LDAP group under specified path
"""
self.logger.debug("LDAP create-groups-secrets starting")
self.logger.info("Creating groups folders under secret path '/%s'" %
self.kwargs.create_groups_secrets)
self.logger.debug("Creating groups secrets under %s" %
self.kwargs.create_groups_secrets)
existing_folders = self.vault_client.list(
self.kwargs.create_groups_secrets)
if len(existing_folders):
existing_folders = [
e.replace("/", "") for e in existing_folders['keys']
]
self.logger.debug("Already existing folders: " + str(existing_folders))
for group in self.conf["groups"]["groups_to_add"]:
if group not in existing_folders:
self.logger.info("Creating folder: " + group)
self.vault_client.write(
self.kwargs.create_groups_secrets + "/" + group +
"/description", {group: "group private secrets space"})
for group in existing_folders:
if group not in self.conf["groups"]["groups_to_add"]:
tree = self.vault_client.get_secrets_tree(
self.kwargs.create_groups_secrets + "/" + group)
self.logger.info("Deleting folder " + group +
" and associated secrets " + str(tree))
for secret in tree:
self.vault_client.delete(secret)
def ldap_create_users_secrets(self):
"""
Method running the create-users-secrets function of LDAP module
Create a secret folder for each LDAP user under specified path
"""
self.logger.debug("LDAP create-users-secrets starting")
self.logger.info("Creating users folders under secret path '/%s'" %
self.kwargs.create_users_secrets)
self.logger.debug("Creating users secrets under %s" %
self.kwargs.create_users_secrets)
enabled_users = []
for user in self.ldap_users:
groups_of_user = list(
set(self.conf["groups"]["groups_to_add"]).intersection(
self.ldap_users[user]))
if len(groups_of_user):
enabled_users.append(user)
existing_folders = self.vault_client.list(
self.kwargs.create_users_secrets)
if len(existing_folders):
existing_folders = [
e.replace("/", "") for e in existing_folders['keys']
]
self.logger.debug("Already existing folders: " + str(existing_folders))
for user in enabled_users:
if user not in existing_folders:
self.logger.info("Creating folder: " + user)
self.vault_client.write(
self.kwargs.create_users_secrets + "/" + user +
"/description", {user: "******"})
for user in existing_folders:
if user not in enabled_users:
tree = self.vault_client.get_secrets_tree(
self.kwargs.create_users_secrets + "/" + user)
self.logger.info("Deleting folder " + user +
" and associated secrets " + str(tree))
for secret in tree:
self.vault_client.delete(secret)
def run(self, kwargs):
"""
Module entry point
:param kwargs: Arguments parsed
:type kwargs: dict
"""
# Convert kwargs to an Object with kwargs dict as class vars
self.kwargs = namedtuple("KwArgs", kwargs.keys())(*kwargs.values())
self.logger.debug("Module " + self.module_name + " started")
if not self.check_args_integrity():
self.subparser.print_help()
return False
missing_args = utils.keys_exists_in_dict(self.logger,
dict(self.kwargs._asdict()),
[{
"key": "vault_config",
"exc": [None, False, '']
}])
if len(missing_args):
raise ValueError(
"Following arguments are missing %s\n" %
[k['key'].replace("_", "-") for k in missing_args])
self.policies_folder = os.path.join(self.kwargs.vault_config,
"policies")
self.user_policies_folder = os.path.join(self.policies_folder, "user")
self.kubernetes_policies_folder = os.path.join(self.policies_folder,
"service", "kubernetes")
self.group_policies_folder = os.path.join(self.policies_folder,
"group")
self.group_policies_to_create = []
self.user_policies_to_create = []
self.kubernetes_policies_to_create = []
if not self.read_configuration() or not self.read_ldap_configuration():
return False
self.vault_client = VaultClient(self.base_logger,
vault_addr=self.kwargs.vault_addr,
dry=self.kwargs.dry_run,
skip_tls=self.kwargs.skip_tls)
self.vault_client.authenticate(self.kwargs.vault_token)
if not self.get_ldap_data():
return False
if self.kwargs.list_groups:
self.ldap_list_groups()
return True
if self.kwargs.create_policies:
self.ldap_create_policies()
return True
missing_args = utils.keys_exists_in_dict(self.logger,
dict(self.kwargs._asdict()),
[{
"key": "vault_addr",
"exc": [None, '']
}, {
"key": "vault_token",
"exc": [None, False]
}, {
"key": "vault_config",
"exc": [None, False, '']
}])
if len(missing_args):
raise ValueError(
"Following arguments are missing %s\n" %
[k['key'].replace("_", "-") for k in missing_args])
self.vault_client = self.connect_to_vault(self.kwargs.vault_addr,
self.kwargs.vault_token)
if self.kwargs.manage_ldap_groups:
self.ldap_manage_ldap_groups()
if self.kwargs.manage_ldap_users:
self.ldap_manage_ldap_users()
if self.kwargs.create_groups_secrets:
self.ldap_create_groups_secrets()
if self.kwargs.create_users_secrets:
self.ldap_create_users_secrets()
class VaultManagerPolicies:
logger = None
subparser = None
parsed_args = None
arg_parser = None
module_name = None
vault_client = None
base_logger = None
policies_folder = None
def __init__(self, base_logger, subparsers):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
self.logger = logging.getLogger(base_logger + "." + self.__class__.__name__)
self.logger.debug("Initializing VaultManagerPolicies")
self.initialize_subparser(subparsers)
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = \
subparsers.add_parser(self.module_name,
help=self.module_name + ' management')
self.subparser.add_argument("--pull", action='store_true',
help="Pull distant policies from Vault")
self.subparser.add_argument("--push", action='store_true',
help="Push local policies to Vault")
self.subparser.set_defaults(module_name=self.module_name)
def get_subparser(self):
"""
Module subparser getter
:return: argparse.ArgumentParser.add_subparsers().add_parser()
"""
return self.subparser
def check_args_integrity(self):
"""
Checking provided arguments integrity
"""
self.logger.debug("Checking arguments integrity")
if self.parsed_args.pull and self.parsed_args.push:
self.logger.critical("push and pull args cannot "
"be specified at the same time")
return False
elif not self.parsed_args.pull and not self.parsed_args.push:
self.logger.critical("You must specify pull or push")
return False
return True
def check_env_vars(self):
"""
Check if all needed env vars are set
:return: bool
"""
self.logger.debug("Checking env variables")
needed_env_vars = ["VAULT_ADDR", "VAULT_TOKEN", "VAULT_CONFIG"]
if not all(env_var in os.environ for env_var in needed_env_vars):
self.logger.critical("The following env vars must be set")
self.logger.critical(str(needed_env_vars))
return False
self.logger.debug("All env vars are set")
if not os.path.isdir(os.environ["VAULT_CONFIG"]):
self.logger.critical(
os.environ["VAULT_CONFIG"] + " is not a valid folder")
return False
self.logger.info("Vault address: " + os.environ["VAULT_ADDR"])
self.logger.info("Vault config folder: " + os.environ["VAULT_CONFIG"])
return True
def pull_policies(self):
"""
Pull policies from vault
"""
self.logger.debug("Pulling policies")
distant_policies = self.vault_client.policy_list()
self.logger.info("Distant policies found:" + str(distant_policies))
for policy in distant_policies:
# policy name will always be 'type_name_policy'
splitted = policy.split("_")
if len(splitted) != 3 or splitted[2] != "policy":
self.logger.warning("Policy " + policy +
" does not match policy name pattern "
"and will not be pulled")
continue
# create the parent folder policy if doest not exists (user, etc...)
policy_folder = os.path.join(self.policies_folder, splitted[0])
if not os.path.isdir(policy_folder):
self.logger.debug("Folder " + policy_folder +
" doest not exists, creating...")
os.makedirs(policy_folder)
# create the policy file
policy_path = os.path.join(policy_folder, splitted[1] + ".hcl")
with open(policy_path, 'w+') as fd:
fd.write(self.vault_client.policy_get(policy))
self.logger.info("Policy " + policy_path + " saved")
self.logger.info("Policies fetched in policies folder")
def push_policies(self):
"""
Push all policies from policies folder to Vault
"""
self.logger.debug("Push all policies")
distant_policies = self.vault_client.policy_list()
local_policies = []
# Building local policies list
for policy_file in glob.iglob(os.path.join(self.policies_folder,
"*/*.hcl"), recursive=True):
name = os.path.splitext(os.path.basename(policy_file))[0]
prefix = policy_file.split(os.sep)[-2]
self.logger.debug("Local policy %s - prefix: %s - name: %s found"
% (policy_file, prefix, name))
with open(policy_file, 'r') as fd:
local_policies.append({"name": prefix + "_" + name + "_policy",
"content": fd.read()})
# Removing distant policies which doesn't exists locally
for distant_policy in distant_policies:
if distant_policy not in [pol["name"] for pol in local_policies]:
self.logger.info("Removing distant policy " + distant_policy)
self.vault_client.policy_delete(distant_policy)
# Push local policies
for policy in local_policies:
self.vault_client.policy_set(policy_name=policy["name"],
policy_content=policy["content"])
if policy["name"] in distant_policies:
self.logger.info("Policy %s has been updated" % policy["name"])
else:
self.logger.info("Policy %s has been created" % policy["name"])
self.logger.info("Policies pushed to Vault")
def run(self, arg_parser, parsed_args):
"""
Module entry point
:param arg_parser: Arguments parser instance
:param parsed_args: Arguments parsed fir this module
:type parsed_args: argparse.ArgumentParser.parse_args()
"""
self.parsed_args = parsed_args
self.logger.debug(self.parsed_args)
self.arg_parser = arg_parser
self.logger.debug("Module " + self.module_name + " started")
if not self.check_args_integrity():
self.arg_parser.print_help()
return False
if not self.check_env_vars():
return False
self.policies_folder = os.path.join(os.environ["VAULT_CONFIG"],
"policies")
if not os.path.isdir(self.policies_folder):
os.mkdir(self.policies_folder)
self.vault_client = VaultClient(
self.base_logger,
self.parsed_args.dry_run
)
self.vault_client.authenticate()
if self.parsed_args.pull:
self.logger.info("Pulling Policies from Vault")
self.pull_policies()
if self.parsed_args.push:
self.logger.info("Pushing Policies to Vault")
self.push_policies()
class VaultManagerAuth:
"""
Authentication module
"""
logger = None
base_logger = None
subparser = None
parsed_args = None
arg_parser = None
module_name = None
conf = None
vault_client = None
local_auth_methods = None
distant_auth_methods = None
def __init__(self, base_logger=None):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
if base_logger:
self.logger = logging.getLogger(base_logger + "." +
self.__class__.__name__)
else:
self.logger = logging.getLogger()
self.logger.debug("Initializing VaultManagerAuth")
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = subparsers.add_parser(self.module_name,
help=self.module_name +
' management')
self.subparser.add_argument("--push",
action='store_true',
help="Push auth methods to Vault")
self.subparser.set_defaults(module_name=self.module_name)
def check_args_integrity(self):
"""
Checking provided arguments integrity
"""
self.logger.debug("Checking arguments integrity")
args_false_count = [self.kwargs.push].count(False)
args_none_count = [self.kwargs.push].count(None)
no_args_count = args_false_count + args_none_count
if no_args_count in [1]:
self.logger.critical("you must specify a command")
return False
return True
def read_configuration(self):
"""
Read configuration file
"""
self.logger.debug("Reading configuration")
with open(os.path.join(self.kwargs.vault_config, "auth-methods.yml"),
'r') as fd:
try:
self.conf = yaml.safe_load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load conf file: " + str(e))
return False
self.logger.debug("Read conf: " + str(self.conf))
return True
def get_distant_auth_methods(self):
"""
Fetch distant auth methods
"""
self.logger.debug("Fetching distant auth methods")
self.distant_auth_methods = []
raw = self.vault_client.auth_list()
for auth_method in raw:
self.distant_auth_methods.append(
VaultAuthMethod(
type=raw[auth_method]["type"],
path=(raw[auth_method]["path"]
if 'path' in raw[auth_method] else auth_method),
description=raw[auth_method]["description"],
tuning=OrderedDict(
sorted(raw[auth_method]["config"].items()))))
self.logger.debug("Distant auth methods found")
for elem in self.distant_auth_methods:
self.logger.debug(elem)
def get_local_auth_methods(self):
"""
Fetch local auth methods
"""
self.logger.debug("Fetching local auth methods")
self.local_auth_methods = []
for auth_method in self.conf["auth-methods"]:
auth_config = None
if "auth_config" in auth_method:
auth_config = OrderedDict(
sorted(auth_method["auth_config"].items()))
self.local_auth_methods.append(
VaultAuthMethod(type=auth_method["type"],
path=auth_method["path"],
description=auth_method["description"],
tuning=OrderedDict(
sorted(auth_method["tuning"].items())),
auth_config=auth_config))
self.logger.debug("Local auth methods found")
for elem in self.local_auth_methods:
self.logger.debug(elem)
def disable_distant_auth_methods(self):
"""
Disable auth methods not found in conf
"""
self.logger.debug("Disabling auth methods")
for auth_method in self.distant_auth_methods:
if auth_method not in self.local_auth_methods:
self.logger.info("Disabling: " + str(auth_method))
self.vault_client.auth_disable(auth_method.path)
def enable_distant_auth_methods(self):
"""
Enable auth methods found in conf
"""
self.logger.debug("Enabling auth methods")
for auth_method in self.local_auth_methods:
if auth_method not in self.distant_auth_methods:
self.logger.info("Enabling: " + str(auth_method))
self.vault_client.auth_enable(
auth_type=auth_method.type,
path=auth_method.path,
description=auth_method.description)
def tune_auth_method(self, local_auth_method, distant_auth_method):
"""
Tune a auth method
:param local_auth_method: Local auth method
:type local_auth_method: VaultAuthMethod
:param distant_auth_method: Distant auth method
:type distant_auth_method: VaultAuthMethod
"""
self.logger.debug("Local tuning for: " + local_auth_method.path)
self.logger.debug("Description: " + local_auth_method.description)
self.logger.debug("Hash: " + local_auth_method.get_tuning_hash())
self.logger.debug("Tuning: " + str(local_auth_method.tuning))
self.logger.debug("Distant tuning for: " + distant_auth_method.path)
self.logger.debug("Description: " + distant_auth_method.description)
self.logger.debug("Hash: " + distant_auth_method.get_tuning_hash())
self.logger.debug("Tuning: " + str(distant_auth_method.tuning))
self.vault_client.auth_tune(
local_auth_method.path,
default_lease_ttl=local_auth_method.tuning["default_lease_ttl"],
max_lease_ttl=local_auth_method.tuning["max_lease_ttl"],
description=local_auth_method.description)
def find_auth_methods_to_tune(self):
"""
Identify auth methods where a tuning is needed
"""
self.logger.debug("Tuning auth methods")
for distant_auth in self.distant_auth_methods:
for local_auth in self.local_auth_methods:
if distant_auth == local_auth:
distant_tuning_hash = distant_auth.get_tuning_hash()
local_tuning_hash = local_auth.get_tuning_hash()
self.logger.debug("Hashs for %s/" % distant_auth.path)
self.logger.debug("Local: " + local_tuning_hash)
self.logger.debug("Distant: " + distant_tuning_hash)
if distant_tuning_hash != local_tuning_hash:
self.logger.info("The auth method " + local_auth.path +
" will be tuned")
self.tune_auth_method(local_auth, distant_auth)
def auth_push(self):
"""
Push auth methods configuration to Vault
"""
self.logger.info("Pushing auth methods to Vault")
self.read_configuration()
self.get_distant_auth_methods()
self.get_local_auth_methods()
for auth_method in self.local_auth_methods:
if auth_method in self.distant_auth_methods:
self.logger.debug("Auth method remaining unchanged " +
str(auth_method))
if "auth-methods-deletion" in self.conf and \
self.conf["auth-methods-deletion"]:
self.disable_distant_auth_methods()
self.enable_distant_auth_methods()
self.get_distant_auth_methods()
self.logger.info("Auth methods successfully pushed to Vault")
self.logger.info("Tuning auth methods")
self.find_auth_methods_to_tune()
self.logger.info("Auth methods successfully tuned")
self.logger.info("Setting up auth method specific configuration")
for auth_method in self.local_auth_methods:
auth_method_module = None
if auth_method.auth_config:
if auth_method.type == "ldap":
auth_method_module = AuthMethodLDAP(
self.base_logger, auth_method.path,
auth_method.auth_config, self.vault_client)
elif auth_method.type == "approle":
auth_method_module = AuthMethodAppRole(
self.base_logger, auth_method.path,
auth_method.auth_config, self.vault_client)
if auth_method_module:
auth_method_module.auth_method_configuration()
else:
self.logger.debug("No specific auth method configuration")
self.logger.info("Auth method specific configuration OK")
def run(self, kwargs):
"""
Module entry point
:param kwargs: Arguments parsed
:type kwargs: dict
"""
# Convert kwargs to an Object with kwargs dict as class vars
self.kwargs = namedtuple("KwArgs", kwargs.keys())(*kwargs.values())
self.logger.debug("Module " + self.module_name + " started")
if not self.check_args_integrity():
self.subparser.print_help()
return False
missing_args = utils.keys_exists_in_dict(self.logger,
dict(self.kwargs._asdict()),
[{
"key": "vault_addr",
"exc": [None, '']
}, {
"key": "vault_token",
"exc": [None, False]
}, {
"key": "vault_config",
"exc": [None, False, '']
}])
if len(missing_args):
raise ValueError(
"Following arguments are missing %s\n" %
[k['key'].replace("_", "-") for k in missing_args])
self.vault_client = VaultClient(self.base_logger,
vault_addr=self.kwargs.vault_addr,
dry=self.kwargs.dry_run,
skip_tls=self.kwargs.skip_tls)
self.vault_client.authenticate()
if self.kwargs.push:
self.auth_push()
class VaultManagerAudit:
"""
Audit Module
"""
logger = None
base_logger = None
subparser = None
parsed_args = None
arg_parser = None
module_name = None
conf = None
vault_client = None
distant_audit_devices = None
local_audit_devices = None
def __init__(self, base_logger, subparsers):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
self.logger = logging.getLogger(base_logger + "." +
self.__class__.__name__)
self.logger.debug("Initializing VaultManagerLDAP")
self.initialize_subparser(subparsers)
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = subparsers.add_parser(self.module_name,
help=self.module_name +
' management')
self.subparser.add_argument("--push",
action='store_true',
help="Push audit configuration to Vault")
self.subparser.set_defaults(module_name=self.module_name)
def read_configuration(self):
"""
Read configuration file
"""
self.logger.debug("Reading configuration")
with open(
os.path.join(os.environ["VAULT_CONFIG"], "audit-devices.yml"),
'r') as fd:
try:
self.conf = yaml.load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load conf file: " + str(e))
return False
self.logger.debug("Read conf: " + str(self.conf))
return True
def check_env_vars(self):
"""
Check if all needed env vars are set
:return: bool
"""
self.logger.debug("Checking env variables")
needed_env_vars = ["VAULT_ADDR", "VAULT_TOKEN", "VAULT_CONFIG"]
if not all(env_var in os.environ for env_var in needed_env_vars):
self.logger.critical("The following env vars must be set")
self.logger.critical(str(needed_env_vars))
return False
self.logger.debug("All env vars are set")
if not os.path.isdir(os.environ["VAULT_CONFIG"]):
self.logger.critical(os.environ["VAULT_CONFIG"] +
" is not a valid folder")
return False
self.logger.info("Vault address: " + os.environ["VAULT_ADDR"])
self.logger.info("Vault config folder: " + os.environ["VAULT_CONFIG"])
return True
def get_distant_audit_devices(self):
"""
Fetch distant audit devices
"""
self.logger.debug("Fetching distant audit devices")
self.distant_audit_devices = []
raw = self.vault_client.audit_list()
for elem in raw:
self.distant_audit_devices.append(
VaultAuditDevice(raw[elem]["type"], raw[elem]["path"],
raw[elem]["description"],
raw[elem]["options"]))
self.logger.debug("Distant audit devices found")
for elem in self.distant_audit_devices:
self.logger.debug(elem)
def get_local_audit_devices(self):
"""
Fetch local audit devices
"""
self.logger.debug("Fetching local audit devices")
self.local_audit_devices = []
for audit_device in self.conf["audit-devices"]:
self.local_audit_devices.append(
VaultAuditDevice(audit_device["type"], audit_device["path"],
audit_device["description"],
audit_device["options"]))
self.logger.debug("Local audit devices found")
for elem in self.local_audit_devices:
self.logger.debug(elem)
def disable_distant_audit_devices(self):
"""
Disable audit devices not found in conf
"""
self.logger.debug("Disabling audit devices")
for audit_device in self.distant_audit_devices:
if audit_device not in self.local_audit_devices:
self.logger.info("Disabling: " + str(audit_device))
self.vault_client.audit_disable(audit_device.path)
def enable_distant_audit_devices(self):
"""
Enable audit devices found in conf
"""
self.logger.debug("Enabling audit devices")
for audit_device in self.local_audit_devices:
if audit_device not in self.distant_audit_devices:
self.logger.info("Enabling: " + str(audit_device))
self.vault_client.audit_enable(audit_device.type,
audit_device.path,
audit_device.description,
audit_device.options)
def run(self, arg_parser, parsed_args):
"""
Module entry point
:param parsed_args: Arguments parsed fir this module
:type parsed_args: argparse.ArgumentParser.parse_args()
"""
self.parsed_args = parsed_args
self.arg_parser = arg_parser
self.logger.debug("Module " + self.module_name + " started")
if self.parsed_args.push:
if not self.check_env_vars():
return False
self.logger.info("Pushing audit devices configuration to Vault")
self.read_configuration()
self.vault_client = VaultClient(self.base_logger,
self.parsed_args.dry_run)
self.vault_client.authenticate()
self.get_distant_audit_devices()
self.get_local_audit_devices()
for audit_device in self.local_audit_devices:
if audit_device in self.distant_audit_devices:
self.logger.info("Audit device remaining unchanged " +
str(audit_device))
self.disable_distant_audit_devices()
self.enable_distant_audit_devices()
self.logger.info("Audit devices successfully pushed to Vault")
class VaultManagerLDAP:
"""
LDAP Module
"""
logger = None
subparser = None
parsed_args = None
arg_parser = None
module_name = None
base_logger = None
conf = None
ldap_conf = None
vault_client = None
ldap_users = None
policies_folder = None
user_policies_folder = None
group_policies_folder = None
group_policies_to_create = None
user_policies_to_create = None
def __init__(self, base_logger, subparsers):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
self.logger = logging.getLogger(base_logger + "." +
self.__class__.__name__)
self.logger.debug("Initializing VaultManagerLDAP")
self.initialize_subparser(subparsers)
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = \
subparsers.add_parser(self.module_name,
help=self.module_name + ' management')
self.subparser.add_argument("--list-groups",
action='store_true',
help="List LDAP groups")
self.subparser.add_argument(
"--create-policies",
action='store_true',
help="Create policies from LDAP groups and users")
self.subparser.add_argument(
"--manage-ldap-groups",
nargs='?',
metavar="LDAP_mount_point",
help="""Create LDAP groups in Vault with associated
policies at specified mount point""")
self.subparser.add_argument(
"--manage-ldap-users",
nargs='?',
metavar="LDAP_mount_point",
help="""Create LDAP users in Vault with associated
policies and groups at specified mount point""")
self.subparser.add_argument(
"--create-groups-secrets",
nargs='?',
metavar="groups_secrets_folder",
help="Create a folder for each group in <groups_secrets_folder>")
self.subparser.add_argument(
"--create-users-secrets",
nargs='?',
metavar="users_secrets_folder",
help="Create a folder for each user in <users_secrets_folder>")
self.subparser.set_defaults(module_name=self.module_name)
def get_subparser(self):
"""
Module subparser getter
:return: argparse.ArgumentParser.add_subparsers().add_parser()
"""
return self.subparser
def check_args_integrity(self):
"""
Checking provided arguments integrity
"""
self.logger.debug("Checking arguments integrity")
args_false_count = [
self.parsed_args.create_policies,
self.parsed_args.manage_ldap_groups,
self.parsed_args.manage_ldap_users, self.parsed_args.list_groups,
self.parsed_args.create_groups_secrets,
self.parsed_args.create_users_secrets
].count(False)
args_none_count = [
self.parsed_args.create_policies,
self.parsed_args.manage_ldap_groups,
self.parsed_args.manage_ldap_users, self.parsed_args.list_groups,
self.parsed_args.create_groups_secrets,
self.parsed_args.create_users_secrets
].count(None)
no_args_count = args_false_count + args_none_count
if no_args_count in [6, 7]:
self.logger.critical("you must specify a command")
return False
return True
def check_env_vars(self):
"""
Check if all needed env vars are set
:return: bool
"""
self.logger.debug("Checking env variables")
needed_env_vars = ["VAULT_ADDR", "VAULT_TOKEN", "VAULT_CONFIG"]
if not all(env_var in os.environ for env_var in needed_env_vars):
self.logger.critical("The following env vars must be set")
self.logger.critical(str(needed_env_vars))
return False
self.logger.debug("All env vars are set")
if not os.path.isdir(os.environ["VAULT_CONFIG"]):
self.logger.critical(os.environ["VAULT_CONFIG"] +
" is not a valid folder")
return False
self.logger.info("Vault address: " + os.environ["VAULT_ADDR"])
self.logger.info("Vault config folder: " + os.environ["VAULT_CONFIG"])
return True
def read_configuration(self):
"""
Read the policies configuration file
"""
self.logger.debug("Reading configuration")
with open(os.path.join(self.policies_folder, "policies.yml"),
'r') as fd:
try:
self.conf = yaml.load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load conf file: " + str(e))
return False
self.logger.debug("Read conf: " + str(self.conf))
return True
def read_ldap_configuration(self):
"""
Read the LDAP configuration file
"""
self.logger.debug("Reading LDAP configuration file")
with open(os.path.join(os.environ["VAULT_CONFIG"], "ldap.yml"),
'r') as fd:
try:
self.ldap_conf = yaml.load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load LDAP conf file: %s" %
str(e))
return False
self.logger.debug("Read LDAP conf: " + str(self.conf))
return True
def get_ldap_data(self):
"""
Fetch users and groups from LDAP
"""
self.logger.info("Reading LDAP data")
# base_logger, server, user, password, group_dn, user_dn
try:
ldap_password = self.vault_client.read_string_with_secret(
self.ldap_conf["ldap"]["password"])
except TypeError as e:
raise Exception("LDAP password does not exists in Vault at %s" %
str(self.ldap_conf["ldap"]["password"]))
ldap_reader = LDAPReader(self.base_logger,
self.ldap_conf["ldap"]["server"],
self.ldap_conf["ldap"]["username"],
ldap_password,
self.ldap_conf["ldap"]["group_dn"],
self.ldap_conf["ldap"]["user_dn"])
if not ldap_reader.connect_to_ldap():
return False
self.ldap_users = ldap_reader.get_all_users(
ldap_reader.get_all_groups())
self.logger.debug("Users found: " + str(self.ldap_users))
ldap_reader.disconnect_from_ldap()
return True
def create_groups_policies(self):
"""
Create a policy for each group
"""
self.logger.info("Creating groups policies")
ldap_groups = list(
sorted(
set([
group for user in self.ldap_users
for group in self.ldap_users[user]
])))
for read_group in self.conf["groups"]["groups_to_add"]:
if read_group not in ldap_groups:
self.logger.warning("Group " + read_group +
" in conf file 't been found in LDAP "
"groups. Default conf file. "
"The default group policy will be created "
"anyway.")
with open(
os.path.join(self.policies_folder,
self.conf["general"]["group"]["default_policy"]),
'r') as fd:
default_policy = fd.read()
for group in self.conf["groups"]["groups_to_add"]:
policy_file = os.path.join(self.group_policies_folder,
group + ".hcl")
self.group_policies_to_create.append(policy_file)
if os.path.isfile(policy_file):
self.logger.info("Policy for group " + group +
" already exists and will not be overwritten")
else:
with open(policy_file, 'w+') as fd:
fd.write(default_policy.replace("{{GROUP_NAME}}", group))
self.logger.info("Default policy for " + group +
" written")
def create_users_policies(self):
"""
Create policies for each LDAP user
"""
self.logger.info("Creating user policies")
with open(
os.path.join(self.policies_folder,
self.conf["general"]["user"]["default_policy"]),
'r') as fd:
default_policy = fd.read()
for user in self.ldap_users:
if len(
set(self.conf["groups"]["groups_to_add"]).intersection(
self.ldap_users[user])):
policy_file = os.path.join(self.user_policies_folder,
user + ".hcl")
self.user_policies_to_create.append(policy_file)
if os.path.isfile(policy_file):
self.logger.info(
"Policy for user " + user +
" already exists and will not be overwritten")
else:
with open(policy_file, 'w+') as fd:
fd.write(default_policy.replace("{{USER_NAME}}", user))
self.logger.info("Policy for user " + user +
" created")
def deleting_previous_policies(self):
"""
Deleting policies of non existing LDAP users
"""
self.logger.debug(
"Deleting policies of previously existing LDAP users")
for file in os.listdir(self.group_policies_folder):
policy_path = os.path.join(self.group_policies_folder, file)
if policy_path not in self.group_policies_to_create:
self.logger.info("Deleting group policy: " + policy_path)
os.remove(policy_path)
for file in os.listdir(self.user_policies_folder):
policy_path = os.path.join(self.user_policies_folder, file)
if policy_path not in self.user_policies_to_create:
self.logger.info("Deleting user policy: " + policy_path)
os.remove(policy_path)
def manage_groups_in_vault_ldap_conf(self):
"""
Manage groups in Vault LDAP configuration
"""
self.logger.debug("Managing groups to Vault LDAP configuration")
raw_vault_ldap_groups = self.vault_client.list('/auth/ldap/groups')
existing_groups = []
if len(raw_vault_ldap_groups):
existing_groups = raw_vault_ldap_groups["keys"]
for group in self.conf["groups"]["groups_to_add"]:
if group in existing_groups:
existing_groups.remove(group)
policies = ["group_" + group + "_policy"]
if "root" in self.conf["general"]["group"] and \
group in self.conf["general"]["group"]["root"]:
policies.append("root")
self.logger.info("Adding polices %s to group %s" %
(str(policies), group))
self.vault_client.write(
"/auth/ldap/groups/" + group,
{"policies": self.list_to_string(policies, separator="")})
self.logger.debug("Removing groups %s from Vault LDAP conf" %
str(existing_groups))
for group in existing_groups:
self.logger.info("Removing group %s from Vault LDAP conf" % group)
self.vault_client.delete('/auth/ldap/groups/' + group)
def manage_users_in_vault_ldap_conf(self):
"""
Manage users in Vault LDAP configuration
"""
self.logger.debug("Managing users to Vault LDAP configuration")
raw_vault_ldap_users = self.vault_client.list('/auth/ldap/users')
self.logger.debug("Users found: " + str(raw_vault_ldap_users))
existing_users = []
if len(raw_vault_ldap_users):
existing_users = raw_vault_ldap_users["keys"]
for user in self.ldap_users:
groups_of_user = list(
set(self.conf["groups"]["groups_to_add"]).intersection(
self.ldap_users[user]))
if not len(groups_of_user):
continue
if user in existing_users:
existing_users.remove(user)
policies = ["user_" + user + "_policy"]
if "root" in self.conf["general"]["group"] and \
user in self.conf["general"]["user"]["root"]:
policies.append("root")
self.logger.info("Adding polices %s to user %s" %
(str(policies), user))
self.logger.info("Adding groups %s to user %s" %
(str(groups_of_user), user))
self.vault_client.write(
"/auth/ldap/users/" + user, {
"policies": self.list_to_string(policies, separator=""),
"groups": self.list_to_string(groups_of_user, separator="")
})
self.logger.debug("Removing users %s from Vault LDAP conf" %
str(existing_users))
for user in existing_users:
self.logger.info("Removing user %s from Vault LDAP conf" % user)
self.vault_client.delete('/auth/ldap/users/' + user)
def list_to_string(self, list_to_serialize, separator="'"):
"""
Transform a list to a string
:param list_to_serialize: list to transform
:type list_to_serialize: list
:return: str
"""
self.logger.debug("serializing list: " + str(list_to_serialize))
return str(list_to_serialize)\
.replace("[", "")\
.replace("]", "")\
.replace(" ", "")\
.replace("'", separator)
def list_ldap_groups(self):
"""
Display LDAP groups
"""
self.logger.debug("Displaying LDAP groups")
groups = []
for user in self.ldap_users:
for group in self.ldap_users[user]:
if group not in groups:
groups.append(group)
self.logger.info(str(sorted(groups)))
def create_groups_secrets(self):
"""
Create a secret folder for each LDAP group under specified path
"""
self.logger.debug("Creating groups secrets under %s" %
self.parsed_args.create_groups_secrets)
existing_folders = self.vault_client.list(
self.parsed_args.create_groups_secrets)
if len(existing_folders):
existing_folders = [
e.replace("/", "") for e in existing_folders['keys']
]
self.logger.debug("Already existing folders: " + str(existing_folders))
for group in self.conf["groups"]["groups_to_add"]:
if group not in existing_folders:
self.logger.info("Creating folder: " + group)
self.vault_client.write(
self.parsed_args.create_groups_secrets + "/" + group +
"/description", {group: "group private secrets space"})
for group in existing_folders:
if group not in self.conf["groups"]["groups_to_add"]:
tree = self.vault_client.get_secrets_tree(
self.parsed_args.create_groups_secrets + "/" + group)
self.logger.info("Deleting folder " + group +
" and associated secrets " + str(tree))
for secret in tree:
self.vault_client.delete(secret)
def create_users_secrets(self):
"""
Create a secret folder for each LDAP user under specified path
"""
self.logger.debug("Creating users secrets under %s" %
self.parsed_args.create_users_secrets)
enabled_users = []
for user in self.ldap_users:
groups_of_user = list(
set(self.conf["groups"]["groups_to_add"]).intersection(
self.ldap_users[user]))
if len(groups_of_user):
enabled_users.append(user)
existing_folders = self.vault_client.list(
self.parsed_args.create_users_secrets)
if len(existing_folders):
existing_folders = [
e.replace("/", "") for e in existing_folders['keys']
]
self.logger.debug("Already existing folders: " + str(existing_folders))
for user in enabled_users:
if user not in existing_folders:
self.logger.info("Creating folder: " + user)
self.vault_client.write(
self.parsed_args.create_users_secrets + "/" + user +
"/description", {user: "******"})
for user in existing_folders:
if user not in enabled_users:
tree = self.vault_client.get_secrets_tree(
self.parsed_args.create_users_secrets + "/" + user)
self.logger.info("Deleting folder " + user +
" and associated secrets " + str(tree))
for secret in tree:
self.vault_client.delete(secret)
def run(self, arg_parser, parsed_args):
"""
Module entry point
:param arg_parser: Arguments parser instance
:param parsed_args: Arguments parsed fir this module
:type parsed_args: argparse.ArgumentParser.parse_args()
"""
self.parsed_args = parsed_args
self.arg_parser = arg_parser
self.logger.debug("Module " + self.module_name + " started")
if not self.check_args_integrity():
self.subparser.print_help()
return False
if not self.check_env_vars():
return False
self.policies_folder = os.path.join(os.environ["VAULT_CONFIG"],
"policies")
self.user_policies_folder = os.path.join(self.policies_folder, "user")
self.group_policies_folder = os.path.join(self.policies_folder,
"group")
self.group_policies_to_create = []
self.user_policies_to_create = []
if not self.read_configuration() or not self.read_ldap_configuration():
return False
self.vault_client = VaultClient(self.base_logger,
dry=self.parsed_args.dry_run,
skip_tls=self.parsed_args.skip_tls)
self.vault_client.authenticate()
if not self.get_ldap_data():
return False
if self.parsed_args.list_groups:
self.list_ldap_groups()
return True
if self.parsed_args.create_policies:
self.logger.info("Creating LDAP policies")
self.create_groups_policies()
self.create_users_policies()
self.deleting_previous_policies()
if self.parsed_args.manage_ldap_groups:
self.logger.info("Managing groups in Vault LDAP '%s' config" %
self.parsed_args.manage_ldap_groups)
self.manage_groups_in_vault_ldap_conf()
if self.parsed_args.manage_ldap_users:
self.logger.info("Managing users in Vault LDAP '%s' config" %
self.parsed_args.manage_ldap_users)
self.manage_users_in_vault_ldap_conf()
if self.parsed_args.create_groups_secrets:
self.logger.info(
"Creating groups folders under secret path '/%s'" %
self.parsed_args.create_groups_secrets)
self.create_groups_secrets()
if self.parsed_args.create_users_secrets:
self.logger.info("Creating users folders under secret path '/%s'" %
self.parsed_args.create_users_secrets)
self.create_users_secrets()
class VaultManagerAudit:
"""
Audit Module
"""
logger = None
base_logger = None
subparser = None
parsed_args = None
arg_parser = None
module_name = None
conf = None
vault_client = None
distant_audit_devices = None
local_audit_devices = None
def __init__(self, base_logger=None):
"""
:param base_logger: main class name
:type base_logger: string
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
"""
self.base_logger = base_logger
if base_logger:
self.logger = logging.getLogger(base_logger + "." +
self.__class__.__name__)
else:
self.logger = logging.getLogger()
self.logger.debug("Initializing VaultManagerAudit")
def initialize_subparser(self, subparsers):
"""
Add the subparser of this specific module to the list of all subparsers
:param subparsers: list of all subparsers
:type subparsers: argparse.ArgumentParser.add_subparsers()
:return:
"""
self.logger.debug("Initializing subparser")
self.module_name = \
self.__class__.__name__.replace("VaultManager", "").lower()
self.subparser = subparsers.add_parser(self.module_name,
help=self.module_name +
' management')
self.subparser.add_argument("--push",
action='store_true',
help="Push audit configuration to Vault")
self.subparser.set_defaults(module_name=self.module_name)
def read_configuration(self):
"""
Read configuration file
"""
self.logger.debug("Reading configuration")
with open(os.path.join(self.kwargs.vault_config, "audit-devices.yml"),
'r') as fd:
try:
self.conf = yaml.safe_load(fd)
except yaml.YAMLError as e:
self.logger.critical("Impossible to load conf file: " + str(e))
return False
self.logger.debug("Read conf: " + str(self.conf))
return True
def get_distant_audit_devices(self):
"""
Fetch distant audit devices
"""
self.logger.debug("Fetching distant audit devices")
self.distant_audit_devices = []
raw = self.vault_client.audit_list()
for elem in raw:
self.distant_audit_devices.append(
VaultAuditDevice(raw[elem]["type"], raw[elem]["path"],
raw[elem]["description"],
raw[elem]["options"]))
self.logger.debug("Distant audit devices found")
for elem in self.distant_audit_devices:
self.logger.debug(elem)
def get_local_audit_devices(self):
"""
Fetch local audit devices
"""
self.logger.debug("Fetching local audit devices")
self.local_audit_devices = []
for audit_device in self.conf["audit-devices"]:
self.local_audit_devices.append(
VaultAuditDevice(audit_device["type"], audit_device["path"],
audit_device["description"],
audit_device["options"]))
self.logger.debug("Local audit devices found")
for elem in self.local_audit_devices:
self.logger.debug(elem)
def disable_distant_audit_devices(self):
"""
Disable audit devices not found in conf
"""
self.logger.debug("Disabling audit devices")
for audit_device in self.distant_audit_devices:
if audit_device not in self.local_audit_devices:
self.logger.info("Disabling: " + str(audit_device))
self.vault_client.audit_disable(audit_device.path)
def enable_distant_audit_devices(self):
"""
Enable audit devices found in conf
"""
self.logger.debug("Enabling audit devices")
for audit_device in self.local_audit_devices:
if audit_device not in self.distant_audit_devices:
self.logger.info("Enabling: " + str(audit_device))
self.vault_client.audit_enable(audit_device.type,
audit_device.path,
audit_device.description,
audit_device.options)
def check_args_integrity(self):
"""
Checking provided arguments integrity
"""
self.logger.debug("Checking arguments integrity")
args_false_count = [self.kwargs.push].count(False)
args_none_count = [self.kwargs.push].count(None)
no_args_count = args_false_count + args_none_count
if no_args_count in [1]:
self.logger.critical("you must specify a command")
return False
return True
def audit_push(self):
"""
Push secrets engines configuration to Vault
"""
self.logger.info("Pushing audit devices configuration to Vault")
self.read_configuration()
self.get_distant_audit_devices()
self.get_local_audit_devices()
for audit_device in self.local_audit_devices:
if audit_device in self.distant_audit_devices:
self.logger.info("Audit device remaining unchanged " +
str(audit_device))
if "audit-devices-deletion" in self.conf and \
self.conf["audit-devices-deletion"]:
self.disable_distant_audit_devices()
self.enable_distant_audit_devices()
self.logger.info("Audit devices successfully pushed to Vault")
def run(self, kwargs):
"""
Module entry point
:param kwargs: Arguments parsed
:type kwargs: dict
"""
# Convert kwargs to an Object with kwargs dict as class vars
self.kwargs = namedtuple("KwArgs", kwargs.keys())(*kwargs.values())
self.logger.debug("Module " + self.module_name + " started")
if not self.check_args_integrity():
self.subparser.print_help()
return False
missing_args = utils.keys_exists_in_dict(self.logger,
dict(self.kwargs._asdict()),
[{
"key": "vault_addr",
"exc": [None, '']
}, {
"key": "vault_token",
"exc": [None, False]
}, {
"key": "vault_config",
"exc": [None, False, '']
}])
if len(missing_args):
raise ValueError(
"Following arguments are missing %s\n" %
[k['key'].replace("_", "-") for k in missing_args])
self.vault_client = VaultClient(self.base_logger,
vault_addr=self.kwargs.vault_addr,
dry=self.kwargs.dry_run,
skip_tls=self.kwargs.skip_tls)
self.vault_client.authenticate()
if self.kwargs.push:
self.audit_push()