def upload_documents(self):
"""Uploads documents to Shipyard"""
collected_documents = files.collect_files_by_repo(self.site_name)
collection_data = [site.get_deployment_data_doc()]
LOG.info("Processing %d collection(s)", len(collected_documents))
for idx, document in enumerate(collected_documents):
# Decrypt the documents if encrypted
pegleg_secret_mgmt = PeglegSecretManagement(
docs=collected_documents[document])
decrypted_documents = pegleg_secret_mgmt.get_decrypted_secrets()
collection_data.extend(decrypted_documents)
add_representer_ordered_dict()
collection_as_yaml = yaml.dump_all(
collection_data, Dumper=yaml.SafeDumper)
# Append flag is not required for the first
# collection being uploaded to Shipyard. It
# is needed for subsequent collections.
if self.buffer_mode in ['append', 'replace']:
buffer_mode = self.buffer_mode
else:
raise exceptions.InvalidBufferModeException()
try:
self.validate_auth_vars()
resp_text = self.api_client.post_configdocs(
collection_id=self.collection,
buffer_mode=buffer_mode,
document_data=collection_as_yaml)
except AuthValuesError as ave:
resp_text = "Error: {}".format(ave.diagnostic)
raise DocumentUploadError(resp_text)
except Exception as ex:
resp_text = (
"Error: Unable to invoke action due to: {}".format(str(ex)))
LOG.debug(resp_text, exc_info=True)
raise DocumentUploadError(resp_text)
# FIXME: Standardize status_code in Deckhand to avoid this
# workaround.
code = 0
if hasattr(resp_text, 'status_code'):
code = resp_text.status_code
elif hasattr(resp_text, 'code'):
code = resp_text.code
if code >= 400:
if hasattr(resp_text, 'content'):
raise DocumentUploadError(resp_text.content)
else:
raise DocumentUploadError(resp_text)
else:
output = self.formatted_response_handler(resp_text)
LOG.info("Uploaded document in buffer %s ", output)
# Commit in the last iteration of the loop when all the documents
# have been pushed to Shipyard buffer.
return self.commit_documents()
def decrypt(path, site_name=None):
"""Decrypt one secrets file, and print the decrypted file to standard out.
Search the specified file_path for a file.
If the file is found and encrypted, unwrap and decrypt it, and print the
result to standard out.
If the file is found, but it is not encrypted, print the contents of the
file to standard out.
Passphrase and salt for the decryption are read from environment variables.
:param path: Path to the file to be unwrapped and decrypted.
:type path: string
:return: The decrypted secrets
:rtype: dict
"""
LOG.info('Started decrypting...')
file_dict = {}
if not os.path.exists(path):
LOG.error('Path: {} was not found. Check your path and site name, '
'and try again.'.format(path))
return file_dict
if os.path.isfile(path):
file_dict[path] = PeglegSecretManagement(
path, site_name=site_name).decrypt_secrets()
else:
match = os.path.join(path, '**', '*.yaml')
file_list = glob(match, recursive=True)
if not file_list:
LOG.warning(
'No YAML files were discovered in path: {}'.format(path))
for file_path in file_list:
file_dict[file_path] = PeglegSecretManagement(
file_path).decrypt_secrets()
return file_dict
def build_genesis(build_path, encryption_key, validators, debug, site_name):
"""
Build the genesis deployment bundle, and store it in ``build_path``.
Build the genesis.sh script, base65-encode, encrypt and embed the
site configuration source documents in genesis.sh script.
If ``validators`` flag should be True, build the bundle validator
scripts as well.
Store the built deployment bundle in `build_path`.
:param str build_path: Directory path of the built genesis deployment
bundle
:param str encryption_key: Key to use to encrypt the bundled site
configuration in genesis.sh script.
:param bool validators: Whether to generate validator scripts
:param int debug: pegleg debug level to pass to promenade engine
for logging.
:return: None
"""
# Raise an error if the build path exists. We don't want to overwrite it.
if os.path.isdir(build_path):
raise click.ClickException(
"{} already exists, remove it or specify a new "
"directory.".format(build_path))
# Get the list of config files
LOG.info('=== Building bootstrap scripts ===')
# Copy the site config, and site secrets to build directory
os.mkdir(build_path)
documents = util.definition.documents_for_site(site_name)
secret_manager = PeglegSecretManagement(docs=documents)
documents = secret_manager.get_decrypted_secrets()
try:
# Use the promenade engine to build and encrypt the genesis bundle
c = Configuration(documents=documents,
debug=debug,
substitute=True,
allow_missing_substitutions=False,
leave_kubectl=False)
# Both a policy and key are present, generate bundle with encryption
if c.get_path('EncryptionPolicy:scripts.genesis') and encryption_key:
Builder(c, validators=validators).build_all(output_dir=build_path)
# A policy or key are present but not both, raise an error
elif c.get_path('EncryptionPolicy:scripts.genesis') or encryption_key:
raise GenesisBundleEncryptionException()
# Neither policy or key are present, generate bundle without encryption
else:
Builder(c, validators=validators).build_all(output_dir=build_path)
except exceptions.PromenadeException as e:
LOG.error('Build genesis bundle failed! {}.'.format(
e.display(debug=debug)))
raise GenesisBundleGenerateException()
LOG.info('=== Done! ===')
def test_check_expiry(temp_deployment_files):
""" Validates check_expiry """
repo_path = str(
git.git_handler(TEST_PARAMS["repo_url"], ref=TEST_PARAMS["repo_rev"]))
with mock.patch.dict(config.GLOBAL_CONTEXT, {"site_repo": repo_path}):
pki_generator = PKIGenerator(duration=365,
sitename=TEST_PARAMS["site_name"])
generated_files = pki_generator.generate()
pki_util = pki_utility.PKIUtility(duration=0)
assert len(generated_files), 'No secrets were generated'
for generated_file in generated_files:
if "certificate" not in generated_file:
continue
with open(generated_file, 'r') as f:
results = yaml.safe_load_all(f) # Validate valid YAML.
results = PeglegSecretManagement(
docs=results).get_decrypted_secrets()
for result in results:
if result['schema'] == \
"deckhand/Certificate/v1":
cert = result['data']
cert_info = pki_util.check_expiry(cert)
assert cert_info['expired'] is False, \
"%s is expired/expiring on %s" % \
(result['metadata']['name'],
cert_info['expiry_date'])
def wrap_secret(author,
filename,
output_path,
schema,
name,
layer,
encrypt,
site_name=None):
"""Wrap a bare secrets file in a YAML and ManagedDocument.
:param author: author for ManagedDocument
:param filename: file path for input file
:param output_path: file path for output file
:param schema: schema for wrapped document
:param name: name for wrapped document
:param layer: layer for wrapped document
:param encrypt: whether to encrypt the output doc
"""
if not output_path:
output_path = os.path.splitext(filename)[0] + ".yaml"
with open(filename, 'r') as in_fi:
data = in_fi.read()
inner_doc = OrderedDict([
("schema", schema), ("data", data),
("metadata",
OrderedDict([("layeringDefinition",
OrderedDict([("abstract", False), ("layer", layer)])),
("name", name), ("schema", "metadata/Document/v1"),
("storagePolicy",
"encrypted" if encrypt else "cleartext")]))
])
managed_secret = PeglegManagedSecret(inner_doc, author=author)
if encrypt:
psm = PeglegSecretManagement(docs=[inner_doc],
author=author,
site_name=site_name)
output_doc = psm.get_encrypted_secrets()[0][0]
else:
output_doc = managed_secret.pegleg_document
files.safe_dump(output_doc,
output_path,
sort_keys=False,
explicit_start=True,
explicit_end=True)
def test_encrypt_decrypt_using_docs(tmpdir):
# write the test data to temp file
test_data = list(yaml.safe_load_all(TEST_DATA))
save_path = os.path.join(tmpdir, 'encrypted_secrets_file.yaml')
# encrypt documents and validate that they were encrypted
doc_mgr = PeglegSecretManagement(docs=test_data, author='test_author')
doc_mgr.encrypt_secrets(save_path)
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'test_author'
# read back the encrypted file
with open(save_path) as stream:
encrypted_data = list(yaml.safe_load_all(stream))
# decrypt documents and validate that they were decrypted
doc_mgr = PeglegSecretManagement(docs=encrypted_data, author='test_author')
decrypted_data = doc_mgr.get_decrypted_secrets()
assert test_data[0]['data'] == decrypted_data[0]['data']
assert test_data[0]['schema'] == decrypted_data[0]['schema']
assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][
'name']
assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][
'metadata']['storagePolicy']
def test_pegleg_secret_management_constructor_with_invalid_arguments():
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(file_path=None, docs=None)
assert 'Either `file_path` or `docs` must be specified.' in str(
err_info.value)
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(file_path='file_path', docs=['doc1'])
assert 'Either `file_path` or `docs` must be specified.' in str(
err_info.value)
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(file_path='file_path',
generated=True,
author='test_author')
assert 'If the document is generated, author and catalog must be ' \
'specified.' in str(err_info.value)
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(docs=['doc'], generated=True)
assert 'If the document is generated, author and catalog must be ' \
'specified.' in str(err_info.value)
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(docs=['doc'],
generated=True,
author='test_author')
assert 'If the document is generated, author and catalog must be ' \
'specified.' in str(err_info.value)
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(docs=['doc'], generated=True, catalog='catalog')
assert 'If the document is generated, author and catalog must be ' \
'specified.' in str(err_info.value)
def _get_or_gen(self, generator, kinds, document_name, *args, **kwargs):
docs = self._find_docs(kinds, document_name)
if not docs or self._regenerate_all:
docs = generator(document_name, *args, **kwargs)
else:
docs = PeglegSecretManagement(docs=docs)
# Adding these to output should be idempotent, so we use a dict.
for wrapper_doc in docs:
wrapped_doc = wrapper_doc['data']['managedDocument']
schema = wrapped_doc['schema']
name = wrapped_doc['metadata']['name']
self.outputs[schema][name] = wrapper_doc
return docs
def _write(self, output_dir):
documents = self.get_documents()
output_paths = set()
# First, delete each of the output paths below because we do an append
# action in the `open` call below. This means that for regeneration
# of certs, the original paths must be deleted.
for document in documents:
output_file_path = md.get_document_path(
sitename=self._sitename,
wrapper_document=document,
cert_to_ca_map=self._cert_to_ca_map)
output_path = os.path.join(output_dir, 'site', output_file_path)
# NOTE(felipemonteiro): This is currently an entirely safe
# operation as these files are being removed in the temporarily
# replicated versions of the local repositories.
if os.path.exists(output_path):
os.remove(output_path)
# Next, generate (or regenerate) the certificates.
for document in documents:
output_file_path = md.get_document_path(
sitename=self._sitename,
wrapper_document=document,
cert_to_ca_map=self._cert_to_ca_map)
output_path = os.path.join(output_dir, 'site', output_file_path)
dir_name = os.path.dirname(output_path)
if not os.path.exists(dir_name):
LOG.debug('Creating secrets path: %s', dir_name)
os.makedirs(os.path.abspath(dir_name))
# Encrypt the document
document['data']['managedDocument']['metadata']['storagePolicy']\
= 'encrypted'
document = PeglegSecretManagement(
docs=[document]).get_encrypted_secrets()[0][0]
util.files.dump(document,
output_path,
flag='a',
default_flow_style=False,
explicit_start=True,
indent=2)
output_paths.add(output_path)
return output_paths
def check_cert_expiry(site_name, duration=60):
"""
Check certs from a sites PKICatalog to determine if they are expired or
expiring within N days
:param str site_name: The site to read from
:param int duration: Number of days from today to check cert
expirations
:rtype: str
"""
cert_schemas = [
'deckhand/Certificate/v1', 'deckhand/CertificateAuthority/v1'
]
pki_util = PKIUtility(duration=duration)
# Create a table to output expired/expiring certs for this site.
cert_table = PrettyTable()
cert_table.field_names = ['file', 'cert_name', 'expiration_date']
expired_certs_exist = False
s = definition.site_files(site_name)
for doc in s:
if 'certificate' in doc:
with open(doc, 'r') as f:
results = yaml.safe_load_all(f) # Validate valid YAML.
results = PeglegSecretManagement(
docs=results).get_decrypted_secrets()
for result in results:
if result['schema'] in cert_schemas:
text = result['data']
header_pattern = '-----BEGIN CERTIFICATE-----'
footer_pattern = '-----END CERTIFICATE-----'
find_pattern = r'%s.*?%s' % (header_pattern,
footer_pattern)
certs = re.findall(find_pattern, text, re.DOTALL)
for cert in certs:
cert_info = pki_util.check_expiry(cert)
if cert_info['expired'] is True:
cert_table.add_row([
doc, result['metadata']['name'],
cert_info['expiry_date']
])
expired_certs_exist = True
# Return table of cert names and expiration dates that are expiring
return expired_certs_exist, cert_table.get_string()
def test_global_encrypt_decrypt(temp_deployment_files, tmpdir):
# Create site files
site_dir = tmpdir.join("deployment_files", "site", "cicd")
# Create and encrypt global passphrase and salt file using site keys
with open(os.path.join(str(site_dir), 'secrets',
'passphrases',
'cicd-global-passphrase-encrypted.yaml'), "w") \
as outfile:
outfile.write(GLOBAL_PASSPHRASE_SALT_DOC)
save_location = tmpdir.mkdir("encrypted_site_files")
save_location_str = str(save_location)
# Encrypt the global passphrase and salt file using site passphrase/salt
config.set_global_enc_keys("cicd")
secrets.encrypt(save_location_str, "pytest", "cicd")
# Create and encrypt a global type document
global_doc_path = os.path.join(str(site_dir), 'secrets', 'passphrases',
'globally_encrypted_doc.yaml')
with open(global_doc_path, "w") as outfile:
outfile.write(TEST_GLOBAL_DATA)
# encrypt documents and validate that they were encrypted
doc_mgr = PeglegSecretManagement(file_path=global_doc_path,
author='pytest',
site_name='cicd')
doc_mgr.encrypt_secrets(global_doc_path)
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'pytest'
doc_mgr = PeglegSecretManagement(file_path=global_doc_path,
author='pytest',
site_name='cicd')
decrypted_data = doc_mgr.get_decrypted_secrets()
test_data = list(yaml.safe_load_all(TEST_GLOBAL_DATA))
assert test_data[0]['data'] == decrypted_data[0]['data']
assert test_data[0]['schema'] == decrypted_data[0]['schema']
def encrypt(save_location, author, site_name, path=None):
"""
Encrypt all secrets documents for a site identifies by site_name.
Parse through all documents related to ``site_name`` and encrypt all
site documents, which have metadata.storagePolicy: encrypted, and
are not already encrypted and wrapped in a PeglegManagedDocument.
``Passphrase`` and ``salt`` for the encryption are read from environment
variables``$PEGLEG_PASSPHRASE`` and ``$PEGLEG_SALT`` respectively.
By default, the resulting output files will overwrite the original
unencrypted secrets documents.
:param str save_location: if provided, is used as the base directory to
store the encrypted secrets files. If not provided, the encrypted
secrets files will overwrite the original unencrypted files (default
behavior).
:param str author: Identifies the individual or application, who
encrypts the secrets documents.
:param str site_name: The name of the site to encrypt its secrets files.
:param str path: The path to the directory or file to be encrypted.
"""
files.check_file_save_location(save_location)
LOG.debug('Save location is %s', save_location)
file_sets = []
path_exists = path and os.path.exists(path)
if path_exists:
if os.path.isfile(path):
LOG.debug('Specified path is a file')
file_sets = [(None, path)]
elif os.path.isdir(path):
LOG.debug('Specified path is a directory')
file_sets = []
for filename in glob(os.path.join(path, '**/*.yaml'),
recursive=True):
LOG.debug('Discovered %s', filename)
file_sets.append((None, filename))
else:
LOG.debug('No path specified, searching all repos')
file_sets = list(definition.site_files_by_repo(site_name))
LOG.info('Started encrypting...')
secrets_found = False
for repo_base, file_path in file_sets:
LOG.debug('Looking at %s in %s repo', file_path, repo_base)
secrets_found = True
secret = PeglegSecretManagement(file_path=file_path,
author=author,
site_name=site_name)
if path_exists:
if save_location:
output_path = os.path.join(save_location.rstrip(os.path.sep),
file_path.lstrip(os.path.sep))
else:
output_path = file_path
else:
output_path = _get_dest_path(repo_base, file_path, save_location)
LOG.debug('Outputting encrypted data to %s', output_path)
secret.encrypt_secrets(output_path)
if secrets_found:
LOG.info('Encryption of all secret files was completed.')
else:
LOG.warning(
'No secret documents were found for site: {}'.format(site_name))
def test_encrypt_decrypt_using_file_path(tmpdir):
# write the test data to temp file
test_data = list(yaml.safe_load_all(TEST_DATA))
file_path = os.path.join(tmpdir, 'secrets_file.yaml')
files.write(test_data, file_path)
save_path = os.path.join(tmpdir, 'encrypted_secrets_file.yaml')
# encrypt documents and validate that they were encrypted
doc_mgr = PeglegSecretManagement(file_path=file_path, author='test_author')
doc_mgr.encrypt_secrets(save_path)
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'test_author'
# decrypt documents and validate that they were decrypted
doc_mgr = PeglegSecretManagement(file_path=file_path, author='test_author')
doc_mgr.encrypt_secrets(save_path)
# read back the encrypted file
doc_mgr = PeglegSecretManagement(file_path=save_path, author='test_author')
decrypted_data = doc_mgr.get_decrypted_secrets()
assert test_data[0]['data'] == decrypted_data[0]['data']
assert test_data[0]['schema'] == decrypted_data[0]['schema']
def test_pegleg_secret_management_double_encrypt():
encrypted_doc = PeglegSecretManagement(
docs=[yaml.safe_load(TEST_DATA)]).get_encrypted_secrets()[0][0]
encrypted_doc_2 = PeglegSecretManagement(
docs=[encrypted_doc]).get_encrypted_secrets()[0][0]
assert encrypted_doc == encrypted_doc_2
def test_short_salt():
with pytest.raises(exceptions.SaltInsufficientLengthException):
PeglegSecretManagement(file_path='file_path', author='test_author')
def generate(self, interactive=False, force_cleartext=False):
"""
For each passphrase entry in the passphrase catalog, generate a
random passphrase string, based on a passphrase specification in the
catalog. Create a pegleg managed document, wrap the generated
passphrase document in the pegleg managed document, and encrypt the
passphrase. Write the wrapped and encrypted document in a file at
<repo_name>/site/<site_name>/secrets/passphrases/passphrase_name.yaml.
:param bool interactive: If true, allow input
:param bool force_cleartext: If true, don't encrypt
"""
for p_name in self._catalog.get_passphrase_names:
# Check if this secret is present and should not be regenerated
save_path = self.get_save_path(p_name)
regenerable = self._catalog.is_passphrase_regenerable(p_name)
if os.path.exists(save_path) and not regenerable:
continue
# Generate secret as it either does not exist yet or is a
# regenerable secret and does exist but should be rotated.
passphrase = None
passphrase_type = self._catalog.get_passphrase_type(p_name)
prompt = self._catalog.is_passphrase_prompt(p_name)
profile = self._catalog.get_passphrase_profile(p_name)
if interactive and prompt:
auto_allowed = regenerable
if passphrase_type == 'uuid': # nosec
passphrase = self._prompt_user_passphrase_and_validate(
p_name,
'UUID',
self.validate_uuid,
auto_allowed=auto_allowed)
elif passphrase_type == 'base64': # nosec
passphrase = self._prompt_user_passphrase_and_validate(
p_name,
'passphrase (b64)',
self.validate_base64,
auto_allowed=auto_allowed)
elif passphrase_type == 'passphrase':
passphrase = self._prompt_user_passphrase_and_validate(
p_name,
'passphrase',
self.validate_passphrase,
auto_allowed=auto_allowed)
elif not interactive and prompt:
LOG.debug('Skipping interactive input for %s', p_name)
continue
if not passphrase:
if passphrase_type == 'uuid': # nosec
passphrase = uuidutils.generate_uuid()
else:
passphrase = CryptoString(profile).get_crypto_string(
self._catalog.get_length(p_name))
if passphrase_type == 'base64': # nosec
# Take the randomly generated string and convert to a
# random base64 string
passphrase = passphrase.encode()
passphrase = base64.b64encode(passphrase).decode()
docs = list()
if force_cleartext:
storage_policy = passphrase_catalog.P_CLEARTEXT
LOG.warning("Passphrases for {} will be "
"generated in clear text.".format(p_name))
else:
storage_policy = self._catalog.get_storage_policy(p_name)
docs.append(
self.generate_doc(KIND, p_name, storage_policy, passphrase))
if storage_policy == passphrase_catalog.P_ENCRYPTED:
PeglegSecretManagement(
docs=docs,
generated=True,
author=self._author,
catalog=self._catalog).encrypt_secrets(save_path)
else:
files.write(docs, save_path)