def upload_documents(self): """Uploads documents to Shipyard""" collected_documents = files.collect_files_by_repo(self.site_name) collection_data = [site.get_deployment_data_doc()] LOG.info("Processing %d collection(s)", len(collected_documents)) for idx, document in enumerate(collected_documents): # Decrypt the documents if encrypted pegleg_secret_mgmt = PeglegSecretManagement( docs=collected_documents[document]) decrypted_documents = pegleg_secret_mgmt.get_decrypted_secrets() collection_data.extend(decrypted_documents) add_representer_ordered_dict() collection_as_yaml = yaml.dump_all( collection_data, Dumper=yaml.SafeDumper) # Append flag is not required for the first # collection being uploaded to Shipyard. It # is needed for subsequent collections. if self.buffer_mode in ['append', 'replace']: buffer_mode = self.buffer_mode else: raise exceptions.InvalidBufferModeException() try: self.validate_auth_vars() resp_text = self.api_client.post_configdocs( collection_id=self.collection, buffer_mode=buffer_mode, document_data=collection_as_yaml) except AuthValuesError as ave: resp_text = "Error: {}".format(ave.diagnostic) raise DocumentUploadError(resp_text) except Exception as ex: resp_text = ( "Error: Unable to invoke action due to: {}".format(str(ex))) LOG.debug(resp_text, exc_info=True) raise DocumentUploadError(resp_text) # FIXME: Standardize status_code in Deckhand to avoid this # workaround. code = 0 if hasattr(resp_text, 'status_code'): code = resp_text.status_code elif hasattr(resp_text, 'code'): code = resp_text.code if code >= 400: if hasattr(resp_text, 'content'): raise DocumentUploadError(resp_text.content) else: raise DocumentUploadError(resp_text) else: output = self.formatted_response_handler(resp_text) LOG.info("Uploaded document in buffer %s ", output) # Commit in the last iteration of the loop when all the documents # have been pushed to Shipyard buffer. return self.commit_documents()
def decrypt(path, site_name=None): """Decrypt one secrets file, and print the decrypted file to standard out. Search the specified file_path for a file. If the file is found and encrypted, unwrap and decrypt it, and print the result to standard out. If the file is found, but it is not encrypted, print the contents of the file to standard out. Passphrase and salt for the decryption are read from environment variables. :param path: Path to the file to be unwrapped and decrypted. :type path: string :return: The decrypted secrets :rtype: dict """ LOG.info('Started decrypting...') file_dict = {} if not os.path.exists(path): LOG.error('Path: {} was not found. Check your path and site name, ' 'and try again.'.format(path)) return file_dict if os.path.isfile(path): file_dict[path] = PeglegSecretManagement( path, site_name=site_name).decrypt_secrets() else: match = os.path.join(path, '**', '*.yaml') file_list = glob(match, recursive=True) if not file_list: LOG.warning( 'No YAML files were discovered in path: {}'.format(path)) for file_path in file_list: file_dict[file_path] = PeglegSecretManagement( file_path).decrypt_secrets() return file_dict
def build_genesis(build_path, encryption_key, validators, debug, site_name): """ Build the genesis deployment bundle, and store it in ``build_path``. Build the genesis.sh script, base65-encode, encrypt and embed the site configuration source documents in genesis.sh script. If ``validators`` flag should be True, build the bundle validator scripts as well. Store the built deployment bundle in `build_path`. :param str build_path: Directory path of the built genesis deployment bundle :param str encryption_key: Key to use to encrypt the bundled site configuration in genesis.sh script. :param bool validators: Whether to generate validator scripts :param int debug: pegleg debug level to pass to promenade engine for logging. :return: None """ # Raise an error if the build path exists. We don't want to overwrite it. if os.path.isdir(build_path): raise click.ClickException( "{} already exists, remove it or specify a new " "directory.".format(build_path)) # Get the list of config files LOG.info('=== Building bootstrap scripts ===') # Copy the site config, and site secrets to build directory os.mkdir(build_path) documents = util.definition.documents_for_site(site_name) secret_manager = PeglegSecretManagement(docs=documents) documents = secret_manager.get_decrypted_secrets() try: # Use the promenade engine to build and encrypt the genesis bundle c = Configuration(documents=documents, debug=debug, substitute=True, allow_missing_substitutions=False, leave_kubectl=False) # Both a policy and key are present, generate bundle with encryption if c.get_path('EncryptionPolicy:scripts.genesis') and encryption_key: Builder(c, validators=validators).build_all(output_dir=build_path) # A policy or key are present but not both, raise an error elif c.get_path('EncryptionPolicy:scripts.genesis') or encryption_key: raise GenesisBundleEncryptionException() # Neither policy or key are present, generate bundle without encryption else: Builder(c, validators=validators).build_all(output_dir=build_path) except exceptions.PromenadeException as e: LOG.error('Build genesis bundle failed! {}.'.format( e.display(debug=debug))) raise GenesisBundleGenerateException() LOG.info('=== Done! ===')
def test_check_expiry(temp_deployment_files): """ Validates check_expiry """ repo_path = str( git.git_handler(TEST_PARAMS["repo_url"], ref=TEST_PARAMS["repo_rev"])) with mock.patch.dict(config.GLOBAL_CONTEXT, {"site_repo": repo_path}): pki_generator = PKIGenerator(duration=365, sitename=TEST_PARAMS["site_name"]) generated_files = pki_generator.generate() pki_util = pki_utility.PKIUtility(duration=0) assert len(generated_files), 'No secrets were generated' for generated_file in generated_files: if "certificate" not in generated_file: continue with open(generated_file, 'r') as f: results = yaml.safe_load_all(f) # Validate valid YAML. results = PeglegSecretManagement( docs=results).get_decrypted_secrets() for result in results: if result['schema'] == \ "deckhand/Certificate/v1": cert = result['data'] cert_info = pki_util.check_expiry(cert) assert cert_info['expired'] is False, \ "%s is expired/expiring on %s" % \ (result['metadata']['name'], cert_info['expiry_date'])
def wrap_secret(author, filename, output_path, schema, name, layer, encrypt, site_name=None): """Wrap a bare secrets file in a YAML and ManagedDocument. :param author: author for ManagedDocument :param filename: file path for input file :param output_path: file path for output file :param schema: schema for wrapped document :param name: name for wrapped document :param layer: layer for wrapped document :param encrypt: whether to encrypt the output doc """ if not output_path: output_path = os.path.splitext(filename)[0] + ".yaml" with open(filename, 'r') as in_fi: data = in_fi.read() inner_doc = OrderedDict([ ("schema", schema), ("data", data), ("metadata", OrderedDict([("layeringDefinition", OrderedDict([("abstract", False), ("layer", layer)])), ("name", name), ("schema", "metadata/Document/v1"), ("storagePolicy", "encrypted" if encrypt else "cleartext")])) ]) managed_secret = PeglegManagedSecret(inner_doc, author=author) if encrypt: psm = PeglegSecretManagement(docs=[inner_doc], author=author, site_name=site_name) output_doc = psm.get_encrypted_secrets()[0][0] else: output_doc = managed_secret.pegleg_document files.safe_dump(output_doc, output_path, sort_keys=False, explicit_start=True, explicit_end=True)
def test_encrypt_decrypt_using_docs(tmpdir): # write the test data to temp file test_data = list(yaml.safe_load_all(TEST_DATA)) save_path = os.path.join(tmpdir, 'encrypted_secrets_file.yaml') # encrypt documents and validate that they were encrypted doc_mgr = PeglegSecretManagement(docs=test_data, author='test_author') doc_mgr.encrypt_secrets(save_path) doc = doc_mgr.documents[0] assert doc.is_encrypted() assert doc.data['encrypted']['by'] == 'test_author' # read back the encrypted file with open(save_path) as stream: encrypted_data = list(yaml.safe_load_all(stream)) # decrypt documents and validate that they were decrypted doc_mgr = PeglegSecretManagement(docs=encrypted_data, author='test_author') decrypted_data = doc_mgr.get_decrypted_secrets() assert test_data[0]['data'] == decrypted_data[0]['data'] assert test_data[0]['schema'] == decrypted_data[0]['schema'] assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][ 'name'] assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][ 'metadata']['storagePolicy']
def test_pegleg_secret_management_constructor_with_invalid_arguments(): with pytest.raises(ValueError) as err_info: PeglegSecretManagement(file_path=None, docs=None) assert 'Either `file_path` or `docs` must be specified.' in str( err_info.value) with pytest.raises(ValueError) as err_info: PeglegSecretManagement(file_path='file_path', docs=['doc1']) assert 'Either `file_path` or `docs` must be specified.' in str( err_info.value) with pytest.raises(ValueError) as err_info: PeglegSecretManagement(file_path='file_path', generated=True, author='test_author') assert 'If the document is generated, author and catalog must be ' \ 'specified.' in str(err_info.value) with pytest.raises(ValueError) as err_info: PeglegSecretManagement(docs=['doc'], generated=True) assert 'If the document is generated, author and catalog must be ' \ 'specified.' in str(err_info.value) with pytest.raises(ValueError) as err_info: PeglegSecretManagement(docs=['doc'], generated=True, author='test_author') assert 'If the document is generated, author and catalog must be ' \ 'specified.' in str(err_info.value) with pytest.raises(ValueError) as err_info: PeglegSecretManagement(docs=['doc'], generated=True, catalog='catalog') assert 'If the document is generated, author and catalog must be ' \ 'specified.' in str(err_info.value)
def _get_or_gen(self, generator, kinds, document_name, *args, **kwargs): docs = self._find_docs(kinds, document_name) if not docs or self._regenerate_all: docs = generator(document_name, *args, **kwargs) else: docs = PeglegSecretManagement(docs=docs) # Adding these to output should be idempotent, so we use a dict. for wrapper_doc in docs: wrapped_doc = wrapper_doc['data']['managedDocument'] schema = wrapped_doc['schema'] name = wrapped_doc['metadata']['name'] self.outputs[schema][name] = wrapper_doc return docs
def _write(self, output_dir): documents = self.get_documents() output_paths = set() # First, delete each of the output paths below because we do an append # action in the `open` call below. This means that for regeneration # of certs, the original paths must be deleted. for document in documents: output_file_path = md.get_document_path( sitename=self._sitename, wrapper_document=document, cert_to_ca_map=self._cert_to_ca_map) output_path = os.path.join(output_dir, 'site', output_file_path) # NOTE(felipemonteiro): This is currently an entirely safe # operation as these files are being removed in the temporarily # replicated versions of the local repositories. if os.path.exists(output_path): os.remove(output_path) # Next, generate (or regenerate) the certificates. for document in documents: output_file_path = md.get_document_path( sitename=self._sitename, wrapper_document=document, cert_to_ca_map=self._cert_to_ca_map) output_path = os.path.join(output_dir, 'site', output_file_path) dir_name = os.path.dirname(output_path) if not os.path.exists(dir_name): LOG.debug('Creating secrets path: %s', dir_name) os.makedirs(os.path.abspath(dir_name)) # Encrypt the document document['data']['managedDocument']['metadata']['storagePolicy']\ = 'encrypted' document = PeglegSecretManagement( docs=[document]).get_encrypted_secrets()[0][0] util.files.dump(document, output_path, flag='a', default_flow_style=False, explicit_start=True, indent=2) output_paths.add(output_path) return output_paths
def check_cert_expiry(site_name, duration=60): """ Check certs from a sites PKICatalog to determine if they are expired or expiring within N days :param str site_name: The site to read from :param int duration: Number of days from today to check cert expirations :rtype: str """ cert_schemas = [ 'deckhand/Certificate/v1', 'deckhand/CertificateAuthority/v1' ] pki_util = PKIUtility(duration=duration) # Create a table to output expired/expiring certs for this site. cert_table = PrettyTable() cert_table.field_names = ['file', 'cert_name', 'expiration_date'] expired_certs_exist = False s = definition.site_files(site_name) for doc in s: if 'certificate' in doc: with open(doc, 'r') as f: results = yaml.safe_load_all(f) # Validate valid YAML. results = PeglegSecretManagement( docs=results).get_decrypted_secrets() for result in results: if result['schema'] in cert_schemas: text = result['data'] header_pattern = '-----BEGIN CERTIFICATE-----' footer_pattern = '-----END CERTIFICATE-----' find_pattern = r'%s.*?%s' % (header_pattern, footer_pattern) certs = re.findall(find_pattern, text, re.DOTALL) for cert in certs: cert_info = pki_util.check_expiry(cert) if cert_info['expired'] is True: cert_table.add_row([ doc, result['metadata']['name'], cert_info['expiry_date'] ]) expired_certs_exist = True # Return table of cert names and expiration dates that are expiring return expired_certs_exist, cert_table.get_string()
def test_global_encrypt_decrypt(temp_deployment_files, tmpdir): # Create site files site_dir = tmpdir.join("deployment_files", "site", "cicd") # Create and encrypt global passphrase and salt file using site keys with open(os.path.join(str(site_dir), 'secrets', 'passphrases', 'cicd-global-passphrase-encrypted.yaml'), "w") \ as outfile: outfile.write(GLOBAL_PASSPHRASE_SALT_DOC) save_location = tmpdir.mkdir("encrypted_site_files") save_location_str = str(save_location) # Encrypt the global passphrase and salt file using site passphrase/salt config.set_global_enc_keys("cicd") secrets.encrypt(save_location_str, "pytest", "cicd") # Create and encrypt a global type document global_doc_path = os.path.join(str(site_dir), 'secrets', 'passphrases', 'globally_encrypted_doc.yaml') with open(global_doc_path, "w") as outfile: outfile.write(TEST_GLOBAL_DATA) # encrypt documents and validate that they were encrypted doc_mgr = PeglegSecretManagement(file_path=global_doc_path, author='pytest', site_name='cicd') doc_mgr.encrypt_secrets(global_doc_path) doc = doc_mgr.documents[0] assert doc.is_encrypted() assert doc.data['encrypted']['by'] == 'pytest' doc_mgr = PeglegSecretManagement(file_path=global_doc_path, author='pytest', site_name='cicd') decrypted_data = doc_mgr.get_decrypted_secrets() test_data = list(yaml.safe_load_all(TEST_GLOBAL_DATA)) assert test_data[0]['data'] == decrypted_data[0]['data'] assert test_data[0]['schema'] == decrypted_data[0]['schema']
def encrypt(save_location, author, site_name, path=None): """ Encrypt all secrets documents for a site identifies by site_name. Parse through all documents related to ``site_name`` and encrypt all site documents, which have metadata.storagePolicy: encrypted, and are not already encrypted and wrapped in a PeglegManagedDocument. ``Passphrase`` and ``salt`` for the encryption are read from environment variables``$PEGLEG_PASSPHRASE`` and ``$PEGLEG_SALT`` respectively. By default, the resulting output files will overwrite the original unencrypted secrets documents. :param str save_location: if provided, is used as the base directory to store the encrypted secrets files. If not provided, the encrypted secrets files will overwrite the original unencrypted files (default behavior). :param str author: Identifies the individual or application, who encrypts the secrets documents. :param str site_name: The name of the site to encrypt its secrets files. :param str path: The path to the directory or file to be encrypted. """ files.check_file_save_location(save_location) LOG.debug('Save location is %s', save_location) file_sets = [] path_exists = path and os.path.exists(path) if path_exists: if os.path.isfile(path): LOG.debug('Specified path is a file') file_sets = [(None, path)] elif os.path.isdir(path): LOG.debug('Specified path is a directory') file_sets = [] for filename in glob(os.path.join(path, '**/*.yaml'), recursive=True): LOG.debug('Discovered %s', filename) file_sets.append((None, filename)) else: LOG.debug('No path specified, searching all repos') file_sets = list(definition.site_files_by_repo(site_name)) LOG.info('Started encrypting...') secrets_found = False for repo_base, file_path in file_sets: LOG.debug('Looking at %s in %s repo', file_path, repo_base) secrets_found = True secret = PeglegSecretManagement(file_path=file_path, author=author, site_name=site_name) if path_exists: if save_location: output_path = os.path.join(save_location.rstrip(os.path.sep), file_path.lstrip(os.path.sep)) else: output_path = file_path else: output_path = _get_dest_path(repo_base, file_path, save_location) LOG.debug('Outputting encrypted data to %s', output_path) secret.encrypt_secrets(output_path) if secrets_found: LOG.info('Encryption of all secret files was completed.') else: LOG.warning( 'No secret documents were found for site: {}'.format(site_name))
def test_encrypt_decrypt_using_file_path(tmpdir): # write the test data to temp file test_data = list(yaml.safe_load_all(TEST_DATA)) file_path = os.path.join(tmpdir, 'secrets_file.yaml') files.write(test_data, file_path) save_path = os.path.join(tmpdir, 'encrypted_secrets_file.yaml') # encrypt documents and validate that they were encrypted doc_mgr = PeglegSecretManagement(file_path=file_path, author='test_author') doc_mgr.encrypt_secrets(save_path) doc = doc_mgr.documents[0] assert doc.is_encrypted() assert doc.data['encrypted']['by'] == 'test_author' # decrypt documents and validate that they were decrypted doc_mgr = PeglegSecretManagement(file_path=file_path, author='test_author') doc_mgr.encrypt_secrets(save_path) # read back the encrypted file doc_mgr = PeglegSecretManagement(file_path=save_path, author='test_author') decrypted_data = doc_mgr.get_decrypted_secrets() assert test_data[0]['data'] == decrypted_data[0]['data'] assert test_data[0]['schema'] == decrypted_data[0]['schema']
def test_pegleg_secret_management_double_encrypt(): encrypted_doc = PeglegSecretManagement( docs=[yaml.safe_load(TEST_DATA)]).get_encrypted_secrets()[0][0] encrypted_doc_2 = PeglegSecretManagement( docs=[encrypted_doc]).get_encrypted_secrets()[0][0] assert encrypted_doc == encrypted_doc_2
def test_short_salt(): with pytest.raises(exceptions.SaltInsufficientLengthException): PeglegSecretManagement(file_path='file_path', author='test_author')
def generate(self, interactive=False, force_cleartext=False): """ For each passphrase entry in the passphrase catalog, generate a random passphrase string, based on a passphrase specification in the catalog. Create a pegleg managed document, wrap the generated passphrase document in the pegleg managed document, and encrypt the passphrase. Write the wrapped and encrypted document in a file at <repo_name>/site/<site_name>/secrets/passphrases/passphrase_name.yaml. :param bool interactive: If true, allow input :param bool force_cleartext: If true, don't encrypt """ for p_name in self._catalog.get_passphrase_names: # Check if this secret is present and should not be regenerated save_path = self.get_save_path(p_name) regenerable = self._catalog.is_passphrase_regenerable(p_name) if os.path.exists(save_path) and not regenerable: continue # Generate secret as it either does not exist yet or is a # regenerable secret and does exist but should be rotated. passphrase = None passphrase_type = self._catalog.get_passphrase_type(p_name) prompt = self._catalog.is_passphrase_prompt(p_name) profile = self._catalog.get_passphrase_profile(p_name) if interactive and prompt: auto_allowed = regenerable if passphrase_type == 'uuid': # nosec passphrase = self._prompt_user_passphrase_and_validate( p_name, 'UUID', self.validate_uuid, auto_allowed=auto_allowed) elif passphrase_type == 'base64': # nosec passphrase = self._prompt_user_passphrase_and_validate( p_name, 'passphrase (b64)', self.validate_base64, auto_allowed=auto_allowed) elif passphrase_type == 'passphrase': passphrase = self._prompt_user_passphrase_and_validate( p_name, 'passphrase', self.validate_passphrase, auto_allowed=auto_allowed) elif not interactive and prompt: LOG.debug('Skipping interactive input for %s', p_name) continue if not passphrase: if passphrase_type == 'uuid': # nosec passphrase = uuidutils.generate_uuid() else: passphrase = CryptoString(profile).get_crypto_string( self._catalog.get_length(p_name)) if passphrase_type == 'base64': # nosec # Take the randomly generated string and convert to a # random base64 string passphrase = passphrase.encode() passphrase = base64.b64encode(passphrase).decode() docs = list() if force_cleartext: storage_policy = passphrase_catalog.P_CLEARTEXT LOG.warning("Passphrases for {} will be " "generated in clear text.".format(p_name)) else: storage_policy = self._catalog.get_storage_policy(p_name) docs.append( self.generate_doc(KIND, p_name, storage_policy, passphrase)) if storage_policy == passphrase_catalog.P_ENCRYPTED: PeglegSecretManagement( docs=docs, generated=True, author=self._author, catalog=self._catalog).encrypt_secrets(save_path) else: files.write(docs, save_path)