def process_site_repository(update_config=False):
"""Process and setup site repository including ensuring we are at the right
revision based on the site's own site-definition.yaml file.
:param bool update_config: Whether to update Pegleg config with computed
site repo path.
"""
# Retrieve the main site repository and validate it.
site_repo_or_path = config.get_site_repo()
if not site_repo_or_path:
raise ValueError("Site repository directory (%s) must be specified" %
site_repo_or_path)
repo_url_or_path, repo_revision = _extract_repo_url_and_revision(
site_repo_or_path)
repo_url_or_path = _format_url_with_repo_username(repo_url_or_path)
new_repo_path = _process_repository(repo_url_or_path, repo_revision)
if update_config:
# Overwrite the site repo in the config because further processing will
# fail if they contain revision info in their paths.
LOG.debug("Updating site_repo=%s in config", new_repo_path)
config.set_site_repo(new_repo_path)
return new_repo_path
def __init__(self,
sitename,
block_strings=True,
author=None,
duration=365,
regenerate_all=False,
save_location=None):
"""Constructor for ``PKIGenerator``.
:param int duration: Duration in days that generated certificates
are valid.
:param str sitename: Site name for which to retrieve documents used for
certificate and keypair generation.
:param bool block_strings: Whether to dump out certificate data as
block-style YAML string. Defaults to true.
:param str author: Identifying name of the author generating new
certificates.
:param bool regenerate_all: If Pegleg should regenerate all certs.
"""
self._regenerate_all = regenerate_all
self._sitename = sitename
self._documents = site.get_rendered_docs(sitename)
self._author = author
self._save_location = save_location or config.get_site_repo()
self.keys = pki_utility.PKIUtility(block_strings=block_strings,
duration=duration)
self.outputs = collections.defaultdict(dict)
# Maps certificates to CAs in order to derive certificate paths.
self._cert_to_ca_map = {}
def _collect_to_file(site_name, save_location):
"""Collects all documents related to ``site_name`` and outputs them to
the file denoted by ``save_location``.
"""
files.check_file_save_location(save_location)
save_files = dict()
curr_site_repo = files.path_leaf(config.get_site_repo())
try:
for repo_base, filename in util.definition.site_files_by_repo(
site_name):
repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
save_file = os.path.join(save_location, repo_name + '.yaml')
if repo_name not in save_files:
save_files[repo_name] = open(save_file, 'w')
LOG.debug("Collecting file %s to file %s", filename, save_file)
save_files[repo_name].writelines(_read_and_format_yaml(filename))
add_representer_ordered_dict()
save_files[curr_site_repo].writelines(
yaml.safe_dump(get_deployment_data_doc(site_name),
default_flow_style=False,
explicit_start=True,
explicit_end=True))
except Exception as ex:
raise click.ClickException("Error saving output: %s" % str(ex))
finally:
for f in save_files.values():
f.close()
def __wrap(secrets_document, generated=False, catalog=None, author=None):
"""
Embeds a valid deckhand document in a pegleg managed document.
:param secrets_document: secrets document to be embedded in a
pegleg managed document.
:type secrets_document: dict
:param bool generated: A flag to indicate the documents are
auto-generated by pegleg (True), or manually created (False).
:return: pegleg manged document with the wrapped original secrets
document.
:rtype: dict
"""
layer = secrets_document.get('metadata',
{}).get('layeringDefinition',
{}).get('layer', DEFAULT_LAYER)
layering_definition = OrderedDict(
[('abstract', False), ('layer', layer)])
metadata = OrderedDict(
[
(
'name', '{}/{}'.format(
secrets_document['schema'],
secrets_document['metadata']['name'])),
('schema', 'metadata/Document/v1'),
('labels', secrets_document['metadata'].get('labels', {})),
('layeringDefinition', layering_definition),
('storagePolicy', 'cleartext')
])
data = OrderedDict(
[
(
'managedDocument',
OrderedDict(
[
('schema', secrets_document['schema']),
('metadata', secrets_document['metadata']),
('data', secrets_document['data'])
]))
])
doc = OrderedDict(
[
('schema', PEGLEG_MANAGED_SCHEMA), ('metadata', metadata),
('data', data)
])
if generated:
doc['data'][GENERATED] = {
'at': datetime.utcnow().isoformat(),
'by': author,
'specifiedBy': {
'repo': git.repo_url(config.get_site_repo()),
'reference': config.get_site_rev() or 'master',
'path': catalog.catalog_path,
},
}
return doc
def test_read_incompatible_file(self, temp_deployment_files):
# NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
# Deckhand-formatted document currently but probably shouldn't be,
# because it has no business being in Deckhand. As such, validate that
# it is ignored.
path = os.path.join(config.get_site_repo(), 'site', 'cicd',
'site-definition.yaml')
documents = files.read(path)
assert not documents, ("Documents returned should be empty for "
"site-definition.yaml")
def list_types(primary_repo_base=None):
"""Get a list of type directories in the primary repo."""
if not primary_repo_base:
primary_repo_base = config.get_site_repo()
full_type_path = os.path.join(primary_repo_base,
config.get_rel_type_path())
for path in os.listdir(full_type_path):
joined_path = os.path.join(full_type_path, path)
if os.path.isdir(joined_path):
yield path
def _set_catalog_path(self):
repo_name = git.repo_url(config.get_site_repo())
catalog_name = self._get_document_name('{}.yaml'.format(self._kind))
for file_path in definition.site_files(self.site_name):
if file_path.endswith(catalog_name):
self._catalog_path.append(file_path)
if not self._catalog_path:
# Cound not find the Catalog for this generated passphrase
# raise an exception.
LOG.error('Catalog path: {} was not found in repo: {}'.format(
catalog_name, repo_name))
raise PassphraseCatalogNotFoundException()
def test_write(self, temp_deployment_files):
path = os.path.join(config.get_site_repo(), 'site', 'cicd',
'test_out.yaml')
files.write("test text", path)
with open(path, "r") as out_fi:
assert out_fi.read() == "test text"
files.write({"a": 1}, path)
with open(path, "r") as out_fi:
assert yaml.safe_load(out_fi) == {"a": 1}
files.write([{"a": 1}], path)
with open(path, "r") as out_fi:
assert list(yaml.safe_load_all(out_fi)) == [{"a": 1}]
with pytest.raises(ValueError):
files.write(object(), path)
def run_encrypt(author, save_location, site_name, path=None):
"""Wraps and encrypts site secret documents
:param author: identifies author generating new certificates for
tracking information
:param save_location: path to save encrypted documents to, if None the
original documents are overwritten
:param site_name: site name to process
:param path: path to the document(s) to encrypt
:return:
"""
config.set_global_enc_keys(site_name)
if save_location is None and path is None:
save_location = config.get_site_repo()
engine.secrets.encrypt(save_location,
author,
site_name=site_name,
path=path)
def create_tmp_pki_structure(tmpdir):
"""Fixture that creates a temporary site directory structure include pki/
subfolder for validating PKIGenerator logic.
:returns: Function pointer, which, when called, creates a temporary file
structure with pki/ subfolder.
"""
def _create_tmp_folder_system(sitename, pki_catalog):
"""Creates a temporary site folder system.
:param str sitename: Name of the site.
:param str pki_catalog: YAML-formatted string that adheres to
pki-catalog.yaml structure.
"""
# Create site directories and files.
p = tmpdir.mkdir("deployment_files")
config.set_site_repo(p.strpath)
site_definition = copy.deepcopy(_SITE_DEFINITION)
site_definition = site_definition % {'sitename': sitename}
pki_catalog = copy.deepcopy(pki_catalog)
pki_catalog = pki_catalog.format(sitename=sitename)
test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE)
test_structure['files']['site-definition.yaml'] = yaml.safe_load(
site_definition)
test_structure['files']['layering-definition.yaml'] = yaml.safe_load(
_LAYERING_DEFINITION)
test_structure['directories']['pki']['files'][
'pki-catalog.yaml'] = yaml.safe_load(pki_catalog)
test_path = os.path.join(p.strpath, files._site_path(sitename))
files._create_tree(test_path, tree=test_structure)
return p.strpath
try:
yield _create_tmp_folder_system
finally:
temp_path = config.get_site_repo()
if temp_path != './' and os.path.exists(temp_path):
shutil.rmtree(temp_path, ignore_errors=True)
def create_tmp_site_structure(tmpdir):
"""Fixture that creates a temporary site directory structure
:returns: Function pointer, which, when called, creates a temporary file
structure.
"""
def _create_tmp_folder_system(sitename):
"""Creates a temporary site folder system.
:param str sitename: Name of the site.
"""
# Create site directories and files.
p = tmpdir.mkdir("deployment_files")
config.set_site_repo(p.strpath)
site_definition = copy.deepcopy(_SITE_DEFINITION)
site_definition = site_definition % {'sitename': sitename}
test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE)
test_structure['files']['site-definition.yaml'] = yaml.safe_load(
site_definition)
test_structure['files']['layering-definition.yaml'] = yaml.safe_load(
_LAYERING_DEFINITION)
test_structure['directories']['secrets']['directories']['passphrases'][
'files']['plaintext.yaml'] = yaml.safe_load(_PLAINTEXT_SECRET)
test_structure['directories']['secrets']['directories']['passphrases'][
'files']['managed.yaml'] = yaml.safe_load(_MANAGED_SECRET)
test_structure['directories']['secrets']['directories']['passphrases'][
'files']['encrypted.yaml'] = yaml.safe_load(_ENCRYPTED_SECRET)
test_path = os.path.join(p.strpath, files._site_path(sitename))
files._create_tree(test_path, tree=test_structure)
return p.strpath
try:
yield _create_tmp_folder_system
finally:
temp_path = config.get_site_repo()
if temp_path != './' and os.path.exists(temp_path):
shutil.rmtree(temp_path, ignore_errors=True)
def test_read_compatible_file(self, temp_deployment_files):
path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets',
'passphrases', 'cicd-passphrase.yaml')
documents = files.read(path)
assert 1 == len(documents)
def path(site_name, primary_repo_base=None):
"""Retrieve path to the site-definition.yaml file for ``site_name``."""
if not primary_repo_base:
primary_repo_base = config.get_site_repo()
return os.path.join(primary_repo_base, 'site', site_name,
'site-definition.yaml')
def path(site_name, primary_repo_base=None):
if not primary_repo_base:
primary_repo_base = config.get_site_repo()
return os.path.join(primary_repo_base, 'site', site_name,
'site-definition.yaml')
def test_dir_permissions(self, temp_deployment_files):
path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'test_dir')
os.makedirs(path)
assert oct(os.stat(path).st_mode & 0o777) == EXPECTED_DIR_PERM
def test_file_permissions(self, temp_deployment_files):
path = os.path.join(config.get_site_repo(), 'site', 'cicd',
'test_out.yaml')
files.write("test text", path)
assert oct(os.stat(path).st_mode & 0o777) == EXPECTED_FILE_PERM
def _get_repo_url_and_rev():
repo_path_or_url = config.get_site_repo()
repo_url = git.repo_url(repo_path_or_url)
repo_rev = config.get_site_rev()
return repo_url, repo_rev