def test_lint_warns_P004(self, mock_layering):
"""Test warn flag for P004 - Duplicate Deckhand DataSchema document
detected.
"""
# Stub out Deckhand render logic.
mock_layering.DocumentLayering.return_value.render.return_value = []
exclude_lint = self._exclude_all(except_code='P004')
warn_lint = ['P004']
config.set_site_repo(self.site_yaml_path)
documents = {
mock.sentinel.site: [
{
# Create 2 duplicate DataSchema documents.
"schema": "deckhand/DataSchema/v1",
"metadata": {
"name": mock.sentinel.document_name
},
"data": {}
}
] * 2
}
with mock.patch(
'pegleg.engine.util.definition.documents_for_each_site',
autospec=True, return_value=documents):
result = lint.full(
False, exclude_lint=exclude_lint, warn_lint=warn_lint)
assert len(result) == 1
assert result[0].startswith(warn_lint[0])
def _create_tmp_folder_system(sitename):
"""Creates a temporary site folder system.
:param str sitename: Name of the site.
"""
# Create site directories and files.
p = tmpdir.mkdir("deployment_files")
config.set_site_repo(p.strpath)
site_definition = copy.deepcopy(_SITE_DEFINITION)
site_definition = site_definition % {'sitename': sitename}
test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE)
test_structure['files']['site-definition.yaml'] = yaml.safe_load(
site_definition)
test_structure['files']['layering-definition.yaml'] = yaml.safe_load(
_LAYERING_DEFINITION)
test_structure['directories']['secrets']['directories']['passphrases'][
'files']['plaintext.yaml'] = yaml.safe_load(_PLAINTEXT_SECRET)
test_structure['directories']['secrets']['directories']['passphrases'][
'files']['managed.yaml'] = yaml.safe_load(_MANAGED_SECRET)
test_structure['directories']['secrets']['directories']['passphrases'][
'files']['encrypted.yaml'] = yaml.safe_load(_ENCRYPTED_SECRET)
test_path = os.path.join(p.strpath, files._site_path(sitename))
files._create_tree(test_path, tree=test_structure)
return p.strpath
def _create_tmp_folder_system(sitename, pki_catalog):
"""Creates a temporary site folder system.
:param str sitename: Name of the site.
:param str pki_catalog: YAML-formatted string that adheres to
pki-catalog.yaml structure.
"""
# Create site directories and files.
p = tmpdir.mkdir("deployment_files")
config.set_site_repo(p.strpath)
site_definition = copy.deepcopy(_SITE_DEFINITION)
site_definition = site_definition % {'sitename': sitename}
pki_catalog = copy.deepcopy(pki_catalog)
pki_catalog = pki_catalog.format(sitename=sitename)
test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE)
test_structure['files']['site-definition.yaml'] = yaml.safe_load(
site_definition)
test_structure['files']['layering-definition.yaml'] = yaml.safe_load(
_LAYERING_DEFINITION)
test_structure['directories']['pki']['files'][
'pki-catalog.yaml'] = yaml.safe_load(pki_catalog)
test_path = os.path.join(p.strpath, files._site_path(sitename))
files._create_tree(test_path, tree=test_structure)
return p.strpath
def test_lint_warns_P005(self, mock_layering):
"""Test warn flag for P005 - Deckhand rendering exception."""
# Make Deckhand render expected exception to trigger error code.
mock_layering.DocumentLayering.return_value.render.side_effect = (
dh_errors.DeckhandException)
exclude_lint = self._exclude_all(except_code='P005')
warn_lint = ['P005']
config.set_site_repo(self.site_yaml_path)
documents = {
mock.sentinel.site: [
{
"schema": "deckhand/DataSchema/v1",
"metadata": {
"name": mock.sentinel.document_name
},
"data": {}
}
]
}
with mock.patch(
'pegleg.engine.util.definition.documents_for_each_site',
autospec=True, return_value=documents):
result = lint.full(
False, exclude_lint=exclude_lint, warn_lint=warn_lint)
assert len(result) == 1
assert result[0].startswith(warn_lint[0])
def process_site_repository(update_config=False):
"""Process and setup site repository including ensuring we are at the right
revision based on the site's own site-definition.yaml file.
:param bool update_config: Whether to update Pegleg config with computed
site repo path.
"""
# Retrieve the main site repository and validate it.
site_repo_or_path = config.get_site_repo()
if not site_repo_or_path:
raise ValueError("Site repository directory (%s) must be specified" %
site_repo_or_path)
repo_url_or_path, repo_revision = _extract_repo_url_and_revision(
site_repo_or_path)
repo_url_or_path = _format_url_with_repo_username(repo_url_or_path)
new_repo_path = _process_repository(repo_url_or_path, repo_revision)
if update_config:
# Overwrite the site repo in the config because further processing will
# fail if they contain revision info in their paths.
LOG.debug("Updating site_repo=%s in config", new_repo_path)
config.set_site_repo(new_repo_path)
return new_repo_path
def _do_test(site_repo, expected):
config.set_site_repo(site_repo)
with mock.patch.object(repository, '_handle_repository', autospec=True,
side_effect=lambda x, *a, **k: x):
result = repository.process_site_repository()
assert os.path.normpath(expected) == os.path.normpath(result)
def run_config(site_repository,
clone_path,
repo_key,
repo_username,
extra_repositories,
run_umask=True,
decrypt_repos=False):
"""Initializes pegleg configuration data
:param site_repository: path or URL for site repository
:param clone_path: directory in which to clone the site_repository
:param repo_key: key for remote repository URL if needed
:param repo_username: username to replace REPO_USERNAME in repository URL
if needed
:param extra_repositories: list of extra repositories to read in documents
from, specified as "type=REPO_URL/PATH"
:param run_umask: if True, runs set_umask for os file output
:param decrypt_repos: if True, decrypts repos before executing command
:return:
"""
config.set_site_repo(site_repository)
config.set_clone_path(clone_path)
if extra_repositories:
config.set_extra_repo_overrides(extra_repositories)
config.set_repo_key(repo_key)
config.set_repo_username(repo_username)
if run_umask:
config.set_umask()
config.set_decrypt_repos(decrypt_repos)
def test_lint_warns_P002(*args):
warn_lint = ['P002']
config.set_site_repo('../pegleg/site_yamls/')
with mock.patch.object(lint, '_verify_deckhand_render') as mock_method:
lint.full(False, [], warn_lint)
mock_method.assert_called()
def test_lint_warns_P003(*args):
warn_lint = ['P003']
config.set_site_repo('../pegleg/site_yamls/')
with mock.patch.object(lint,
'_verify_no_unexpected_files') as mock_method:
lint.full(False, [], warn_lint)
mock_method.assert_called()
def test_lint_excludes_P002(*args):
exclude_lint = ['P002']
config.set_site_repo('../pegleg/site_yamls/')
with mock.patch.object(lint,
'_verify_deckhand_render',
return_value=[('P002', 'test message')
]) as mock_method:
lint.full(False, exclude_lint, [])
mock_method.assert_called()
def repo(*, site_repository, clone_path, repo_key, repo_username):
"""Group for repo-level actions, which include:
* lint: lint all sites across the repository
"""
config.set_site_repo(site_repository)
config.set_clone_path(clone_path)
config.set_repo_key(repo_key)
config.set_repo_username(repo_username)
def test_no_non_yamls(tmpdir):
p = tmpdir.mkdir("deployment_files").mkdir("global")
for x in range(3): # Create 3 YAML files
p.join("good-%d.yaml" % x).write('fake-content')
p.join("bad.txt").write("fake-content")
config.set_site_repo(str(tmpdir.listdir()[0]))
results = list(files.all())
assert 3 == len(results)
# Make sure only YAML files are returned
for i in results:
assert i.endswith('.yaml')
def type(*, site_repository, clone_path, extra_repositories, repo_key,
repo_username):
"""Group for repo-level actions, which include:
* list: list all types across the repository
"""
config.set_site_repo(site_repository)
config.set_clone_path(clone_path)
config.set_extra_repo_overrides(extra_repositories or [])
config.set_repo_key(repo_key)
config.set_repo_username(repo_username)
def test_lint_warns_P003(self, *args):
"""Test warn flag for P003 - All repos contain expected directories."""
exclude_lint = self._exclude_all(except_code='P003')
warn_lint = ['P003']
config.set_site_repo(self.site_yaml_path)
with mock.patch.object(lint,
'_verify_no_unexpected_files') as mock_method:
result = lint.full(
False, exclude_lint=exclude_lint, warn_lint=warn_lint)
mock_method.assert_called()
assert len(result) == 1
assert result[0].startswith(warn_lint[0])
def site(*, site_repository, clone_path, extra_repositories, repo_key,
repo_username):
"""Group for site-level actions, which include:
* list: list available sites in a manifests repo
* lint: lint a site along with all its dependencies
* render: render a site using Deckhand
* show: show a sites' files
"""
config.set_site_repo(site_repository)
config.set_clone_path(clone_path)
config.set_extra_repo_overrides(extra_repositories or [])
config.set_repo_key(repo_key)
config.set_repo_username(repo_username)
def test_lint_warns_P006(self, tmpdir):
"""Test warn flag for P006 - YAML file missing document header."""
exclude_lint = self._exclude_all(except_code='P006')
warn_lint = ['P006']
config.set_site_repo(self.site_yaml_path)
p = tmpdir.mkdir(self.__class__.__name__).join("test.yaml")
p.write("foo: bar")
with mock.patch('pegleg.engine.util.files.all', autospec=True,
return_value=[p.strpath]):
result = lint.full(
False, exclude_lint=exclude_lint, warn_lint=warn_lint)
assert len(result) == 1
assert result[0].startswith(warn_lint[0])
def test_lint_warns_P007(self, tmpdir):
"""Test warn flag for P007 - YAML file is not valid YAML."""
exclude_lint = self._exclude_all(except_code='P007')
warn_lint = ['P007']
config.set_site_repo(self.site_yaml_path)
p = tmpdir.mkdir(self.__class__.__name__).join("test.yaml")
# Invalid YAML - will trigger error.
p.write("---\nfoo: bar: baz")
with mock.patch('pegleg.engine.util.files.all', autospec=True,
return_value=[p.strpath]):
result = lint.full(
False, exclude_lint=exclude_lint, warn_lint=warn_lint)
assert len(result) == 1
assert result[0].startswith(warn_lint[0])
def test_lint_warns_P001(*args):
warn_lint = ['P001']
config.set_site_repo('../pegleg/site_yamls/')
code_1 = 'X001'
msg_1 = 'is a secret, but has unexpected storagePolicy: "cleartext"'
code_2 = 'X002'
msg_2 = 'test msg'
msgs = [(code_1, msg_1), (code_2, msg_2)]
with mock.patch.object(lint,
'_verify_file_contents',
return_value=msgs) as mock_methed:
with pytest.raises(click.ClickException) as expected_exc:
lint.full(False, [], warn_lint)
assert msg_1 not in expected_exc
assert msg_2 in expected_exc
def test_lint_excludes_P001(self, *args):
"""Test exclude flag for P001 - Document has storagePolicy cleartext
(expected is encrypted) yet its schema is a mandatory encrypted type.
"""
exclude_lint = ['P001']
config.set_site_repo(self.site_yaml_path)
code_1 = 'X001'
msg_1 = 'is a secret, but has unexpected storagePolicy: "cleartext"'
code_2 = 'X002'
msg_2 = 'test msg'
msgs = [(code_1, msg_1), (code_2, msg_2)]
with mock.patch.object(lint, '_verify_file_contents',
return_value=msgs) as mock_methed:
with pytest.raises(click.ClickException) as expected_exc:
lint.full(False, exclude_lint, [])
assert msg_1 in expected_exc
assert msg_2 in expected_exc
def test_failed_deckhand_validation(tmpdir):
# Write the test data to temp file
config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
base_config_dir = os.path.join(tmpdir, 'config_dir')
config.set_site_repo(base_config_dir)
config_dir = os.path.join(base_config_dir, 'site', 'test_site')
config_path = os.path.join(config_dir, 'config_file.yaml')
build_dir = os.path.join(tmpdir, 'build_dir')
os.makedirs(config_dir)
files.write(config_data, config_path)
files.write(
yaml.safe_load_all(SITE_DEFINITION),
os.path.join(config_dir, "site-definition.yaml"))
key = 'MyverYSecretEncryptionKey382803'
with pytest.raises(GenesisBundleGenerateException,
match=r'.*failed on deckhand validation.*'):
bundle.build_genesis(
build_path=build_dir,
encryption_key=key,
validators=False,
debug=logging.ERROR,
site_name="test_site")
def test_no_encryption_key(tmpdir):
# Write the test data to temp file
config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
base_config_dir = os.path.join(tmpdir, 'config_dir')
config.set_site_repo(base_config_dir)
config_dir = os.path.join(base_config_dir, 'site', 'test_site')
config_path = os.path.join(config_dir, 'config_file.yaml')
build_dir = os.path.join(tmpdir, 'build_dir')
os.makedirs(config_dir)
files.write(config_data, config_path)
files.write(
yaml.safe_load_all(SITE_DEFINITION),
os.path.join(config_dir, "site-definition.yaml"))
with pytest.raises(GenesisBundleEncryptionException,
match=r'.*no encryption policy or key is specified.*'):
bundle.build_genesis(
build_path=build_dir,
encryption_key=None,
validators=False,
debug=logging.ERROR,
site_name="test_site")
def process_repositories(site_name):
"""Process and setup all repositories including ensuring we are at the
right revision based on the site's own site-definition.yaml file.
:param site_name: Site name for which to clone relevant repos.
"""
# Only tracks extra repositories - not the site (primary) repository.
extra_repos = []
site_repo = process_site_repository()
# Retrieve extra repo data from site-definition.yaml files.
site_data = util.definition.load_as_params(site_name,
primary_repo_base=site_repo)
site_def_repos = _get_and_validate_site_repositories(site_name, site_data)
# Dict mapping repository names to associated URL/revision info for clone.
repo_overrides = _process_repository_overrides(site_def_repos)
if not site_def_repos:
LOG.info(
'No repositories found in site-definition.yaml for site: %s. '
'Defaulting to specified repository overrides.', site_name)
site_def_repos = repo_overrides
# Extract user/key that we will use for all repositories.
repo_key = config.get_repo_key()
repo_user = config.get_repo_username()
for repo_alias in site_def_repos.keys():
if repo_alias == "site":
LOG.warning(
"The primary site repository path must be specified "
"via the -r flag. Ignoring the provided "
"site-definition entry: %s", site_def_repos[repo_alias])
continue
# Extract URL and revision, prioritizing overrides over the defaults in
# the site-definition.yaml.
if repo_alias in repo_overrides:
repo_url_or_path = repo_overrides[repo_alias]['url']
repo_revision = repo_overrides[repo_alias]['revision']
else:
repo_url_or_path = site_def_repos[repo_alias]['url']
repo_revision = site_def_repos[repo_alias]['revision']
repo_url_or_path = _format_url_with_repo_username(repo_url_or_path)
LOG.info(
"Processing repository %s with url=%s, repo_key=%s, "
"repo_username=%s, revision=%s", repo_alias, repo_url_or_path,
repo_key, repo_user, repo_revision)
temp_extra_repo = _process_repository(repo_url_or_path, repo_revision)
extra_repos.append(temp_extra_repo)
# Overwrite the site repo and extra repos in the config because further
# processing will fail if they contain revision info in their paths.
LOG.debug("Updating site_repo=%s extra_repo_list=%s in config", site_repo,
extra_repos)
config.set_site_repo(site_repo)
config.set_extra_repo_list(extra_repos)
def create_tmp_deployment_files(tmpdir):
"""Fixture that creates a temporary directory structure."""
sitenames = ['cicd', 'lab']
SITE_TEST_STRUCTURE = {
'directories': {
'secrets': {
'directories': {
'passphrases': {
'files': {}
},
},
},
'software': {
'directories': {
'charts': {
'files': {}
},
},
},
},
'files': {}
}
p = tmpdir.mkdir("deployment_files")
config.set_site_repo(str(p))
# Create global directories and files.
files._create_tree(os.path.join(str(p), 'global'),
tree={
'directories': {
'common': {
'files': {
'global-common.yaml':
_gen_document(name="global-common",
layer='global')
}
},
'v1.0': {
'files': {
'global-v1.0.yaml':
_gen_document(name="global-v1.0",
layer='global')
}
}
}
})
# Create type directories and files.
files._create_tree(
os.path.join(str(p), 'type'),
tree={
'directories': {
site: {
'directories': {
'common': {
'files': {
'%s-type-common.yaml' % site:
_gen_document(name="%s-type-common" % site,
layer='type')
}
},
'v1.0': {
'files': {
'%s-type-v1.0.yaml' % site:
_gen_document(name="%s-type-v1.0" % site,
layer='type')
}
}
}
}
for site in sitenames
}
})
# Create site directories and files.
for site in sitenames:
site_definition = """
---
data:
repositories:
global:
revision: v1.0
url: http://nothing.com
site_type: %s
metadata:
layeringDefinition: {abstract: false, layer: site}
name: %s
schema: metadata/Document/v1
storagePolicy: cleartext
schema: pegleg/SiteDefinition/v1
""" % (site, site)
test_structure = SITE_TEST_STRUCTURE.copy()
test_structure['directories']['secrets']['directories']['passphrases'][
'files'] = {
'%s-passphrase.yaml' % site:
_gen_document(name="%s-passphrase" % site, layer='site')
}
test_structure['directories']['software']['directories']['charts'][
'files'] = {
'%s-chart.yaml' % site:
_gen_document(name="%s-chart" % site, layer='site')
}
test_structure['files']['site-definition.yaml'] = yaml.safe_load(
site_definition)
cicd_path = os.path.join(str(p), files._site_path(site))
files._create_tree(cicd_path, tree=test_structure)
yield
