def test_lint_warns_P004(self, mock_layering): """Test warn flag for P004 - Duplicate Deckhand DataSchema document detected. """ # Stub out Deckhand render logic. mock_layering.DocumentLayering.return_value.render.return_value = [] exclude_lint = self._exclude_all(except_code='P004') warn_lint = ['P004'] config.set_site_repo(self.site_yaml_path) documents = { mock.sentinel.site: [ { # Create 2 duplicate DataSchema documents. "schema": "deckhand/DataSchema/v1", "metadata": { "name": mock.sentinel.document_name }, "data": {} } ] * 2 } with mock.patch( 'pegleg.engine.util.definition.documents_for_each_site', autospec=True, return_value=documents): result = lint.full( False, exclude_lint=exclude_lint, warn_lint=warn_lint) assert len(result) == 1 assert result[0].startswith(warn_lint[0])
def _create_tmp_folder_system(sitename): """Creates a temporary site folder system. :param str sitename: Name of the site. """ # Create site directories and files. p = tmpdir.mkdir("deployment_files") config.set_site_repo(p.strpath) site_definition = copy.deepcopy(_SITE_DEFINITION) site_definition = site_definition % {'sitename': sitename} test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE) test_structure['files']['site-definition.yaml'] = yaml.safe_load( site_definition) test_structure['files']['layering-definition.yaml'] = yaml.safe_load( _LAYERING_DEFINITION) test_structure['directories']['secrets']['directories']['passphrases'][ 'files']['plaintext.yaml'] = yaml.safe_load(_PLAINTEXT_SECRET) test_structure['directories']['secrets']['directories']['passphrases'][ 'files']['managed.yaml'] = yaml.safe_load(_MANAGED_SECRET) test_structure['directories']['secrets']['directories']['passphrases'][ 'files']['encrypted.yaml'] = yaml.safe_load(_ENCRYPTED_SECRET) test_path = os.path.join(p.strpath, files._site_path(sitename)) files._create_tree(test_path, tree=test_structure) return p.strpath
def _create_tmp_folder_system(sitename, pki_catalog): """Creates a temporary site folder system. :param str sitename: Name of the site. :param str pki_catalog: YAML-formatted string that adheres to pki-catalog.yaml structure. """ # Create site directories and files. p = tmpdir.mkdir("deployment_files") config.set_site_repo(p.strpath) site_definition = copy.deepcopy(_SITE_DEFINITION) site_definition = site_definition % {'sitename': sitename} pki_catalog = copy.deepcopy(pki_catalog) pki_catalog = pki_catalog.format(sitename=sitename) test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE) test_structure['files']['site-definition.yaml'] = yaml.safe_load( site_definition) test_structure['files']['layering-definition.yaml'] = yaml.safe_load( _LAYERING_DEFINITION) test_structure['directories']['pki']['files'][ 'pki-catalog.yaml'] = yaml.safe_load(pki_catalog) test_path = os.path.join(p.strpath, files._site_path(sitename)) files._create_tree(test_path, tree=test_structure) return p.strpath
def test_lint_warns_P005(self, mock_layering): """Test warn flag for P005 - Deckhand rendering exception.""" # Make Deckhand render expected exception to trigger error code. mock_layering.DocumentLayering.return_value.render.side_effect = ( dh_errors.DeckhandException) exclude_lint = self._exclude_all(except_code='P005') warn_lint = ['P005'] config.set_site_repo(self.site_yaml_path) documents = { mock.sentinel.site: [ { "schema": "deckhand/DataSchema/v1", "metadata": { "name": mock.sentinel.document_name }, "data": {} } ] } with mock.patch( 'pegleg.engine.util.definition.documents_for_each_site', autospec=True, return_value=documents): result = lint.full( False, exclude_lint=exclude_lint, warn_lint=warn_lint) assert len(result) == 1 assert result[0].startswith(warn_lint[0])
def process_site_repository(update_config=False): """Process and setup site repository including ensuring we are at the right revision based on the site's own site-definition.yaml file. :param bool update_config: Whether to update Pegleg config with computed site repo path. """ # Retrieve the main site repository and validate it. site_repo_or_path = config.get_site_repo() if not site_repo_or_path: raise ValueError("Site repository directory (%s) must be specified" % site_repo_or_path) repo_url_or_path, repo_revision = _extract_repo_url_and_revision( site_repo_or_path) repo_url_or_path = _format_url_with_repo_username(repo_url_or_path) new_repo_path = _process_repository(repo_url_or_path, repo_revision) if update_config: # Overwrite the site repo in the config because further processing will # fail if they contain revision info in their paths. LOG.debug("Updating site_repo=%s in config", new_repo_path) config.set_site_repo(new_repo_path) return new_repo_path
def _do_test(site_repo, expected): config.set_site_repo(site_repo) with mock.patch.object(repository, '_handle_repository', autospec=True, side_effect=lambda x, *a, **k: x): result = repository.process_site_repository() assert os.path.normpath(expected) == os.path.normpath(result)
def run_config(site_repository, clone_path, repo_key, repo_username, extra_repositories, run_umask=True, decrypt_repos=False): """Initializes pegleg configuration data :param site_repository: path or URL for site repository :param clone_path: directory in which to clone the site_repository :param repo_key: key for remote repository URL if needed :param repo_username: username to replace REPO_USERNAME in repository URL if needed :param extra_repositories: list of extra repositories to read in documents from, specified as "type=REPO_URL/PATH" :param run_umask: if True, runs set_umask for os file output :param decrypt_repos: if True, decrypts repos before executing command :return: """ config.set_site_repo(site_repository) config.set_clone_path(clone_path) if extra_repositories: config.set_extra_repo_overrides(extra_repositories) config.set_repo_key(repo_key) config.set_repo_username(repo_username) if run_umask: config.set_umask() config.set_decrypt_repos(decrypt_repos)
def test_lint_warns_P002(*args): warn_lint = ['P002'] config.set_site_repo('../pegleg/site_yamls/') with mock.patch.object(lint, '_verify_deckhand_render') as mock_method: lint.full(False, [], warn_lint) mock_method.assert_called()
def test_lint_warns_P003(*args): warn_lint = ['P003'] config.set_site_repo('../pegleg/site_yamls/') with mock.patch.object(lint, '_verify_no_unexpected_files') as mock_method: lint.full(False, [], warn_lint) mock_method.assert_called()
def test_lint_excludes_P002(*args): exclude_lint = ['P002'] config.set_site_repo('../pegleg/site_yamls/') with mock.patch.object(lint, '_verify_deckhand_render', return_value=[('P002', 'test message') ]) as mock_method: lint.full(False, exclude_lint, []) mock_method.assert_called()
def repo(*, site_repository, clone_path, repo_key, repo_username): """Group for repo-level actions, which include: * lint: lint all sites across the repository """ config.set_site_repo(site_repository) config.set_clone_path(clone_path) config.set_repo_key(repo_key) config.set_repo_username(repo_username)
def test_no_non_yamls(tmpdir): p = tmpdir.mkdir("deployment_files").mkdir("global") for x in range(3): # Create 3 YAML files p.join("good-%d.yaml" % x).write('fake-content') p.join("bad.txt").write("fake-content") config.set_site_repo(str(tmpdir.listdir()[0])) results = list(files.all()) assert 3 == len(results) # Make sure only YAML files are returned for i in results: assert i.endswith('.yaml')
def type(*, site_repository, clone_path, extra_repositories, repo_key, repo_username): """Group for repo-level actions, which include: * list: list all types across the repository """ config.set_site_repo(site_repository) config.set_clone_path(clone_path) config.set_extra_repo_overrides(extra_repositories or []) config.set_repo_key(repo_key) config.set_repo_username(repo_username)
def test_lint_warns_P003(self, *args): """Test warn flag for P003 - All repos contain expected directories.""" exclude_lint = self._exclude_all(except_code='P003') warn_lint = ['P003'] config.set_site_repo(self.site_yaml_path) with mock.patch.object(lint, '_verify_no_unexpected_files') as mock_method: result = lint.full( False, exclude_lint=exclude_lint, warn_lint=warn_lint) mock_method.assert_called() assert len(result) == 1 assert result[0].startswith(warn_lint[0])
def site(*, site_repository, clone_path, extra_repositories, repo_key, repo_username): """Group for site-level actions, which include: * list: list available sites in a manifests repo * lint: lint a site along with all its dependencies * render: render a site using Deckhand * show: show a sites' files """ config.set_site_repo(site_repository) config.set_clone_path(clone_path) config.set_extra_repo_overrides(extra_repositories or []) config.set_repo_key(repo_key) config.set_repo_username(repo_username)
def test_lint_warns_P006(self, tmpdir): """Test warn flag for P006 - YAML file missing document header.""" exclude_lint = self._exclude_all(except_code='P006') warn_lint = ['P006'] config.set_site_repo(self.site_yaml_path) p = tmpdir.mkdir(self.__class__.__name__).join("test.yaml") p.write("foo: bar") with mock.patch('pegleg.engine.util.files.all', autospec=True, return_value=[p.strpath]): result = lint.full( False, exclude_lint=exclude_lint, warn_lint=warn_lint) assert len(result) == 1 assert result[0].startswith(warn_lint[0])
def test_lint_warns_P007(self, tmpdir): """Test warn flag for P007 - YAML file is not valid YAML.""" exclude_lint = self._exclude_all(except_code='P007') warn_lint = ['P007'] config.set_site_repo(self.site_yaml_path) p = tmpdir.mkdir(self.__class__.__name__).join("test.yaml") # Invalid YAML - will trigger error. p.write("---\nfoo: bar: baz") with mock.patch('pegleg.engine.util.files.all', autospec=True, return_value=[p.strpath]): result = lint.full( False, exclude_lint=exclude_lint, warn_lint=warn_lint) assert len(result) == 1 assert result[0].startswith(warn_lint[0])
def test_lint_warns_P001(*args): warn_lint = ['P001'] config.set_site_repo('../pegleg/site_yamls/') code_1 = 'X001' msg_1 = 'is a secret, but has unexpected storagePolicy: "cleartext"' code_2 = 'X002' msg_2 = 'test msg' msgs = [(code_1, msg_1), (code_2, msg_2)] with mock.patch.object(lint, '_verify_file_contents', return_value=msgs) as mock_methed: with pytest.raises(click.ClickException) as expected_exc: lint.full(False, [], warn_lint) assert msg_1 not in expected_exc assert msg_2 in expected_exc
def test_lint_excludes_P001(self, *args): """Test exclude flag for P001 - Document has storagePolicy cleartext (expected is encrypted) yet its schema is a mandatory encrypted type. """ exclude_lint = ['P001'] config.set_site_repo(self.site_yaml_path) code_1 = 'X001' msg_1 = 'is a secret, but has unexpected storagePolicy: "cleartext"' code_2 = 'X002' msg_2 = 'test msg' msgs = [(code_1, msg_1), (code_2, msg_2)] with mock.patch.object(lint, '_verify_file_contents', return_value=msgs) as mock_methed: with pytest.raises(click.ClickException) as expected_exc: lint.full(False, exclude_lint, []) assert msg_1 in expected_exc assert msg_2 in expected_exc
def test_failed_deckhand_validation(tmpdir): # Write the test data to temp file config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA)) base_config_dir = os.path.join(tmpdir, 'config_dir') config.set_site_repo(base_config_dir) config_dir = os.path.join(base_config_dir, 'site', 'test_site') config_path = os.path.join(config_dir, 'config_file.yaml') build_dir = os.path.join(tmpdir, 'build_dir') os.makedirs(config_dir) files.write(config_data, config_path) files.write( yaml.safe_load_all(SITE_DEFINITION), os.path.join(config_dir, "site-definition.yaml")) key = 'MyverYSecretEncryptionKey382803' with pytest.raises(GenesisBundleGenerateException, match=r'.*failed on deckhand validation.*'): bundle.build_genesis( build_path=build_dir, encryption_key=key, validators=False, debug=logging.ERROR, site_name="test_site")
def test_no_encryption_key(tmpdir): # Write the test data to temp file config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA)) base_config_dir = os.path.join(tmpdir, 'config_dir') config.set_site_repo(base_config_dir) config_dir = os.path.join(base_config_dir, 'site', 'test_site') config_path = os.path.join(config_dir, 'config_file.yaml') build_dir = os.path.join(tmpdir, 'build_dir') os.makedirs(config_dir) files.write(config_data, config_path) files.write( yaml.safe_load_all(SITE_DEFINITION), os.path.join(config_dir, "site-definition.yaml")) with pytest.raises(GenesisBundleEncryptionException, match=r'.*no encryption policy or key is specified.*'): bundle.build_genesis( build_path=build_dir, encryption_key=None, validators=False, debug=logging.ERROR, site_name="test_site")
def process_repositories(site_name): """Process and setup all repositories including ensuring we are at the right revision based on the site's own site-definition.yaml file. :param site_name: Site name for which to clone relevant repos. """ # Only tracks extra repositories - not the site (primary) repository. extra_repos = [] site_repo = process_site_repository() # Retrieve extra repo data from site-definition.yaml files. site_data = util.definition.load_as_params(site_name, primary_repo_base=site_repo) site_def_repos = _get_and_validate_site_repositories(site_name, site_data) # Dict mapping repository names to associated URL/revision info for clone. repo_overrides = _process_repository_overrides(site_def_repos) if not site_def_repos: LOG.info( 'No repositories found in site-definition.yaml for site: %s. ' 'Defaulting to specified repository overrides.', site_name) site_def_repos = repo_overrides # Extract user/key that we will use for all repositories. repo_key = config.get_repo_key() repo_user = config.get_repo_username() for repo_alias in site_def_repos.keys(): if repo_alias == "site": LOG.warning( "The primary site repository path must be specified " "via the -r flag. Ignoring the provided " "site-definition entry: %s", site_def_repos[repo_alias]) continue # Extract URL and revision, prioritizing overrides over the defaults in # the site-definition.yaml. if repo_alias in repo_overrides: repo_url_or_path = repo_overrides[repo_alias]['url'] repo_revision = repo_overrides[repo_alias]['revision'] else: repo_url_or_path = site_def_repos[repo_alias]['url'] repo_revision = site_def_repos[repo_alias]['revision'] repo_url_or_path = _format_url_with_repo_username(repo_url_or_path) LOG.info( "Processing repository %s with url=%s, repo_key=%s, " "repo_username=%s, revision=%s", repo_alias, repo_url_or_path, repo_key, repo_user, repo_revision) temp_extra_repo = _process_repository(repo_url_or_path, repo_revision) extra_repos.append(temp_extra_repo) # Overwrite the site repo and extra repos in the config because further # processing will fail if they contain revision info in their paths. LOG.debug("Updating site_repo=%s extra_repo_list=%s in config", site_repo, extra_repos) config.set_site_repo(site_repo) config.set_extra_repo_list(extra_repos)
def create_tmp_deployment_files(tmpdir): """Fixture that creates a temporary directory structure.""" sitenames = ['cicd', 'lab'] SITE_TEST_STRUCTURE = { 'directories': { 'secrets': { 'directories': { 'passphrases': { 'files': {} }, }, }, 'software': { 'directories': { 'charts': { 'files': {} }, }, }, }, 'files': {} } p = tmpdir.mkdir("deployment_files") config.set_site_repo(str(p)) # Create global directories and files. files._create_tree(os.path.join(str(p), 'global'), tree={ 'directories': { 'common': { 'files': { 'global-common.yaml': _gen_document(name="global-common", layer='global') } }, 'v1.0': { 'files': { 'global-v1.0.yaml': _gen_document(name="global-v1.0", layer='global') } } } }) # Create type directories and files. files._create_tree( os.path.join(str(p), 'type'), tree={ 'directories': { site: { 'directories': { 'common': { 'files': { '%s-type-common.yaml' % site: _gen_document(name="%s-type-common" % site, layer='type') } }, 'v1.0': { 'files': { '%s-type-v1.0.yaml' % site: _gen_document(name="%s-type-v1.0" % site, layer='type') } } } } for site in sitenames } }) # Create site directories and files. for site in sitenames: site_definition = """ --- data: repositories: global: revision: v1.0 url: http://nothing.com site_type: %s metadata: layeringDefinition: {abstract: false, layer: site} name: %s schema: metadata/Document/v1 storagePolicy: cleartext schema: pegleg/SiteDefinition/v1 """ % (site, site) test_structure = SITE_TEST_STRUCTURE.copy() test_structure['directories']['secrets']['directories']['passphrases'][ 'files'] = { '%s-passphrase.yaml' % site: _gen_document(name="%s-passphrase" % site, layer='site') } test_structure['directories']['software']['directories']['charts'][ 'files'] = { '%s-chart.yaml' % site: _gen_document(name="%s-chart" % site, layer='site') } test_structure['files']['site-definition.yaml'] = yaml.safe_load( site_definition) cicd_path = os.path.join(str(p), files._site_path(site)) files._create_tree(cicd_path, tree=test_structure) yield