def process_site_repository(update_config=False): """Process and setup site repository including ensuring we are at the right revision based on the site's own site-definition.yaml file. :param bool update_config: Whether to update Pegleg config with computed site repo path. """ # Retrieve the main site repository and validate it. site_repo_or_path = config.get_site_repo() if not site_repo_or_path: raise ValueError("Site repository directory (%s) must be specified" % site_repo_or_path) repo_url_or_path, repo_revision = _extract_repo_url_and_revision( site_repo_or_path) repo_url_or_path = _format_url_with_repo_username(repo_url_or_path) new_repo_path = _process_repository(repo_url_or_path, repo_revision) if update_config: # Overwrite the site repo in the config because further processing will # fail if they contain revision info in their paths. LOG.debug("Updating site_repo=%s in config", new_repo_path) config.set_site_repo(new_repo_path) return new_repo_path
def __init__(self, sitename, block_strings=True, author=None, duration=365, regenerate_all=False, save_location=None): """Constructor for ``PKIGenerator``. :param int duration: Duration in days that generated certificates are valid. :param str sitename: Site name for which to retrieve documents used for certificate and keypair generation. :param bool block_strings: Whether to dump out certificate data as block-style YAML string. Defaults to true. :param str author: Identifying name of the author generating new certificates. :param bool regenerate_all: If Pegleg should regenerate all certs. """ self._regenerate_all = regenerate_all self._sitename = sitename self._documents = site.get_rendered_docs(sitename) self._author = author self._save_location = save_location or config.get_site_repo() self.keys = pki_utility.PKIUtility(block_strings=block_strings, duration=duration) self.outputs = collections.defaultdict(dict) # Maps certificates to CAs in order to derive certificate paths. self._cert_to_ca_map = {}
def _collect_to_file(site_name, save_location): """Collects all documents related to ``site_name`` and outputs them to the file denoted by ``save_location``. """ files.check_file_save_location(save_location) save_files = dict() curr_site_repo = files.path_leaf(config.get_site_repo()) try: for repo_base, filename in util.definition.site_files_by_repo( site_name): repo_name = os.path.normpath(repo_base).split(os.sep)[-1] save_file = os.path.join(save_location, repo_name + '.yaml') if repo_name not in save_files: save_files[repo_name] = open(save_file, 'w') LOG.debug("Collecting file %s to file %s", filename, save_file) save_files[repo_name].writelines(_read_and_format_yaml(filename)) add_representer_ordered_dict() save_files[curr_site_repo].writelines( yaml.safe_dump(get_deployment_data_doc(site_name), default_flow_style=False, explicit_start=True, explicit_end=True)) except Exception as ex: raise click.ClickException("Error saving output: %s" % str(ex)) finally: for f in save_files.values(): f.close()
def __wrap(secrets_document, generated=False, catalog=None, author=None): """ Embeds a valid deckhand document in a pegleg managed document. :param secrets_document: secrets document to be embedded in a pegleg managed document. :type secrets_document: dict :param bool generated: A flag to indicate the documents are auto-generated by pegleg (True), or manually created (False). :return: pegleg manged document with the wrapped original secrets document. :rtype: dict """ layer = secrets_document.get('metadata', {}).get('layeringDefinition', {}).get('layer', DEFAULT_LAYER) layering_definition = OrderedDict( [('abstract', False), ('layer', layer)]) metadata = OrderedDict( [ ( 'name', '{}/{}'.format( secrets_document['schema'], secrets_document['metadata']['name'])), ('schema', 'metadata/Document/v1'), ('labels', secrets_document['metadata'].get('labels', {})), ('layeringDefinition', layering_definition), ('storagePolicy', 'cleartext') ]) data = OrderedDict( [ ( 'managedDocument', OrderedDict( [ ('schema', secrets_document['schema']), ('metadata', secrets_document['metadata']), ('data', secrets_document['data']) ])) ]) doc = OrderedDict( [ ('schema', PEGLEG_MANAGED_SCHEMA), ('metadata', metadata), ('data', data) ]) if generated: doc['data'][GENERATED] = { 'at': datetime.utcnow().isoformat(), 'by': author, 'specifiedBy': { 'repo': git.repo_url(config.get_site_repo()), 'reference': config.get_site_rev() or 'master', 'path': catalog.catalog_path, }, } return doc
def test_read_incompatible_file(self, temp_deployment_files): # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a # Deckhand-formatted document currently but probably shouldn't be, # because it has no business being in Deckhand. As such, validate that # it is ignored. path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'site-definition.yaml') documents = files.read(path) assert not documents, ("Documents returned should be empty for " "site-definition.yaml")
def list_types(primary_repo_base=None): """Get a list of type directories in the primary repo.""" if not primary_repo_base: primary_repo_base = config.get_site_repo() full_type_path = os.path.join(primary_repo_base, config.get_rel_type_path()) for path in os.listdir(full_type_path): joined_path = os.path.join(full_type_path, path) if os.path.isdir(joined_path): yield path
def _set_catalog_path(self): repo_name = git.repo_url(config.get_site_repo()) catalog_name = self._get_document_name('{}.yaml'.format(self._kind)) for file_path in definition.site_files(self.site_name): if file_path.endswith(catalog_name): self._catalog_path.append(file_path) if not self._catalog_path: # Cound not find the Catalog for this generated passphrase # raise an exception. LOG.error('Catalog path: {} was not found in repo: {}'.format( catalog_name, repo_name)) raise PassphraseCatalogNotFoundException()
def test_write(self, temp_deployment_files): path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'test_out.yaml') files.write("test text", path) with open(path, "r") as out_fi: assert out_fi.read() == "test text" files.write({"a": 1}, path) with open(path, "r") as out_fi: assert yaml.safe_load(out_fi) == {"a": 1} files.write([{"a": 1}], path) with open(path, "r") as out_fi: assert list(yaml.safe_load_all(out_fi)) == [{"a": 1}] with pytest.raises(ValueError): files.write(object(), path)
def run_encrypt(author, save_location, site_name, path=None): """Wraps and encrypts site secret documents :param author: identifies author generating new certificates for tracking information :param save_location: path to save encrypted documents to, if None the original documents are overwritten :param site_name: site name to process :param path: path to the document(s) to encrypt :return: """ config.set_global_enc_keys(site_name) if save_location is None and path is None: save_location = config.get_site_repo() engine.secrets.encrypt(save_location, author, site_name=site_name, path=path)
def create_tmp_pki_structure(tmpdir): """Fixture that creates a temporary site directory structure include pki/ subfolder for validating PKIGenerator logic. :returns: Function pointer, which, when called, creates a temporary file structure with pki/ subfolder. """ def _create_tmp_folder_system(sitename, pki_catalog): """Creates a temporary site folder system. :param str sitename: Name of the site. :param str pki_catalog: YAML-formatted string that adheres to pki-catalog.yaml structure. """ # Create site directories and files. p = tmpdir.mkdir("deployment_files") config.set_site_repo(p.strpath) site_definition = copy.deepcopy(_SITE_DEFINITION) site_definition = site_definition % {'sitename': sitename} pki_catalog = copy.deepcopy(pki_catalog) pki_catalog = pki_catalog.format(sitename=sitename) test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE) test_structure['files']['site-definition.yaml'] = yaml.safe_load( site_definition) test_structure['files']['layering-definition.yaml'] = yaml.safe_load( _LAYERING_DEFINITION) test_structure['directories']['pki']['files'][ 'pki-catalog.yaml'] = yaml.safe_load(pki_catalog) test_path = os.path.join(p.strpath, files._site_path(sitename)) files._create_tree(test_path, tree=test_structure) return p.strpath try: yield _create_tmp_folder_system finally: temp_path = config.get_site_repo() if temp_path != './' and os.path.exists(temp_path): shutil.rmtree(temp_path, ignore_errors=True)
def create_tmp_site_structure(tmpdir): """Fixture that creates a temporary site directory structure :returns: Function pointer, which, when called, creates a temporary file structure. """ def _create_tmp_folder_system(sitename): """Creates a temporary site folder system. :param str sitename: Name of the site. """ # Create site directories and files. p = tmpdir.mkdir("deployment_files") config.set_site_repo(p.strpath) site_definition = copy.deepcopy(_SITE_DEFINITION) site_definition = site_definition % {'sitename': sitename} test_structure = copy.deepcopy(_SITE_TEST_STRUCTURE) test_structure['files']['site-definition.yaml'] = yaml.safe_load( site_definition) test_structure['files']['layering-definition.yaml'] = yaml.safe_load( _LAYERING_DEFINITION) test_structure['directories']['secrets']['directories']['passphrases'][ 'files']['plaintext.yaml'] = yaml.safe_load(_PLAINTEXT_SECRET) test_structure['directories']['secrets']['directories']['passphrases'][ 'files']['managed.yaml'] = yaml.safe_load(_MANAGED_SECRET) test_structure['directories']['secrets']['directories']['passphrases'][ 'files']['encrypted.yaml'] = yaml.safe_load(_ENCRYPTED_SECRET) test_path = os.path.join(p.strpath, files._site_path(sitename)) files._create_tree(test_path, tree=test_structure) return p.strpath try: yield _create_tmp_folder_system finally: temp_path = config.get_site_repo() if temp_path != './' and os.path.exists(temp_path): shutil.rmtree(temp_path, ignore_errors=True)
def test_read_compatible_file(self, temp_deployment_files): path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets', 'passphrases', 'cicd-passphrase.yaml') documents = files.read(path) assert 1 == len(documents)
def path(site_name, primary_repo_base=None): """Retrieve path to the site-definition.yaml file for ``site_name``.""" if not primary_repo_base: primary_repo_base = config.get_site_repo() return os.path.join(primary_repo_base, 'site', site_name, 'site-definition.yaml')
def path(site_name, primary_repo_base=None): if not primary_repo_base: primary_repo_base = config.get_site_repo() return os.path.join(primary_repo_base, 'site', site_name, 'site-definition.yaml')
def test_dir_permissions(self, temp_deployment_files): path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'test_dir') os.makedirs(path) assert oct(os.stat(path).st_mode & 0o777) == EXPECTED_DIR_PERM
def test_file_permissions(self, temp_deployment_files): path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'test_out.yaml') files.write("test text", path) assert oct(os.stat(path).st_mode & 0o777) == EXPECTED_FILE_PERM
def _get_repo_url_and_rev(): repo_path_or_url = config.get_site_repo() repo_url = git.repo_url(repo_path_or_url) repo_rev = config.get_site_rev() return repo_url, repo_rev