def get_ref_pkg_list(cluster_conf, tmp_dpath): """Get the current reference package list. :return: list, JSON-formatted data """ dstor_root_url = (cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_pkglist_path = (cluster_conf.get('distribution-storage', {}).get('pkglistpath', '')) # Unblock irrelevant local operations if not cluster_conf or dstor_pkglist_path == 'NOP': return [] rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path) try: rpl_fpath = cm_utl.download(rpl_url, tmp_dpath) LOG.debug(f'Reference package list: Download: {rpl_fpath}: {rpl_url}') except Exception as e: raise cr_exc.RCDownloadError( f'Reference package list: Download: {rpl_fpath}: {rpl_url}:' f' {type(e).__name__}: {e}') from e try: return cr_utl.rc_load_json( rpl_fpath, emheading=f'Reference package list: {rpl_fpath}') except cr_exc.RCError as e: raise e finally: rpl_fpath.unlink()
def get_ref_pkg_list(self): """Get the current reference package list. :return: list, JSON-formatted data """ dstor_root_url = (self.cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_pkglist_path = (self.cluster_conf.get('distribution-storage', {}).get('pkglistpath', '')) # Unblock irrelevant local operations if self.cluster_conf_nop or dstor_pkglist_path == 'NOP': LOG.info(f'{self.msg_src}: ref_pkg_list: NOP') return [] rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path) rpl_fname = Path(dstor_pkglist_path).name try: cm_utl.download(rpl_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: Reference package list: Download:' f' {rpl_fname}: {rpl_url}') except Exception as e: raise cr_exc.RCDownloadError( f'Reference package list: Download: {rpl_fname}: {rpl_url}:' f' {type(e).__name__}: {e}') from e rpl_fpath = self.inst_storage.tmp_dpath.joinpath(rpl_fname) try: return cr_utl.rc_load_json( rpl_fpath, emheading=f'Reference package list: {rpl_fname}') except cr_exc.RCError as e: raise e finally: rpl_fpath.unlink()
def load(cls, fpath): """Load package manifest from a file. :param fpath: pathlib.Path, path to a JSON-formatted manifest file. :return: dict, package manifest. """ m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest') # TODO: Add content verification (jsonschema) for m_body. Raise # ValueError, if conformance was not confirmed. try: manifest = cls( pkg_id=PackageId(pkg_id=m_body.get('pkg_id')), istor_nodes=IStorNodes( **{ k: Path(v) for k, v in m_body.get('context').get( 'istor_nodes').items() }), cluster_conf=m_body.get('context').get('cluster_conf'), pkg_info=m_body.get('pkg_info'), pkg_extcfg=m_body.get('pkg_extcfg'), pkg_svccfg=m_body.get('pkg_svccfg'), ) LOG.debug(f'Package manifest: Load: {fpath}') except (ValueError, AssertionError, TypeError) as e: err_msg = (f'Package manifest: Load:' f' {fpath}: {type(e).__name__}: {e}') raise cr_exc.RCInvalidError(err_msg) from e return manifest
def mock_get_dcos_conf(cls): template_fpath = Path( cls.stubs_location()).joinpath('dcos-config-windows.yaml') template = load_dcos_conf_template(template_fpath) values = utils.rc_load_json( Path(cls.stubs_location()).joinpath('expanded.config.full.json'), emheading=f'DC/OS aggregated config: Values') return {'template': template, 'values': values}
def _load_pkg_info(self): """Load package info descriptor from a file. :return: dict, package info descriptor """ fpath = getattr(self._istor_nodes, ISTOR_NODE.PKGREPO).joinpath(self._pkg_id.pkg_id, cr_const.PKG_INFO_FPATH) try: pkg_info = cr_utl.rc_load_json(fpath, emheading='Package info descriptor', render=True, context=self._context) except cr_exc.RCNotFoundError: pkg_info = {} return pkg_info
def get_ref_pkg_list(self): """Get the current reference package list. :return: list, JSON-formatted data """ # TODO: Functionality implemented in this method needs to be reused # in other application parts (e.g. CmdConfigUpgrade) and so, it # has been arranged as a standalone function get_ref_pkg_list(). # Thus the CmdConfigSetup is to be moved to use that standalone # function instead of this method to avoid massive code # duplication. dstor_root_url = (self.cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_pkglist_path = (self.cluster_conf.get('distribution-storage', {}).get('pkglistpath', '')) # Unblock irrelevant local operations if self.cluster_conf_nop or dstor_pkglist_path == 'NOP': LOG.info(f'{self.msg_src}: ref_pkg_list: NOP') return [] rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path) rpl_fname = Path(dstor_pkglist_path).name try: cm_utl.download(rpl_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: Reference package list: Download:' f' {rpl_fname}: {rpl_url}') except Exception as e: raise cr_exc.RCDownloadError( f'Reference package list: Download: {rpl_fname}: {rpl_url}:' f' {type(e).__name__}: {e}') from e rpl_fpath = self.inst_storage.tmp_dpath.joinpath(rpl_fname) try: return cr_utl.rc_load_json( rpl_fpath, emheading=f'Reference package list: {rpl_fname}') except cr_exc.RCError as e: raise e finally: rpl_fpath.unlink()
def load(cls, fpath): """Load package manifest from a file. :param fpath: pathlib.Path, path to a JSON-formatted manifest file. :return: dict, package manifest. """ m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest') try: manifest = cls( pkg_id=PackageId(pkg_id=m_body.get('pkg_id')), pkgrepo_dpath=Path(m_body.get('pkgrepo_dpath')), pkgactive_dpath=Path(m_body.get('pkgactive_dpath')), pkg_info=m_body.get('pkg_info'), pkg_ini=m_body.get('pkg_ini'), svc_conf=m_body.get('svc_conf'), ) LOG.debug(f'Package manifest: Load: {fpath}') except (ValueError, AssertionError) as e: err_msg = (f'Package manifest: Load:' f' {fpath}: {type(e).__name__}: {e}') raise cr_exc.RCInvalidError(err_msg) return manifest
def get_dstor_dcoscfgpkg_path(self, dstor_root_url: str, dstor_lpi_path: str): """Retrieve the Linux Package Index (LPI) object from the DC/OS distribution storage and discover a relative URL to the DC/OS aggregated configuration package. LPI is expected to be a JSON-formatted file containing descriptors for DC/OS distribution packages: { "<pkg-name>":{ "filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz", "id":"<pkg-name>--<pkg-version>" }, ... } :param dstor_root_url: str, DC/OS distribution storage root URL :param dstor_lpi_path: str, URL path to the DC/OS Linux package index object at the DC/OS distribution storage :return dstor_dcoscfgpkg_path: str, URL path to the DC/OS aggregated config package at the DC/OS distribution storage """ # TODO: Functionality implemented in this method needs to be reused # in other application parts (e.g. CmdConfigUpgrade) and so, it # has been arranged as a standalone function # get_dstor_dcoscfgpkg_path(). # Thus the CmdConfigSetup is to be moved to use that standalone # function instead of this method to avoid massive code # duplication. dcos_conf_pkg_name = 'dcos-config-win' # Linux package index direct URL lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path) try: lpi_fpath = cm_utl.download(lpi_url, self.inst_storage.tmp_dpath) LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:' f' {lpi_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}:' f' {e}') from e try: lpi = cr_utl.rc_load_json(lpi_fpath, emheading='DC/OS Linux package index') if not isinstance(lpi, dict): raise cr_exc.RCInvalidError( f'DC/OS Linux package index: {lpi_url}: Invalid structure') dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name) if dcos_conf_pkg_desc is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor is missed:' f' {dcos_conf_pkg_name}') if not isinstance(dcos_conf_pkg_desc, dict): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS' f' aggregated config package descriptor:' f' {dcos_conf_pkg_desc}') dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename') if dstor_dcoscfgpkg_path is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path is' f' missed: {dcos_conf_pkg_desc}') if not isinstance(dstor_dcoscfgpkg_path, str): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path:' f' Invalid type: {dstor_dcoscfgpkg_path}') finally: lpi_fpath.unlink() return dstor_dcoscfgpkg_path
def get_dcos_conf(self): """Get the DC/OS aggregated configuration object. :return: dict, set of DC/OS shared and package specific configuration templates coupled with 'key=value' substitution data container: { 'template': { 'package': [ {'path': <str>, 'content': <str>}, ... ] }, 'values': { key: value, ... } } """ # TODO: Functionality implemented in this method needs to be reused # in other application parts (e.g. CmdConfigUpgrade) and so, it # has been arranged as a standalone function get_dcos_conf(). # Thus the CmdConfigSetup is to be moved to use that standalone # function instead of this method to avoid massive code # duplication. dstor_root_url = self.root_url dstor_linux_pkg_index_path = self.dcosclusterpkginfopath template_fname = 'dcos-config-windows.yaml' values_fname = 'expanded.config.full.json' # Unblock irrelevant local operations if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP': LOG.info(f'{self.msg_src}: dcos_conf: NOP') return {} # Discover relative URL to the DC/OS aggregated configuration package. dstor_dcoscfg_pkg_path = self.get_dstor_dcoscfgpkg_path( dstor_root_url, dstor_linux_pkg_index_path) dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path) # Download DC/OS aggregated configuration package ... try: dcoscfg_pkg_fpath = cm_utl.download(dcoscfg_pkg_url, self.inst_storage.tmp_dpath) LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:' f' Download: {dcoscfg_pkg_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS aggregated config package: {dcoscfg_pkg_url}:' f' {type(e).__name__}: {e}') from e # Process DC/OS aggregated configuration package. try: with tf.TemporaryDirectory( dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath: cm_utl.unpack(dcoscfg_pkg_fpath, tmp_dpath) LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:' f' {dcoscfg_pkg_fpath}: Extract: OK') values_fpath = Path(tmp_dpath).joinpath(values_fname) values = cr_utl.rc_load_json( values_fpath, emheading=f'DC/OS aggregated config: Values') template_fpath = Path(tmp_dpath).joinpath(template_fname) template = self.load_dcos_conf_template(template_fpath) except Exception as e: if not isinstance(e, cr_exc.RCError): raise cr_exc.RCExtractError( f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:' f' {type(e).__name__}: {e}') else: raise else: LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:' f' {dcoscfg_pkg_fpath}: Preprocess: OK') return {'template': template, 'values': values} finally: dcoscfg_pkg_fpath.unlink()
def test_rc_load_json_should_return_dict(*args): """Check does rc_load_json output equal json.load output.""" data = utils.rc_load_json(Path()) assert data == {}
def test_rc_load_json_template_should_return_dict(mock_template, *args): """Check empty json transformation to dict.""" mock_template().render.return_value = '{}' json = utils.rc_load_json(Path(), render=True) assert json == {}
def test_rc_load_template_issue_should_fail(*args): """Check template errors handling.""" with pytest.raises(RCInvalidError): utils.rc_load_json(Path(), render=True)
def get_dcos_conf(self): """Get the DC/OS aggregated configuration object. :return: dict, set of DC/OS shared and package specific configuration templates coupled with 'key=value' substitution data container: { 'template': { 'package': [ {'path': <str>, 'content': <str>}, ... ] }, 'values': { key: value, ... } } """ dstor_root_url = (self.cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_linux_pkg_index_path = (self.cluster_conf.get( 'distribution-storage', {}).get('dcosclusterpkginfopath', '')) dcos_conf_pkg_name = 'dcos-config-win' template_fname = 'dcos-config-windows.yaml' values_fname = 'expanded.config.full.json' # Unblock irrelevant local operations if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP': LOG.info(f'{self.msg_src}: dcos_conf: NOP') return {} # Linux package index direct URL lpi_url = posixpath.join(dstor_root_url, dstor_linux_pkg_index_path) lpi_fname = Path(dstor_linux_pkg_index_path).name try: cm_utl.download(lpi_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:' f' {lpi_fname}: {lpi_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS Linux package index: Download: {lpi_fname}:' f' {lpi_url}: {type(e).__name__}: {e}') from e lpi_fpath = self.inst_storage.tmp_dpath.joinpath(lpi_fname) try: lpi = cr_utl.rc_load_json( lpi_fpath, emheading=f'DC/OS Linux package index: {lpi_fname}') if (not isinstance(lpi, dict) or not isinstance(lpi.get(dcos_conf_pkg_name), dict)): raise cr_exc.RCInvalidError( f'DC/OS Linux package index: {lpi}') dstor_dcoscfg_pkg_path = lpi.get(dcos_conf_pkg_name).get( 'filename') if not isinstance(dstor_dcoscfg_pkg_path, str): raise cr_exc.RCElementError( f'DC/OS Linux package index: DC/OS config package' f' distribution storage path: {dstor_dcoscfg_pkg_path}') except cr_exc.RCError as e: raise e finally: lpi_fpath.unlink() dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path) dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name # Download DC/OS aggregated configuration package ... try: cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:' f' {dcoscfg_pkg_fname}: {dcoscfg_pkg_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS aggregated config: Download: {dcoscfg_pkg_fname}:' f' {dcoscfg_pkg_url}: {type(e).__name__}: {e}') from e dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath( dcoscfg_pkg_fname) try: with tf.TemporaryDirectory( dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath: cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath) LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Extract:' f' OK') values_fpath = Path(tmp_dpath).joinpath(values_fname) values = cr_utl.rc_load_json( values_fpath, emheading=f'DC/OS aggregated config: Values: {values_fname}' ) template_fpath = Path(tmp_dpath).joinpath(template_fname) template = self.load_dcos_conf_templete(template_fpath) except Exception as e: if not isinstance(e, cr_exc.RCError): raise cr_exc.RCExtractError( f'DC/OS aggregated config: {type(e).__name__}: {e}') else: raise else: return {'template': template, 'values': values} finally: dcoscfg_pkg_fpath.unlink()
def test_rc_load_unavailable_template_should_fail(*args): """Check template not found issue error handling.""" with pytest.raises(RCNotFoundError): utils.rc_load_json(Path(), render=True)
def _load_pkg_info(self): """Load package info descriptor from a file.""" fpath = self.pkgrepo_dpath.joinpath(str(self.pkg_id), self._pkginfo_fpath) return cr_utl.rc_load_json(fpath, emheading='Package info descriptor')
def get_dcos_conf(cluster_conf, tmp_dpath: Path): """Get the DC/OS aggregated configuration object. :return: dict, set of DC/OS shared and package specific configuration templates coupled with 'key=value' substitution data container: { 'template': { 'package': [ {'path': <str>, 'content': <str>}, ... ] }, 'values': { key: value, ... } } """ dstor_root_url = (cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_linux_pkg_index_path = (cluster_conf.get( 'distribution-storage', {}).get('dcosclusterpkginfopath', '')) template_fname = 'dcos-config-windows.yaml' values_fname = 'expanded.config.full.json' # Unblock irrelevant local operations if not cluster_conf or dstor_linux_pkg_index_path == 'NOP': return {} # Discover relative URL to the DC/OS aggregated configuration package. dstor_dcoscfg_pkg_path = get_dstor_dcoscfgpkg_path( dstor_root_url, dstor_linux_pkg_index_path, tmp_dpath) # Download DC/OS aggregated configuration package ... dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path) try: dcoscfg_pkg_fpath = cm_utl.download(dcoscfg_pkg_url, tmp_dpath) LOG.debug(f'DC/OS aggregated config package:' f' Download: {dcoscfg_pkg_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS aggregated config package: {dcoscfg_pkg_url}:' f' {type(e).__name__}: {e}') from e # Process DC/OS aggregated configuration package. try: with tf.TemporaryDirectory(dir=str(tmp_dpath)) as tmp_dpath_: cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath_) LOG.debug(f'DC/OS aggregated config package:' f' {dcoscfg_pkg_fpath}: Extract: OK') values_fpath = Path(tmp_dpath_).joinpath(values_fname) values = cr_utl.rc_load_json( values_fpath, emheading=f'DC/OS aggregated config: Values') template_fpath = Path(tmp_dpath_).joinpath(template_fname) template = load_dcos_conf_template(template_fpath) except Exception as e: if not isinstance(e, cr_exc.RCError): raise cr_exc.RCExtractError( f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:' f' {type(e).__name__}: {e}') else: raise else: LOG.debug(f'DC/OS aggregated config package:' f' {dcoscfg_pkg_fpath}: Preprocess: OK') return {'template': template, 'values': values} finally: dcoscfg_pkg_fpath.unlink()
def get_dstor_dcoscfgpkg_path(dstor_root_url: str, dstor_lpi_path: str, tmp_dpath: str): """Retrieve the Linux Package Index (LPI) object from the DC/OS distribution storage and discover a relative URL to the DC/OS aggregated configuration package. LPI is expected to be a JSON-formatted file containing descriptors for DC/OS distribution packages: { "<pkg-name>":{ "filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz", "id":"<pkg-name>--<pkg-version>" }, ... } :param dstor_root_url: str, DC/OS distribution storage root URL :param dstor_lpi_path: str, URL path to the DC/OS Linux package index object at the DC/OS distribution storage :return tmp_dpath: str, URL path to the DC/OS aggregated config package at the DC/OS distribution storage """ dcos_conf_pkg_name = 'dcos-config-win' # Linux package index direct URL lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path) try: lpi_fpath = cm_utl.download(lpi_url, tmp_dpath) LOG.debug(f'DC/OS Linux package index: Download: {lpi_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}: {e}' ) from e try: lpi = cr_utl.rc_load_json(lpi_fpath, emheading='DC/OS Linux package index') if not isinstance(lpi, dict): raise cr_exc.RCInvalidError( f'DC/OS Linux package index: {lpi_url}: Invalid structure') dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name) if dcos_conf_pkg_desc is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor is missed:' f' {dcos_conf_pkg_name}') if not isinstance(dcos_conf_pkg_desc, dict): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS' f' aggregated config package descriptor:' f' {dcos_conf_pkg_desc}') dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename') if dstor_dcoscfgpkg_path is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path is' f' missed: {dcos_conf_pkg_desc}') if not isinstance(dstor_dcoscfgpkg_path, str): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path:' f' Invalid type: {dstor_dcoscfgpkg_path}') finally: lpi_fpath.unlink() return dstor_dcoscfgpkg_path
def test_rc_load_directory_should_fail(*args): """Check error validation when source is directory.""" with pytest.raises(RCError): utils.rc_load_json(Path())