def get_ref_pkg_list(cluster_conf, tmp_dpath):
"""Get the current reference package list.
:return: list, JSON-formatted data
"""
dstor_root_url = (cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_pkglist_path = (cluster_conf.get('distribution-storage',
{}).get('pkglistpath', ''))
# Unblock irrelevant local operations
if not cluster_conf or dstor_pkglist_path == 'NOP':
return []
rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path)
try:
rpl_fpath = cm_utl.download(rpl_url, tmp_dpath)
LOG.debug(f'Reference package list: Download: {rpl_fpath}: {rpl_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Reference package list: Download: {rpl_fpath}: {rpl_url}:'
f' {type(e).__name__}: {e}') from e
try:
return cr_utl.rc_load_json(
rpl_fpath, emheading=f'Reference package list: {rpl_fpath}')
except cr_exc.RCError as e:
raise e
finally:
rpl_fpath.unlink()
def get_ref_pkg_list(self):
"""Get the current reference package list.
:return: list, JSON-formatted data
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_pkglist_path = (self.cluster_conf.get('distribution-storage',
{}).get('pkglistpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_pkglist_path == 'NOP':
LOG.info(f'{self.msg_src}: ref_pkg_list: NOP')
return []
rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path)
rpl_fname = Path(dstor_pkglist_path).name
try:
cm_utl.download(rpl_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: Reference package list: Download:'
f' {rpl_fname}: {rpl_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Reference package list: Download: {rpl_fname}: {rpl_url}:'
f' {type(e).__name__}: {e}') from e
rpl_fpath = self.inst_storage.tmp_dpath.joinpath(rpl_fname)
try:
return cr_utl.rc_load_json(
rpl_fpath, emheading=f'Reference package list: {rpl_fname}')
except cr_exc.RCError as e:
raise e
finally:
rpl_fpath.unlink()
def load(cls, fpath):
"""Load package manifest from a file.
:param fpath: pathlib.Path, path to a JSON-formatted manifest file.
:return: dict, package manifest.
"""
m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest')
# TODO: Add content verification (jsonschema) for m_body. Raise
# ValueError, if conformance was not confirmed.
try:
manifest = cls(
pkg_id=PackageId(pkg_id=m_body.get('pkg_id')),
istor_nodes=IStorNodes(
**{
k: Path(v)
for k, v in m_body.get('context').get(
'istor_nodes').items()
}),
cluster_conf=m_body.get('context').get('cluster_conf'),
pkg_info=m_body.get('pkg_info'),
pkg_extcfg=m_body.get('pkg_extcfg'),
pkg_svccfg=m_body.get('pkg_svccfg'),
)
LOG.debug(f'Package manifest: Load: {fpath}')
except (ValueError, AssertionError, TypeError) as e:
err_msg = (f'Package manifest: Load:'
f' {fpath}: {type(e).__name__}: {e}')
raise cr_exc.RCInvalidError(err_msg) from e
return manifest
def mock_get_dcos_conf(cls):
template_fpath = Path(
cls.stubs_location()).joinpath('dcos-config-windows.yaml')
template = load_dcos_conf_template(template_fpath)
values = utils.rc_load_json(
Path(cls.stubs_location()).joinpath('expanded.config.full.json'),
emheading=f'DC/OS aggregated config: Values')
return {'template': template, 'values': values}
def _load_pkg_info(self):
"""Load package info descriptor from a file.
:return: dict, package info descriptor
"""
fpath = getattr(self._istor_nodes,
ISTOR_NODE.PKGREPO).joinpath(self._pkg_id.pkg_id,
cr_const.PKG_INFO_FPATH)
try:
pkg_info = cr_utl.rc_load_json(fpath,
emheading='Package info descriptor',
render=True,
context=self._context)
except cr_exc.RCNotFoundError:
pkg_info = {}
return pkg_info
def get_ref_pkg_list(self):
"""Get the current reference package list.
:return: list, JSON-formatted data
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function get_ref_pkg_list().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_pkglist_path = (self.cluster_conf.get('distribution-storage',
{}).get('pkglistpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_pkglist_path == 'NOP':
LOG.info(f'{self.msg_src}: ref_pkg_list: NOP')
return []
rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path)
rpl_fname = Path(dstor_pkglist_path).name
try:
cm_utl.download(rpl_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: Reference package list: Download:'
f' {rpl_fname}: {rpl_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Reference package list: Download: {rpl_fname}: {rpl_url}:'
f' {type(e).__name__}: {e}') from e
rpl_fpath = self.inst_storage.tmp_dpath.joinpath(rpl_fname)
try:
return cr_utl.rc_load_json(
rpl_fpath, emheading=f'Reference package list: {rpl_fname}')
except cr_exc.RCError as e:
raise e
finally:
rpl_fpath.unlink()
def load(cls, fpath):
"""Load package manifest from a file.
:param fpath: pathlib.Path, path to a JSON-formatted manifest file.
:return: dict, package manifest.
"""
m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest')
try:
manifest = cls(
pkg_id=PackageId(pkg_id=m_body.get('pkg_id')),
pkgrepo_dpath=Path(m_body.get('pkgrepo_dpath')),
pkgactive_dpath=Path(m_body.get('pkgactive_dpath')),
pkg_info=m_body.get('pkg_info'),
pkg_ini=m_body.get('pkg_ini'),
svc_conf=m_body.get('svc_conf'),
)
LOG.debug(f'Package manifest: Load: {fpath}')
except (ValueError, AssertionError) as e:
err_msg = (f'Package manifest: Load:'
f' {fpath}: {type(e).__name__}: {e}')
raise cr_exc.RCInvalidError(err_msg)
return manifest
def get_dstor_dcoscfgpkg_path(self, dstor_root_url: str,
dstor_lpi_path: str):
"""Retrieve the Linux Package Index (LPI) object from the DC/OS
distribution storage and discover a relative URL to the DC/OS
aggregated configuration package.
LPI is expected to be a JSON-formatted file containing descriptors for
DC/OS distribution packages:
{
"<pkg-name>":{
"filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz",
"id":"<pkg-name>--<pkg-version>"
},
...
}
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_lpi_path: str, URL path to the DC/OS Linux package
index object at the DC/OS distribution
storage
:return dstor_dcoscfgpkg_path: str, URL path to the DC/OS aggregated
config package at the DC/OS distribution
storage
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function
# get_dstor_dcoscfgpkg_path().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dcos_conf_pkg_name = 'dcos-config-win'
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path)
try:
lpi_fpath = cm_utl.download(lpi_url, self.inst_storage.tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}:'
f' {e}') from e
try:
lpi = cr_utl.rc_load_json(lpi_fpath,
emheading='DC/OS Linux package index')
if not isinstance(lpi, dict):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi_url}: Invalid structure')
dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name)
if dcos_conf_pkg_desc is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor is missed:'
f' {dcos_conf_pkg_name}')
if not isinstance(dcos_conf_pkg_desc, dict):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS'
f' aggregated config package descriptor:'
f' {dcos_conf_pkg_desc}')
dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename')
if dstor_dcoscfgpkg_path is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path is'
f' missed: {dcos_conf_pkg_desc}')
if not isinstance(dstor_dcoscfgpkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path:'
f' Invalid type: {dstor_dcoscfgpkg_path}')
finally:
lpi_fpath.unlink()
return dstor_dcoscfgpkg_path
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function get_dcos_conf().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dstor_root_url = self.root_url
dstor_linux_pkg_index_path = self.dcosclusterpkginfopath
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Discover relative URL to the DC/OS aggregated configuration package.
dstor_dcoscfg_pkg_path = self.get_dstor_dcoscfgpkg_path(
dstor_root_url, dstor_linux_pkg_index_path)
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
# Download DC/OS aggregated configuration package ...
try:
dcoscfg_pkg_fpath = cm_utl.download(dcoscfg_pkg_url,
self.inst_storage.tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' Download: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config package: {dcoscfg_pkg_url}:'
f' {type(e).__name__}: {e}') from e
# Process DC/OS aggregated configuration package.
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(dcoscfg_pkg_fpath, tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Extract: OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath, emheading=f'DC/OS aggregated config: Values')
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_template(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:'
f' {type(e).__name__}: {e}')
else:
raise
else:
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Preprocess: OK')
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
def test_rc_load_json_should_return_dict(*args):
"""Check does rc_load_json output equal json.load output."""
data = utils.rc_load_json(Path())
assert data == {}
def test_rc_load_json_template_should_return_dict(mock_template, *args):
"""Check empty json transformation to dict."""
mock_template().render.return_value = '{}'
json = utils.rc_load_json(Path(), render=True)
assert json == {}
def test_rc_load_template_issue_should_fail(*args):
"""Check template errors handling."""
with pytest.raises(RCInvalidError):
utils.rc_load_json(Path(), render=True)
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (self.cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
dcos_conf_pkg_name = 'dcos-config-win'
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_linux_pkg_index_path)
lpi_fname = Path(dstor_linux_pkg_index_path).name
try:
cm_utl.download(lpi_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_fname}: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: Download: {lpi_fname}:'
f' {lpi_url}: {type(e).__name__}: {e}') from e
lpi_fpath = self.inst_storage.tmp_dpath.joinpath(lpi_fname)
try:
lpi = cr_utl.rc_load_json(
lpi_fpath, emheading=f'DC/OS Linux package index: {lpi_fname}')
if (not isinstance(lpi, dict)
or not isinstance(lpi.get(dcos_conf_pkg_name), dict)):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi}')
dstor_dcoscfg_pkg_path = lpi.get(dcos_conf_pkg_name).get(
'filename')
if not isinstance(dstor_dcoscfg_pkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: DC/OS config package'
f' distribution storage path: {dstor_dcoscfg_pkg_path}')
except cr_exc.RCError as e:
raise e
finally:
lpi_fpath.unlink()
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name
# Download DC/OS aggregated configuration package ...
try:
cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_pkg_fname}: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_pkg_fname}:'
f' {dcoscfg_pkg_url}: {type(e).__name__}: {e}') from e
dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath(
dcoscfg_pkg_fname)
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Extract:'
f' OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath,
emheading=f'DC/OS aggregated config: Values: {values_fname}'
)
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_templete(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config: {type(e).__name__}: {e}')
else:
raise
else:
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
def test_rc_load_unavailable_template_should_fail(*args):
"""Check template not found issue error handling."""
with pytest.raises(RCNotFoundError):
utils.rc_load_json(Path(), render=True)
def _load_pkg_info(self):
"""Load package info descriptor from a file."""
fpath = self.pkgrepo_dpath.joinpath(str(self.pkg_id),
self._pkginfo_fpath)
return cr_utl.rc_load_json(fpath, emheading='Package info descriptor')
def get_dcos_conf(cluster_conf, tmp_dpath: Path):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if not cluster_conf or dstor_linux_pkg_index_path == 'NOP':
return {}
# Discover relative URL to the DC/OS aggregated configuration package.
dstor_dcoscfg_pkg_path = get_dstor_dcoscfgpkg_path(
dstor_root_url, dstor_linux_pkg_index_path, tmp_dpath)
# Download DC/OS aggregated configuration package ...
dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path)
try:
dcoscfg_pkg_fpath = cm_utl.download(dcoscfg_pkg_url, tmp_dpath)
LOG.debug(f'DC/OS aggregated config package:'
f' Download: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config package: {dcoscfg_pkg_url}:'
f' {type(e).__name__}: {e}') from e
# Process DC/OS aggregated configuration package.
try:
with tf.TemporaryDirectory(dir=str(tmp_dpath)) as tmp_dpath_:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath_)
LOG.debug(f'DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Extract: OK')
values_fpath = Path(tmp_dpath_).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath, emheading=f'DC/OS aggregated config: Values')
template_fpath = Path(tmp_dpath_).joinpath(template_fname)
template = load_dcos_conf_template(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:'
f' {type(e).__name__}: {e}')
else:
raise
else:
LOG.debug(f'DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Preprocess: OK')
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
def get_dstor_dcoscfgpkg_path(dstor_root_url: str, dstor_lpi_path: str,
tmp_dpath: str):
"""Retrieve the Linux Package Index (LPI) object from the DC/OS
distribution storage and discover a relative URL to the DC/OS
aggregated configuration package.
LPI is expected to be a JSON-formatted file containing descriptors for
DC/OS distribution packages:
{
"<pkg-name>":{
"filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz",
"id":"<pkg-name>--<pkg-version>"
},
...
}
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_lpi_path: str, URL path to the DC/OS Linux package
index object at the DC/OS distribution
storage
:return tmp_dpath: str, URL path to the DC/OS aggregated
config package at the DC/OS distribution
storage
"""
dcos_conf_pkg_name = 'dcos-config-win'
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path)
try:
lpi_fpath = cm_utl.download(lpi_url, tmp_dpath)
LOG.debug(f'DC/OS Linux package index: Download: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}: {e}'
) from e
try:
lpi = cr_utl.rc_load_json(lpi_fpath,
emheading='DC/OS Linux package index')
if not isinstance(lpi, dict):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi_url}: Invalid structure')
dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name)
if dcos_conf_pkg_desc is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor is missed:'
f' {dcos_conf_pkg_name}')
if not isinstance(dcos_conf_pkg_desc, dict):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS'
f' aggregated config package descriptor:'
f' {dcos_conf_pkg_desc}')
dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename')
if dstor_dcoscfgpkg_path is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path is'
f' missed: {dcos_conf_pkg_desc}')
if not isinstance(dstor_dcoscfgpkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path:'
f' Invalid type: {dstor_dcoscfgpkg_path}')
finally:
lpi_fpath.unlink()
return dstor_dcoscfgpkg_path
def test_rc_load_directory_should_fail(*args):
"""Check error validation when source is directory."""
with pytest.raises(RCError):
utils.rc_load_json(Path())
