def load(cls, fpath):
"""Load package manifest from a file.
:param fpath: pathlib.Path, path to a JSON-formatted manifest file.
:return: dict, package manifest.
"""
m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest')
# TODO: Add content verification (jsonschema) for m_body. Raise
# ValueError, if conformance was not confirmed.
try:
manifest = cls(
pkg_id=PackageId(pkg_id=m_body.get('pkg_id')),
istor_nodes=IStorNodes(
**{
k: Path(v)
for k, v in m_body.get('context').get(
'istor_nodes').items()
}),
cluster_conf=m_body.get('context').get('cluster_conf'),
pkg_info=m_body.get('pkg_info'),
pkg_extcfg=m_body.get('pkg_extcfg'),
pkg_svccfg=m_body.get('pkg_svccfg'),
)
LOG.debug(f'Package manifest: Load: {fpath}')
except (ValueError, AssertionError, TypeError) as e:
err_msg = (f'Package manifest: Load:'
f' {fpath}: {type(e).__name__}: {e}')
raise cr_exc.RCInvalidError(err_msg) from e
return manifest
def rc_load_ini(fpath, emheading=None):
"""Load INI-formatted data from a (resource) file.
:param fpath: pathlib.Path, path to a source file
:param emheading: str, heading to be added to the exception's description
:return: dict, configparser.ConfigParser.read_dict() compatible
data.
"""
assert isinstance(fpath, Path) and fpath.is_absolute(), (
f'Argument: fpath: Absolute pathlib.Path is required: {fpath}')
cfg_parser = cfp.ConfigParser()
try:
with fpath.open() as fp:
cfg_parser.read_file(fp)
except FileNotFoundError:
err_msg = f'Load: {fpath}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCNotFoundError(err_msg)
except (OSError, RuntimeError) as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCError(err_msg)
except cfp.Error as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCInvalidError(err_msg)
else:
return {k: dict(v) for k, v in cfg_parser.items()}
def rc_load_json(fpath, emheading=None):
"""Load JSON-formatted data from a (resource) file.
:param fpath: pathlib.Path, path to a source file.
:param emheading: str, heading to be added to the exception's description
:return: json obj, JSON-formatted data
"""
assert isinstance(fpath, Path) and fpath.is_absolute(), (
f'Argument: fpath: Absolute pathlib.Path is required: {fpath}')
try:
with fpath.open() as fp:
j_body = json.load(fp, strict=False)
except FileNotFoundError:
err_msg = f'Load: {fpath}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCNotFoundError(err_msg)
except (OSError, RuntimeError) as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCError(err_msg)
except cr_exc.JSON_ERRORS as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCInvalidError(err_msg)
else:
return j_body
def rc_load_ini(fpath, emheading=None, render=False, context=None):
"""Load INI-formatted data from a resource file. Content of a resource
file can be pre-processed by Jinja2 rendering engine before being passed to
INI-parser.
:param fpath: pathlib.Path, path to a source file
:param emheading: str, heading to be added to the exception's description
:param render: bool, perform template rendering
:param context: ResourceContext, rendering context data object
:return: dict, configparser.ConfigParser.read_dict() compatible
data.
"""
assert isinstance(fpath, Path) and fpath.is_absolute(), (
f'Argument: fpath: Absolute pathlib.Path is required: {fpath}'
)
if context is None:
context_items = {}
else:
assert isinstance(context, ResourceContext), (
f'Argument: context:'
f' Got {type(context).__name__} instead of ResourceContext'
)
context_items = context.get_items()
cfg_parser = cfp.ConfigParser()
try:
if render is True:
jj2_env = jj2.Environment(
loader=jj2.FileSystemLoader(str(fpath.parent))
)
jj2_tmpl = jj2_env.get_template(str(fpath.name))
ini_str = jj2_tmpl.render(**context_items)
LOG.debug(f'rc_load_ini(): ini_str: {ini_str}')
cfg_parser.read_string(ini_str, source=str(fpath))
else:
with fpath.open() as fp:
cfg_parser.read_file(fp)
except (FileNotFoundError, jj2.TemplateNotFound) as e:
err_msg = f'Load: {fpath}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCNotFoundError(err_msg) from e
except (OSError, RuntimeError) as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCError(err_msg) from e
except (jj2.TemplateError, cfp.Error) as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCInvalidError(err_msg) from e
else:
return {k: dict(v) for k, v in cfg_parser.items()}
def rc_load_json(fpath, emheading=None, render=False, context=None):
"""Load JSON-formatted data from a resource file. Content of a resource
file can be pre-processed by Jinja2 rendering engine before being passed to
JSON-parser.
:param fpath: pathlib.Path, path to a source file.
:param emheading: str, heading to be added to the exception's description
:param render: bool, perform template rendering
:param context: ResourceContext, rendering context data object
:return: json obj, JSON-formatted data
"""
assert isinstance(fpath, Path) and fpath.is_absolute(), (
f'Argument: fpath: Absolute pathlib.Path is required: {fpath}'
)
if context is None:
context_items = {}
else:
assert isinstance(context, ResourceContext), (
f'Argument: context:'
f' Got {type(context).__name__} instead of ResourceContext'
)
context_items = context.get_items(json_ready=True)
try:
if render is True:
jj2_env = jj2.Environment(
loader=jj2.FileSystemLoader(str(fpath.parent))
)
jj2_tmpl = jj2_env.get_template(str(fpath.name))
json_str = jj2_tmpl.render(**context_items)
LOG.debug(f'rc_load_json(): json_str: {json_str}')
j_body = json.loads(json_str, strict=False)
else:
with fpath.open() as fp:
j_body = json.load(fp, strict=False)
except (FileNotFoundError, jj2.TemplateNotFound) as e:
err_msg = f'Load: {fpath}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCNotFoundError(err_msg) from e
except (OSError, RuntimeError) as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCError(err_msg) from e
except (jj2.TemplateError,) + cr_exc.JSON_ERRORS as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCInvalidError(err_msg) from e
else:
return j_body
def wrapper(fpath: Path, emheading: str = None, render: bool = False,
context: ResourceContext = None) -> Any:
try:
content = func(fpath, emheading, render, context)
except (FileNotFoundError, jj2.TemplateNotFound) as e:
err_msg = f'Load: {fpath}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCNotFoundError(err_msg) from e
except (OSError, RuntimeError) as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCError(err_msg) from e
except (jj2.TemplateError, yaml.YAMLError) + cr_exc.JSON_ERRORS as e:
err_msg = f'Load: {fpath}: {type(e).__name__}: {e}'
err_msg = f'{emheading}: {err_msg}' if emheading else err_msg
raise cr_exc.RCInvalidError(err_msg) from e
else:
return content
def load(cls, fpath):
"""Load package manifest from a file.
:param fpath: pathlib.Path, path to a JSON-formatted manifest file.
:return: dict, package manifest.
"""
m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest')
try:
manifest = cls(
pkg_id=PackageId(pkg_id=m_body.get('pkg_id')),
pkgrepo_dpath=Path(m_body.get('pkgrepo_dpath')),
pkgactive_dpath=Path(m_body.get('pkgactive_dpath')),
pkg_info=m_body.get('pkg_info'),
pkg_ini=m_body.get('pkg_ini'),
svc_conf=m_body.get('svc_conf'),
)
LOG.debug(f'Package manifest: Load: {fpath}')
except (ValueError, AssertionError) as e:
err_msg = (f'Package manifest: Load:'
f' {fpath}: {type(e).__name__}: {e}')
raise cr_exc.RCInvalidError(err_msg)
return manifest
def get_dstor_dcoscfgpkg_path(dstor_root_url: str, dstor_lpi_path: str,
tmp_dpath: str):
"""Retrieve the Linux Package Index (LPI) object from the DC/OS
distribution storage and discover a relative URL to the DC/OS
aggregated configuration package.
LPI is expected to be a JSON-formatted file containing descriptors for
DC/OS distribution packages:
{
"<pkg-name>":{
"filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz",
"id":"<pkg-name>--<pkg-version>"
},
...
}
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_lpi_path: str, URL path to the DC/OS Linux package
index object at the DC/OS distribution
storage
:return tmp_dpath: str, URL path to the DC/OS aggregated
config package at the DC/OS distribution
storage
"""
dcos_conf_pkg_name = 'dcos-config-win'
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path)
try:
lpi_fpath = cm_utl.download(lpi_url, tmp_dpath)
LOG.debug(f'DC/OS Linux package index: Download: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}: {e}'
) from e
try:
lpi = cr_utl.rc_load_json(lpi_fpath,
emheading='DC/OS Linux package index')
if not isinstance(lpi, dict):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi_url}: Invalid structure')
dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name)
if dcos_conf_pkg_desc is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor is missed:'
f' {dcos_conf_pkg_name}')
if not isinstance(dcos_conf_pkg_desc, dict):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS'
f' aggregated config package descriptor:'
f' {dcos_conf_pkg_desc}')
dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename')
if dstor_dcoscfgpkg_path is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path is'
f' missed: {dcos_conf_pkg_desc}')
if not isinstance(dstor_dcoscfgpkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path:'
f' Invalid type: {dstor_dcoscfgpkg_path}')
finally:
lpi_fpath.unlink()
return dstor_dcoscfgpkg_path
def get_dstor_dcoscfgpkg_path(self, dstor_root_url: str,
dstor_lpi_path: str):
"""Retrieve the Linux Package Index (LPI) object from the DC/OS
distribution storage and discover a relative URL to the DC/OS
aggregated configuration package.
LPI is expected to be a JSON-formatted file containing descriptors for
DC/OS distribution packages:
{
"<pkg-name>":{
"filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz",
"id":"<pkg-name>--<pkg-version>"
},
...
}
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_lpi_path: str, URL path to the DC/OS Linux package
index object at the DC/OS distribution
storage
:return dstor_dcoscfgpkg_path: str, URL path to the DC/OS aggregated
config package at the DC/OS distribution
storage
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function
# get_dstor_dcoscfgpkg_path().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dcos_conf_pkg_name = 'dcos-config-win'
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path)
try:
lpi_fpath = cm_utl.download(lpi_url, self.inst_storage.tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}:'
f' {e}') from e
try:
lpi = cr_utl.rc_load_json(lpi_fpath,
emheading='DC/OS Linux package index')
if not isinstance(lpi, dict):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi_url}: Invalid structure')
dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name)
if dcos_conf_pkg_desc is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor is missed:'
f' {dcos_conf_pkg_name}')
if not isinstance(dcos_conf_pkg_desc, dict):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS'
f' aggregated config package descriptor:'
f' {dcos_conf_pkg_desc}')
dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename')
if dstor_dcoscfgpkg_path is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path is'
f' missed: {dcos_conf_pkg_desc}')
if not isinstance(dstor_dcoscfgpkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path:'
f' Invalid type: {dstor_dcoscfgpkg_path}')
finally:
lpi_fpath.unlink()
return dstor_dcoscfgpkg_path
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
objects:
{
'package': {[
{'path': <str>, 'content': <str>},
...
]}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_dcoscfg_path = (self.cluster_conf.get('distribution-storage',
{}).get('dcoscfgpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_dcoscfg_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
dcoscfg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_path)
dcoscfg_fname = Path(dstor_dcoscfg_path).name
try:
cm_utl.download(dcoscfg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_fname}: {dcoscfg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_fname}:'
f' {dcoscfg_url}: {type(e).__name__}: {e}') from e
dcoscfg_fpath = self.inst_storage.tmp_dpath.joinpath(dcoscfg_fname)
try:
dcos_conf = cr_utl.rc_load_yaml(
dcoscfg_fpath,
emheading=f'DC/OS aggregated config: {dcoscfg_fname}',
render=True,
context=ResourceContext(
istor_nodes=self.inst_storage.istor_nodes,
cluster_conf=self.cluster_conf))
if (not isinstance(dcos_conf, dict)
or not isinstance(dcos_conf.get('package'), list)):
raise cr_exc.RCInvalidError(
f'DC/OS aggregated config: {dcos_conf}')
for element in dcos_conf.get('package'):
if (not isinstance(element, dict)
or not isinstance(element.get('path'), str)
or not isinstance(element.get('content'), str)):
raise cr_exc.RCElementError(
f'DC/OS aggregated config: {element}')
return dcos_conf
except cr_exc.RCError as e:
raise e
finally:
dcoscfg_fpath.unlink()
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (self.cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
dcos_conf_pkg_name = 'dcos-config-win'
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_linux_pkg_index_path)
lpi_fname = Path(dstor_linux_pkg_index_path).name
try:
cm_utl.download(lpi_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_fname}: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: Download: {lpi_fname}:'
f' {lpi_url}: {type(e).__name__}: {e}') from e
lpi_fpath = self.inst_storage.tmp_dpath.joinpath(lpi_fname)
try:
lpi = cr_utl.rc_load_json(
lpi_fpath, emheading=f'DC/OS Linux package index: {lpi_fname}')
if (not isinstance(lpi, dict)
or not isinstance(lpi.get(dcos_conf_pkg_name), dict)):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi}')
dstor_dcoscfg_pkg_path = lpi.get(dcos_conf_pkg_name).get(
'filename')
if not isinstance(dstor_dcoscfg_pkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: DC/OS config package'
f' distribution storage path: {dstor_dcoscfg_pkg_path}')
except cr_exc.RCError as e:
raise e
finally:
lpi_fpath.unlink()
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name
# Download DC/OS aggregated configuration package ...
try:
cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_pkg_fname}: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_pkg_fname}:'
f' {dcoscfg_pkg_url}: {type(e).__name__}: {e}') from e
dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath(
dcoscfg_pkg_fname)
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Extract:'
f' OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath,
emheading=f'DC/OS aggregated config: Values: {values_fname}'
)
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_templete(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config: {type(e).__name__}: {e}')
else:
raise
else:
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
