def test_clusters_items_should_provide_correct_location(self):
context = ResourceContext(cluster_conf={
'master-node-1': {'privateipaddr': '192.168.1.1', 'zookeeperlistenerport': '2181'},
'master-node-2': {'privateipaddr': '192.168.1.2', 'zookeeperlistenerport': '2182'},
}, extra_values={'privateipaddr': '192.168.1.1'})
items = context.get_items()
assert items[RCCONTEXT_ITEM.MASTER_LOCATION] == '192.168.1.1:2181,192.168.1.2:2181'
def test_cluster_conf_items_should_correspond_stub(self):
context = ResourceContext(istor_nodes=self.get_nodes())
items = context.get_items()
assert items[RCCONTEXT_ITEM.DCOS_INST_DPATH] == 'val_root'
assert items[RCCONTEXT_ITEM.DCOS_TMP_DPATH] == 'val_tmp'
assert items[RCCONTEXT_ITEM.DCOS_BIN_DPATH] == 'val_bin'
assert items[RCCONTEXT_ITEM.DCOS_LIB_DPATH] == 'val_lib'
def test_cluster_items_should_be_not_empty(self):
context = ResourceContext(cluster_conf={}, extra_values={'privateipaddr': '192.168.1.1'})
itms = context.get_items()
assert itms == {
RCCONTEXT_ITEM.MASTER_LOCATION: '127.0.0.1:2181',
RCCONTEXT_ITEM.MASTER_PRIV_IPADDR: '127.0.0.1',
RCCONTEXT_ITEM.LOCAL_PRIV_IPADDR: '192.168.1.1',
RCCONTEXT_ITEM.ZK_CLIENT_PORT: 2181,
'privateipaddr': '192.168.1.1'
}
def __init__(self,
pkg_id,
istor_nodes,
cluster_conf,
pkg_info=None,
pkg_extcfg=None,
pkg_svccfg=None,
extra_context=None):
"""Constructor.
:param pkg_id: PackageId, package ID
:param istor_nodes: IStorNodes, DC/OS installation storage nodes (set
of pathlib.Path objects)
:param cluster_conf: dict, configparser.ConfigParser.read_dict()
compatible data. DC/OS cluster setup parameters
:param pkg_info: dict, package info descriptor from DC/OS package
build system
:param pkg_extcfg: dict, extra package installation options
:param pkg_svccfg: dict, configparser.ConfigParser.read_dict()
compatible. Package system service options
:param extra_context: dict, extra 'key=value' data to be added to the
resource rendering context
"""
assert isinstance(
pkg_id,
PackageId), (f'Argument: pkg_id:'
f' Got {type(pkg_id).__name__} instead of PackageId')
assert isinstance(istor_nodes, IStorNodes), (
f'Argument: istor_nodes:'
f' Got {type(istor_nodes).__name__} instead of IStorNodes')
assert isinstance(
cluster_conf,
dict), (f'Argument: cluster_conf:'
f'Got {type(cluster_conf).__name__} instead of dict')
self._pkg_id = pkg_id
self._istor_nodes = istor_nodes
self._context = ResourceContext(istor_nodes=istor_nodes,
cluster_conf=cluster_conf,
pkg_id=pkg_id,
extra_values=extra_context)
# Load package info descriptor
self._pkg_info = pkg_info if pkg_info is not None else (
self._load_pkg_info())
# Load package extra installation options descriptor
self._pkg_extcfg = pkg_extcfg if pkg_extcfg is not None else (
self._load_pkg_extcfg())
# Load package system service options descriptor
self._pkg_svccfg = pkg_svccfg if pkg_svccfg is not None else (
self._load_pkg_svccfg())
def rc_load_ini(fpath: Path, emheading: str = None, render: bool = False,
context: ResourceContext = None) -> Any:
"""Load INI-formatted data from a resource file. Content of a resource
file can be pre-processed by Jinja2 rendering engine before being passed to
INI-parser.
:param fpath: pathlib.Path, path to a source file
:param emheading: str, heading to be added to the exception's description
:param render: bool, perform template rendering
:param context: ResourceContext, rendering context data object
:return: dict, configparser.ConfigParser.read_dict() compatible
data.
"""
cfg_parser = cfp.ConfigParser()
if render is True:
jj2_env = jj2.Environment(
loader=jj2.FileSystemLoader(str(fpath.parent))
)
jj2_tmpl = jj2_env.get_template(str(fpath.name))
context_items = {} if context is None else context.get_items()
ini_str = jj2_tmpl.render(**context_items)
LOG.debug(f'rc_load_ini(): ini_str: {ini_str}')
cfg_parser.read_string(ini_str, source=str(fpath))
else:
with fpath.open() as fp:
cfg_parser.read_file(fp)
return {k: dict(v) for k, v in cfg_parser.items()}
def rc_load_json(fpath: Path, emheading: str = None, render: bool = False,
context: ResourceContext = None) -> Any:
"""Load JSON-formatted data from a resource file. Content of a resource
file can be pre-processed by Jinja2 rendering engine before being passed to
JSON-parser.
:param fpath: Path, path to a source file.
:param emheading: str, heading to be added to the exception's description
:param render: bool, perform template rendering
:param context: ResourceContext, rendering context data object
:return: json obj, JSON-formatted data
"""
if render is True:
jj2_env = jj2.Environment(
loader=jj2.FileSystemLoader(str(fpath.parent))
)
jj2_tmpl = jj2_env.get_template(str(fpath.name))
context_items = {} if context is None else context.get_items(json_ready=True)
json_str = jj2_tmpl.render(**context_items)
LOG.debug(f'rc_load_json(): json_str: {json_str}')
j_body = json.loads(json_str, strict=False)
else:
with fpath.open() as fp:
j_body = json.load(fp, strict=False)
return j_body
def rc_load_yaml(fpath: Path, emheading: str = "", render: bool = False,
context: ResourceContext = None) -> Any:
"""Load YAML-formatted data from a resource file. Content of a resource
file can be pre-processed by Jinja2 rendering engine before being passed to
YAML-parser.
:param fpath: pathlib.Path, path to a source file.
:param emheading: str, heading to be added to the exception's description
:param render: bool, perform template rendering
:param context: ResourceContext, rendering context data object
:return: yaml obj, YAML-formatted data
"""
if render is True:
jj2_env = jj2.Environment(
loader=jj2.FileSystemLoader(str(fpath.parent))
)
jj2_tmpl = jj2_env.get_template(str(fpath.name))
context_items = {} if context is None else context.get_items()
yaml_str = jj2_tmpl.render(**context_items)
LOG.debug(f'rc_load_yaml(): yaml_str: {yaml_str}')
y_body = yaml.safe_load(yaml_str)
else:
with fpath.open() as fp:
y_body = yaml.safe_load(fp)
return y_body
def __init__(self,
pkg_id: PackageId,
istor_nodes: IStorNodes,
cluster_conf: dict,
pkg_info: dict = None,
pkg_extcfg: dict = None,
pkg_svccfg: dict = None,
extra_context: dict = None):
"""Constructor.
:param pkg_id: PackageId, package ID
:param istor_nodes: IStorNodes, DC/OS installation storage nodes (set
of pathlib.Path objects)
:param cluster_conf: dict, configparser.ConfigParser.read_dict()
compatible data. DC/OS cluster setup parameters
:param pkg_info: dict, package info descriptor from DC/OS package
build system
:param pkg_extcfg: dict, extra package installation options
:param pkg_svccfg: dict, configparser.ConfigParser.read_dict()
compatible. Package system service options
:param extra_context: dict, extra 'key=value' data to be added to the
resource rendering context
"""
self.msg_src = self.__class__.__name__
self._pkg_id = pkg_id
self._istor_nodes = istor_nodes
self._context = ResourceContext(istor_nodes=istor_nodes,
cluster_conf=cluster_conf,
pkg_id=pkg_id,
extra_values=extra_context)
# Load package info descriptor
self._pkg_info = pkg_info if pkg_info is not None else (
self._load_pkg_info())
# Load package extra installation options descriptor
self._pkg_extcfg = pkg_extcfg if pkg_extcfg is not None else (
self._load_pkg_extcfg())
# Load package system service options descriptor
self._pkg_svccfg = pkg_svccfg if pkg_svccfg is not None else (
self._load_pkg_svccfg())
def _process_pkgconf_srcfile(self,
src_fpath: Path,
tmp_dpath: Path,
context: ResourceContext = None):
"""Process DC/OS package configuration source file.
:param src_fpath: Path, path to a source configuration file
:param tmp_dpath: Path, path to a temporary directory to save
intermediate rendered content
:param context: ResourceContext, rendering context data object
"""
if '.j2' in src_fpath.suffixes[-1:]:
dst_fname = src_fpath.stem
json_ready = '.json' in src_fpath.suffixes[-2:-1]
else:
dst_fname = src_fpath.name
json_ready = '.json' in src_fpath.suffixes[-1:]
try:
j2_env = j2.Environment(
loader=j2.FileSystemLoader(str(src_fpath.parent)))
j2_tmpl = j2_env.get_template(str(src_fpath.name))
context_items = {} if context is None else context.get_items(
json_ready=json_ready)
rendered_str = j2_tmpl.render(**context_items)
LOG.debug(f'{self.msg_src}: Process configuration file:'
f' {src_fpath}: Rendered content: {rendered_str}')
dst_fpath = tmp_dpath.joinpath(dst_fname)
dst_fpath.write_text(rendered_str, encoding='utf-8')
LOG.debug(f'{self.msg_src}: Process configuration file:'
f' {src_fpath}: Save: {dst_fpath}')
except (FileNotFoundError, j2.TemplateNotFound) as e:
err_msg = f'Load: {src_fpath}'
raise cfgm_exc.PkgConfFileNotFoundError(err_msg) from e
except (OSError, RuntimeError) as e:
err_msg = f'Load: {src_fpath}: {type(e).__name__}: {e}'
raise cfgm_exc.PkgConfError(err_msg) from e
except j2.TemplateError as e:
err_msg = f'Load: {src_fpath}: {type(e).__name__}: {e}'
raise cfgm_exc.PkgConfFileInvalidError(err_msg) from e
def _deploy_dcos_conf(self):
"""Deploy aggregated DC/OS configuration object."""
LOG.debug(f'{self.msg_src}: Execute: Deploy aggregated config: ...')
context = ResourceContext(
istor_nodes=self.config.inst_storage.istor_nodes,
cluster_conf=self.config.cluster_conf,
extra_values=self.config.dcos_conf.get('values'))
context_items = context.get_items()
context_items_jr = context.get_items(json_ready=True)
t_elements = self.config.dcos_conf.get('template').get('package', [])
for t_element in t_elements:
path = t_element.get('path')
content = t_element.get('content')
try:
j2t = j2.Environment().from_string(path)
rendered_path = j2t.render(**context_items)
dst_fpath = Path(rendered_path)
j2t = j2.Environment().from_string(content)
if '.json' in dst_fpath.suffixes[-1:]:
rendered_content = j2t.render(**context_items_jr)
else:
rendered_content = j2t.render(**context_items)
except j2.TemplateError as e:
err_msg = (f'Execute: Deploy aggregated config: Render:'
f' {path}: {type(e).__name__}: {e}')
raise cfgm_exc.PkgConfFileInvalidError(err_msg) from e
if not dst_fpath.parent.exists():
try:
dst_fpath.parent.mkdir(parents=True, exist_ok=True)
LOG.debug(f'{self.msg_src}: Execute: Deploy aggregated'
f' config: Create directory:'
f' {dst_fpath.parent}: OK')
except (OSError, RuntimeError) as e:
err_msg = (f'Execute: Deploy aggregated config: Create'
f' directory: {dst_fpath.parent}:'
f' {type(e).__name__}: {e}')
raise cr_exc.SetupCommandError(err_msg) from e
elif not dst_fpath.parent.is_dir():
err_msg = (f'Execute: Deploy aggregated config: Save content:'
f' {dst_fpath}: Existing parent is not a directory:'
f' {dst_fpath.parent}')
raise cr_exc.SetupCommandError(err_msg)
elif dst_fpath.exists():
err_msg = (f'Execute: Deploy aggregated config: Save content:'
f' {dst_fpath}: Same-named file already exists!')
raise cr_exc.SetupCommandError(err_msg)
try:
dst_fpath.write_text(rendered_content)
LOG.debug(f'{self.msg_src}: Execute: Deploy aggregated config:'
f' Save content: {dst_fpath}: OK')
except (OSError, RuntimeError) as e:
err_msg = (f'Execute: Deploy aggregated config: Save content:'
f' {dst_fpath}: {type(e).__name__}: {e}')
raise cr_exc.SetupCommandError(err_msg) from e
LOG.debug(f'{self.msg_src}: Execute: Deploy aggregated config: OK')
def test_extra_items_should_provide_all_keys(self):
context = ResourceContext(extra_values={'key': 'val'})
items = context.get_items()
assert items['key'] == 'val'
def test_pkg_items_should_provide_all_keys(self):
pkg_id = mock.Mock()
context = ResourceContext(istor_nodes=self.get_nodes(), pkg_id=pkg_id)
items = context.get_items()
assert list(items.keys()) == RCCONTEXT_ITEMS
def test_default_get_items_should_be_empty(self):
context = ResourceContext()
assert context.get_items() == {}
class PackageManifest:
"""Package manifest container."""
def __init__(self,
pkg_id,
istor_nodes,
cluster_conf,
pkg_info=None,
pkg_extcfg=None,
pkg_svccfg=None,
extra_context=None):
"""Constructor.
:param pkg_id: PackageId, package ID
:param istor_nodes: IStorNodes, DC/OS installation storage nodes (set
of pathlib.Path objects)
:param cluster_conf: dict, configparser.ConfigParser.read_dict()
compatible data. DC/OS cluster setup parameters
:param pkg_info: dict, package info descriptor from DC/OS package
build system
:param pkg_extcfg: dict, extra package installation options
:param pkg_svccfg: dict, configparser.ConfigParser.read_dict()
compatible. Package system service options
:param extra_context: dict, extra 'key=value' data to be added to the
resource rendering context
"""
assert isinstance(
pkg_id,
PackageId), (f'Argument: pkg_id:'
f' Got {type(pkg_id).__name__} instead of PackageId')
assert isinstance(istor_nodes, IStorNodes), (
f'Argument: istor_nodes:'
f' Got {type(istor_nodes).__name__} instead of IStorNodes')
assert isinstance(
cluster_conf,
dict), (f'Argument: cluster_conf:'
f'Got {type(cluster_conf).__name__} instead of dict')
self._pkg_id = pkg_id
self._istor_nodes = istor_nodes
self._context = ResourceContext(istor_nodes=istor_nodes,
cluster_conf=cluster_conf,
pkg_id=pkg_id,
extra_values=extra_context)
# Load package info descriptor
self._pkg_info = pkg_info if pkg_info is not None else (
self._load_pkg_info())
# Load package extra installation options descriptor
self._pkg_extcfg = pkg_extcfg if pkg_extcfg is not None else (
self._load_pkg_extcfg())
# Load package system service options descriptor
self._pkg_svccfg = pkg_svccfg if pkg_svccfg is not None else (
self._load_pkg_svccfg())
# TODO: Add content verification (jsonschema) for self.body. Raise
# ValueError, if conformance was not confirmed.
def __str__(self):
return str(self.body)
@property
def body(self):
""""""
return {
'pkg_id': self._pkg_id.pkg_id,
'context': self._context.as_dict(),
'pkg_info': self._pkg_info,
'pkg_extcfg': self._pkg_extcfg,
'pkg_svccfg': self._pkg_svccfg,
}
@property
def pkg_id(self):
""""""
return self._pkg_id
@property
def istor_nodes(self):
""""""
return self._istor_nodes
@property
def context(self):
""""""
return ResourceContext(self._istor_nodes, self._context._cluster_conf,
self._pkg_id)
@property
def pkg_info(self):
""""""
return self._pkg_info
@property
def pkg_extcfg(self):
""""""
return self._pkg_extcfg
@property
def pkg_svccfg(self):
""""""
return self._pkg_svccfg
def _load_pkg_info(self):
"""Load package info descriptor from a file.
:return: dict, package info descriptor
"""
fpath = getattr(self._istor_nodes,
ISTOR_NODE.PKGREPO).joinpath(self._pkg_id.pkg_id,
cr_const.PKG_INFO_FPATH)
try:
pkg_info = cr_utl.rc_load_json(fpath,
emheading='Package info descriptor',
render=True,
context=self._context)
except cr_exc.RCNotFoundError:
pkg_info = {}
return pkg_info
def _load_pkg_extcfg(self):
"""Load package extra installation options from a file.
:return: dict, package extra installation options descriptor
"""
fpath = getattr(self._istor_nodes, ISTOR_NODE.PKGREPO).joinpath(
self._pkg_id.pkg_id,
cr_const.PKG_EXTCFG_FPATH.format(pkg_name=self._pkg_id.pkg_name))
try:
pkg_extcfg = cr_utl.rc_load_yaml(
fpath,
emheading='Package inst extra descriptor',
render=True,
context=self._context)
except cr_exc.RCNotFoundError:
pkg_extcfg = {}
return pkg_extcfg
def _load_pkg_svccfg(self):
"""Load package system service options from a file.
:return: dict, package system service descriptor
"""
fpath = getattr(self._istor_nodes, ISTOR_NODE.PKGREPO).joinpath(
self._pkg_id.pkg_id,
cr_const.PKG_SVCCFG_FPATH.format(pkg_name=self._pkg_id.pkg_name))
try:
pkg_svccfg = cr_utl.rc_load_ini(
fpath,
emheading='Package service descriptor',
render=True,
context=self._context)
except cr_exc.RCNotFoundError:
pkg_svccfg = {}
return pkg_svccfg
def json(self):
"""Construct JSON representation of the manifest."""
return json.dumps(self.body, indent=4, sort_keys=True)
@classmethod
def load(cls, fpath):
"""Load package manifest from a file.
:param fpath: pathlib.Path, path to a JSON-formatted manifest file.
:return: dict, package manifest.
"""
m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest')
# TODO: Add content verification (jsonschema) for m_body. Raise
# ValueError, if conformance was not confirmed.
try:
manifest = cls(
pkg_id=PackageId(pkg_id=m_body.get('pkg_id')),
istor_nodes=IStorNodes(
**{
k: Path(v)
for k, v in m_body.get('context').get(
'istor_nodes').items()
}),
cluster_conf=m_body.get('context').get('cluster_conf'),
pkg_info=m_body.get('pkg_info'),
pkg_extcfg=m_body.get('pkg_extcfg'),
pkg_svccfg=m_body.get('pkg_svccfg'),
)
LOG.debug(f'Package manifest: Load: {fpath}')
except (ValueError, AssertionError, TypeError) as e:
err_msg = (f'Package manifest: Load:'
f' {fpath}: {type(e).__name__}: {e}')
raise cr_exc.RCInvalidError(err_msg) from e
return manifest
def save(self):
"""Save package manifest to a file within the active packages index."""
fpath = getattr(
self._istor_nodes,
ISTOR_NODE.PKGACTIVE).joinpath(f'{self._pkg_id.pkg_id}.json')
try:
with fpath.open(mode='w') as fp:
json.dump(self.body, fp)
except (OSError, RuntimeError) as e:
err_msg = f'Package manifest: Save: {type(e).__name__}: {e}'
raise cr_exc.RCError(err_msg) from e
LOG.debug(f'Package manifest: Save: {fpath}')
def update_context(self, values=None):
"""Update context data.
:param values: dict, 'key=value' data to be added to / updated in the
resource rendering context.
"""
self._context.update(values=values)
def context(self):
""""""
return ResourceContext(self._istor_nodes, self._context._cluster_conf,
self._pkg_id)
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
objects:
{
'package': {[
{'path': <str>, 'content': <str>},
...
]}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_dcoscfg_path = (self.cluster_conf.get('distribution-storage',
{}).get('dcoscfgpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_dcoscfg_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
dcoscfg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_path)
dcoscfg_fname = Path(dstor_dcoscfg_path).name
try:
cm_utl.download(dcoscfg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_fname}: {dcoscfg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_fname}:'
f' {dcoscfg_url}: {type(e).__name__}: {e}') from e
dcoscfg_fpath = self.inst_storage.tmp_dpath.joinpath(dcoscfg_fname)
try:
dcos_conf = cr_utl.rc_load_yaml(
dcoscfg_fpath,
emheading=f'DC/OS aggregated config: {dcoscfg_fname}',
render=True,
context=ResourceContext(
istor_nodes=self.inst_storage.istor_nodes,
cluster_conf=self.cluster_conf))
if (not isinstance(dcos_conf, dict)
or not isinstance(dcos_conf.get('package'), list)):
raise cr_exc.RCInvalidError(
f'DC/OS aggregated config: {dcos_conf}')
for element in dcos_conf.get('package'):
if (not isinstance(element, dict)
or not isinstance(element.get('path'), str)
or not isinstance(element.get('content'), str)):
raise cr_exc.RCElementError(
f'DC/OS aggregated config: {element}')
return dcos_conf
except cr_exc.RCError as e:
raise e
finally:
dcoscfg_fpath.unlink()