def test_unpack_unavailable_tar_path_should_fail(*args):
"""Unpack wrong tarfile path."""
with pytest.raises(ValueError):
utils.unpack('', '')
def add_package(self, pkg_id, dstor_root_url, dstor_pkgrepo_path):
"""Add a package to the local package repository.
:param pkg_id: PackageId, package ID
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_pkgrepo_path: str, DC/OS distribution storage package
repository root path
"""
msg_src = self.__class__.__name__
# Download a package tarball
pkg_url = self._make_pkg_url(pkg_id=pkg_id,
dstor_root_url=dstor_root_url,
dstor_pkgrepo_path=dstor_pkgrepo_path)
try:
cm_utl.download(pkg_url, str(self.tmp_dpath))
LOG.debug(f'{msg_src}: Add package: Download: {pkg_id}: {pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Add package: {pkg_id}: {pkg_url}: {type(e).__name__}: {e}'
) from e
# Unpack a package tarball
pkgtarball_fpath = (
self.tmp_dpath.joinpath(pkg_id.pkg_id).with_suffix('.tar.xz')
)
try:
# Try to cleanup local package repository before trying to
# create a package installation directory there
pkg_inst_dpath = self.pkgrepo_dpath.joinpath(pkg_id.pkg_id)
try:
if pkg_inst_dpath.exists():
if pkg_inst_dpath.is_dir():
shutil.rmtree(str(pkg_inst_dpath))
elif pkg_inst_dpath.is_file and (
not pkg_inst_dpath.is_symlink()
):
pkg_inst_dpath.unlink()
else:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup'
f' package repository: Removing objects other than'
f' regular directories and files is not supported'
)
LOG.debug(f'{msg_src}: Add package: {pkg_id}: Auto-cleanup:'
f' {pkg_inst_dpath}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup: {pkg_inst_dpath}:'
f' {type(e).__name__}: {e}'
) from e
with tf.TemporaryDirectory(dir=str(self.tmp_dpath)) as temp_dpath:
cm_utl.unpack(str(pkgtarball_fpath), temp_dpath)
try:
# Lookup for a directory named after the package ID
src_dpath = [
path for path in Path(temp_dpath).iterdir() if (
path.name == pkg_id.pkg_id
)
][0]
if src_dpath.is_dir():
shutil.copytree(
str(src_dpath), str(pkg_inst_dpath)
)
else:
# Only a directory may be named after the package ID,
# otherwise a package structure is broken
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: Broken package structure'
)
except IndexError:
# Use the temporary directory as package's container
shutil.copytree(
temp_dpath, str(pkg_inst_dpath)
)
LOG.debug(f'{msg_src}: Add package: Extract: {pkg_id}')
except Exception as e:
if not isinstance(e, cr_exc.RCExtractError):
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}'
)
else:
raise
finally:
pkgtarball_fpath.unlink()
# Create a work, runtime and log data directories for a package.
for host_dpath in self.work_dpath, self.run_dpath, self.log_dpath:
path = host_dpath.joinpath(pkg_id.pkg_name)
if not path.exists():
try:
path.mkdir(parents=True)
LOG.debug(f'{msg_src}: Add package: {pkg_id}:'
f' Create data directory: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: {type(e).__name__}: {e}'
) from e
elif path.is_symlink():
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: Symlink conflict'
)
elif path.is_reserved():
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: Reserved name conflict'
)
elif not path.is_dir():
# Attempt to auto-clean garbage
try:
path.unlink()
LOG.debug(f'{msg_src}: Add package: {pkg_id}: Auto-cleanup:'
f' File: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup: File: {path}:'
f' {type(e).__name__}: {e}'
) from e
# Attempt to create data dir
try:
path.mkdir(parents=True)
LOG.debug(f'{msg_src}: Add package: {pkg_id}:'
f' Create data directory: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: {type(e).__name__}: {e}'
) from e
else:
# Leave existing directories intact
pass
# Workaround for dcos-diagnostics to be able to start
# TODO: Remove this code after correct dcos-diagnostics configuration
# is figured out and all its config files are arranged properly
# to support DC/OS installation storage FS layout.
if pkg_id.pkg_name == 'dcos-diagnostics':
# Move binary and config-files to DC/OS installation storage root
src_dpath = self.pkgrepo_dpath.joinpath(pkg_id.pkg_id, 'bin')
try:
LOG.debug(
f'{msg_src}: Add package: Workaround: Copy list: '
f' {list(src_dpath.glob("*.*"))}'
)
for src_fpath in src_dpath.glob('*.*'):
if not self.root_dpath.joinpath(src_fpath.name).exists():
shutil.copy(str(src_fpath), str(self.root_dpath))
LOG.debug(
f'{msg_src}: Add package: Workaround: Copy file: '
f' {str(src_fpath)} -> {str(self.root_dpath)}'
)
# Create a folder for logs
log_dpath = self.root_dpath.joinpath('mesos-logs')
if not log_dpath.exists():
log_dpath.mkdir()
except Exception as e:
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}'
)
def test_unpack_should_return_destination_location(mock_tar, *args):
"""Check is unpack file result provide valid location."""
mock_tar.__enter__.return_value = mock.Mock()
location = utils.unpack('', os.getcwd())
assert location == os.getcwd()
def get_dcos_conf(cluster_conf, tmp_dpath: Path):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if not cluster_conf or dstor_linux_pkg_index_path == 'NOP':
return {}
# Discover relative URL to the DC/OS aggregated configuration package.
dstor_dcoscfg_pkg_path = get_dstor_dcoscfgpkg_path(
dstor_root_url, dstor_linux_pkg_index_path, tmp_dpath)
dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path)
dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name
# Download DC/OS aggregated configuration package ...
try:
cm_utl.download(dcoscfg_pkg_url, str(tmp_dpath))
LOG.debug(f'DC/OS aggregated config package:'
f' Download: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config package: {dcoscfg_pkg_url}:'
f' {type(e).__name__}: {e}') from e
# Process DC/OS aggregated configuration package.
dcoscfg_pkg_fpath = tmp_dpath.joinpath(dcoscfg_pkg_fname)
try:
with tf.TemporaryDirectory(dir=str(tmp_dpath)) as tmp_dpath_:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath_)
LOG.debug(f'DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Extract: OK')
values_fpath = Path(tmp_dpath_).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath, emheading=f'DC/OS aggregated config: Values')
template_fpath = Path(tmp_dpath_).joinpath(template_fname)
template = load_dcos_conf_template(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:'
f' {type(e).__name__}: {e}')
else:
raise
else:
LOG.debug(f'DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Preprocess: OK')
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function get_dcos_conf().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (self.cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Discover relative URL to the DC/OS aggregated configuration package.
dstor_dcoscfg_pkg_path = self.get_dstor_dcoscfgpkg_path(
dstor_root_url, dstor_linux_pkg_index_path)
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name
# Download DC/OS aggregated configuration package ...
try:
cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' Download: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config package: {dcoscfg_pkg_url}:'
f' {type(e).__name__}: {e}') from e
# Process DC/OS aggregated configuration package.
dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath(
dcoscfg_pkg_fname)
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Extract: OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath, emheading=f'DC/OS aggregated config: Values')
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_template(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:'
f' {type(e).__name__}: {e}')
else:
raise
else:
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Preprocess: OK')
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (self.cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
dcos_conf_pkg_name = 'dcos-config-win'
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_linux_pkg_index_path)
lpi_fname = Path(dstor_linux_pkg_index_path).name
try:
cm_utl.download(lpi_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_fname}: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: Download: {lpi_fname}:'
f' {lpi_url}: {type(e).__name__}: {e}') from e
lpi_fpath = self.inst_storage.tmp_dpath.joinpath(lpi_fname)
try:
lpi = cr_utl.rc_load_json(
lpi_fpath, emheading=f'DC/OS Linux package index: {lpi_fname}')
if (not isinstance(lpi, dict)
or not isinstance(lpi.get(dcos_conf_pkg_name), dict)):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi}')
dstor_dcoscfg_pkg_path = lpi.get(dcos_conf_pkg_name).get(
'filename')
if not isinstance(dstor_dcoscfg_pkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: DC/OS config package'
f' distribution storage path: {dstor_dcoscfg_pkg_path}')
except cr_exc.RCError as e:
raise e
finally:
lpi_fpath.unlink()
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name
# Download DC/OS aggregated configuration package ...
try:
cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_pkg_fname}: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_pkg_fname}:'
f' {dcoscfg_pkg_url}: {type(e).__name__}: {e}') from e
dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath(
dcoscfg_pkg_fname)
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Extract:'
f' OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath,
emheading=f'DC/OS aggregated config: Values: {values_fname}'
)
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_templete(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config: {type(e).__name__}: {e}')
else:
raise
else:
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()