def get_ref_pkg_list(self):
"""Get the current reference package list.
:return: list, JSON-formatted data
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_pkglist_path = (self.cluster_conf.get('distribution-storage',
{}).get('pkglistpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_pkglist_path == 'NOP':
LOG.info(f'{self.msg_src}: ref_pkg_list: NOP')
return []
rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path)
rpl_fname = Path(dstor_pkglist_path).name
try:
cm_utl.download(rpl_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: Reference package list: Download:'
f' {rpl_fname}: {rpl_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Reference package list: Download: {rpl_fname}: {rpl_url}:'
f' {type(e).__name__}: {e}') from e
rpl_fpath = self.inst_storage.tmp_dpath.joinpath(rpl_fname)
try:
return cr_utl.rc_load_json(
rpl_fpath, emheading=f'Reference package list: {rpl_fname}')
except cr_exc.RCError as e:
raise e
finally:
rpl_fpath.unlink()
def get_ref_pkg_list(cluster_conf, tmp_dpath):
"""Get the current reference package list.
:return: list, JSON-formatted data
"""
dstor_root_url = (cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_pkglist_path = (cluster_conf.get('distribution-storage',
{}).get('pkglistpath', ''))
# Unblock irrelevant local operations
if not cluster_conf or dstor_pkglist_path == 'NOP':
return []
rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path)
try:
rpl_fpath = cm_utl.download(rpl_url, tmp_dpath)
LOG.debug(f'Reference package list: Download: {rpl_fpath}: {rpl_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Reference package list: Download: {rpl_fpath}: {rpl_url}:'
f' {type(e).__name__}: {e}') from e
try:
return cr_utl.rc_load_json(
rpl_fpath, emheading=f'Reference package list: {rpl_fpath}')
except cr_exc.RCError as e:
raise e
finally:
rpl_fpath.unlink()
def download_controller(server_download, new_repos_queue):
conn, addr = server_download.accept_new_connection()
message = server_download.receive_message(conn)
params = json.loads(message)
branch = params["branch"]
name = params["url"].split('/')[-1][:-4]
uid = str(uuid.uuid4())
complete_name = "{}-{}-{}".format(uid, name, branch)
try:
download(params["url"], params["branch"],
"./repositories/{}".format(complete_name))
except Exception:
logging.error("Unable to download repository {}".format(params["url"]))
return
register = {"name": name, "branch": branch, "complete_name": complete_name}
new_repos_queue.put(register)
def _load_data(self):
maps = [{'e': 1, 'p': 0},
{'b': 'bell', 'c': 'conical', 'x': 'convex', 'f': 'flat', 'k': 'knobbed', 's': 'sunken'},
{'f': 'fibrous', 'g': 'grooves', 'y': 'scaly', 's': 'smooth'},
{'n': 'brown', 'b': 'buff', 'c': 'cinnamon', 'g': 'gray', 'r': 'green', 'p': 'pink',
'u': 'purple', 'e': 'red', 'w': 'white', 'y': 'yellow'},
{'t': 1, 'f': 0},
{'a': 'almond', 'l': 'anise', 'c': 'creosote', 'y': 'fishy', 'f': 'foul', 'm': 'musty',
'n': 'none', 'p': 'pungent', 's': 'spicy'},
{'a': 'attached', 'd': 'descending', 'f': 'free', 'n': 'notched'},
{'c': 'close', 'w': 'crowded', 'd': 'distant'},
{'b': 'broad', 'n': 'narrow'},
{'k': 'black', 'n': 'brown', 'b': 'buff', 'h': 'chocolate', 'g': 'gray', 'r': 'green',
'o': 'orange', 'p': 'pink', 'u': 'purple', 'e': 'red', 'w': 'white', 'y': 'yellow'},
{'e': 'enlarging', 't': 'tapering'},
{'b': 'bulbous', 'c': 'club', 'u': 'cup', 'e': 'equal',
'z': 'rhizomorphs', 'r': 'rooted', '?': 'missing'},
{'f': 'fibrous', 'y': 'scaly', 'k': 'silky', 's': 'smooth'},
{'f': 'fibrous', 'y': 'scaly', 'k': 'silky', 's': 'smooth'},
{'n': 'brown', 'b': 'buff', 'c': 'cinnamon', 'g': 'gray', 'o': 'orange', 'p': 'pink',
'e': 'red', 'w': 'white', 'y': 'yellow'},
{'n': 'brown', 'b': 'buff', 'c': 'cinnamon', 'g': 'gray', 'o': 'orange', 'p': 'pink',
'e': 'red', 'w': 'white', 'y': 'yellow'},
{'p': 'partial', 'u': 'universal'},
{'n': 'brown', 'o': 'orange', 'w': 'white', 'y': 'yellow'},
{'n': 'none', 'o': 'one', 't': 'two'},
{'c': 'cobwebby', 'e': 'evanescent', 'f': 'flaring', 'l': 'large', 'n': 'none',
'p': 'pendant', 's': 'sheathing', 'z': 'zone'},
{'k': 'black', 'n': 'brown', 'b': 'buff', 'h': 'chocolate', 'r': 'green',
'o': 'orange', 'u': 'purple', 'w': 'white', 'y': 'yellow'},
{'a': 'abundant', 'c': 'clustered', 'n': 'numerous', 's': 'scattered', 'v': 'several', 'y': 'solitary'},
{'g': 'grasses', 'l': 'leaves', 'm': 'meadows', 'p': 'paths', 'u': 'urban', 'w': 'waste', 'd': 'woods'}]
mushrooms = []
with open(download(self.remote_dir / "agaricus-lepiota.data", self.cache_dir)) as f:
for line in f.readlines():
tokens = line.strip().split(',')
mushrooms.append([map[token] for token, map in zip(tokens, maps)])
self._features_cols = ["cap_shape", "cap_surface", "cap_color",
"has_bruises",
"odor",
"gill_attachment", "gill_spacing", "gill_size", "gill_color",
"stalk_shape", "stalk_root",
"stalk_surface_above_ring", "stalk_surface_below_ring",
"stalk_color_above_ring", "stalk_color_below_ring",
"veil_type", "veil_color",
"ring_number", "ring_type",
"spore_print_color",
"population", "habitat"]
self._target_col = "is_eatable"
self._data = DataFrame(data=mushrooms,
columns=self.features_cols + [self.target_col])
def _load_data(self):
self._features_cols = [
'sepal length', 'sepal width', 'petal length', 'petal width'
]
self._target_col = 'iris class'
self._data = pd.read_csv(
download(self.remote_dir / "heart.csv", self.cache_dir))
self._target_col = 'target'
self._features_cols = list(self.data.columns)
self._features_cols.remove(self._target_col)
def test_download(rget, tmp_path):
# requests doesn't support file: URI's so, to avoid starting a HTTP
# server, mock requests behavior.
response = mock.Mock()
rget.return_value = response
response.iter_content.return_value = [
EXPECTED_FILE_CONTENT.encode('utf-8')
]
path = download('', str(tmp_path))
assert path.exists()
assert path.read_text(encoding='utf-8') == EXPECTED_FILE_CONTENT
def get_ref_pkg_list(self):
"""Get the current reference package list.
:return: list, JSON-formatted data
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function get_ref_pkg_list().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_pkglist_path = (self.cluster_conf.get('distribution-storage',
{}).get('pkglistpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_pkglist_path == 'NOP':
LOG.info(f'{self.msg_src}: ref_pkg_list: NOP')
return []
rpl_url = posixpath.join(dstor_root_url, dstor_pkglist_path)
rpl_fname = Path(dstor_pkglist_path).name
try:
cm_utl.download(rpl_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: Reference package list: Download:'
f' {rpl_fname}: {rpl_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Reference package list: Download: {rpl_fname}: {rpl_url}:'
f' {type(e).__name__}: {e}') from e
rpl_fpath = self.inst_storage.tmp_dpath.joinpath(rpl_fname)
try:
return cr_utl.rc_load_json(
rpl_fpath, emheading=f'Reference package list: {rpl_fname}')
except cr_exc.RCError as e:
raise e
finally:
rpl_fpath.unlink()
def get_dstor_dcoscfgpkg_path(self, dstor_root_url: str,
dstor_lpi_path: str):
"""Retrieve the Linux Package Index (LPI) object from the DC/OS
distribution storage and discover a relative URL to the DC/OS
aggregated configuration package.
LPI is expected to be a JSON-formatted file containing descriptors for
DC/OS distribution packages:
{
"<pkg-name>":{
"filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz",
"id":"<pkg-name>--<pkg-version>"
},
...
}
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_lpi_path: str, URL path to the DC/OS Linux package
index object at the DC/OS distribution
storage
:return dstor_dcoscfgpkg_path: str, URL path to the DC/OS aggregated
config package at the DC/OS distribution
storage
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function
# get_dstor_dcoscfgpkg_path().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dcos_conf_pkg_name = 'dcos-config-win'
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path)
try:
lpi_fpath = cm_utl.download(lpi_url, self.inst_storage.tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}:'
f' {e}') from e
try:
lpi = cr_utl.rc_load_json(lpi_fpath,
emheading='DC/OS Linux package index')
if not isinstance(lpi, dict):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi_url}: Invalid structure')
dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name)
if dcos_conf_pkg_desc is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor is missed:'
f' {dcos_conf_pkg_name}')
if not isinstance(dcos_conf_pkg_desc, dict):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS'
f' aggregated config package descriptor:'
f' {dcos_conf_pkg_desc}')
dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename')
if dstor_dcoscfgpkg_path is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path is'
f' missed: {dcos_conf_pkg_desc}')
if not isinstance(dstor_dcoscfgpkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path:'
f' Invalid type: {dstor_dcoscfgpkg_path}')
finally:
lpi_fpath.unlink()
return dstor_dcoscfgpkg_path
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
# TODO: Functionality implemented in this method needs to be reused
# in other application parts (e.g. CmdConfigUpgrade) and so, it
# has been arranged as a standalone function get_dcos_conf().
# Thus the CmdConfigSetup is to be moved to use that standalone
# function instead of this method to avoid massive code
# duplication.
dstor_root_url = self.root_url
dstor_linux_pkg_index_path = self.dcosclusterpkginfopath
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Discover relative URL to the DC/OS aggregated configuration package.
dstor_dcoscfg_pkg_path = self.get_dstor_dcoscfgpkg_path(
dstor_root_url, dstor_linux_pkg_index_path)
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
# Download DC/OS aggregated configuration package ...
try:
dcoscfg_pkg_fpath = cm_utl.download(dcoscfg_pkg_url,
self.inst_storage.tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' Download: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config package: {dcoscfg_pkg_url}:'
f' {type(e).__name__}: {e}') from e
# Process DC/OS aggregated configuration package.
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(dcoscfg_pkg_fpath, tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Extract: OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath, emheading=f'DC/OS aggregated config: Values')
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_template(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:'
f' {type(e).__name__}: {e}')
else:
raise
else:
LOG.debug(f'{self.msg_src}: DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Preprocess: OK')
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
from common.plot import plot_confusion_matrix, plot_pca_2d, plot_tsne_2d, plot_cnn_layer
from common.utils import download, extract, keras_model, retrieve_name, layer_functor
from datasets.cifar_dataset import load_data
if __name__ == '__main__':
np.random.seed(25)
classes_close = ['train', 'bicycle', 'tank', 'motorcycle', 'bus']
classes_far = ['shark', 'sunflower', 'bed', 'motorcycle', 'palm_tree']
train_all = False
model_subtype = "medium"
class_names = classes_far
data_dir = extract(
download(REMOTE_DATA_URL / "cifar" / "cifar100superclass.zip",
KERAS_DIR / "cifar"))
labels = {label: i for i, label in enumerate(class_names)}
img_shape = (32, 32)
num_classes = len(labels)
x_train, y_train = load_data(data_dir / "train", labels, img_shape)
x_test, y_test = load_data(data_dir / "test", labels, img_shape)
print(x_train.shape, y_train.shape)
print(x_test.shape, y_test.shape)
model = cifar_model_builder(model_subtype)(x_train.shape[1:], num_classes)
print(model.summary())
keras_model(
model,
def add_package(self, pkg_id, dstor_root_url, dstor_pkgrepo_path):
"""Add a package to the local package repository.
:param pkg_id: PackageId, package ID
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_pkgrepo_path: str, DC/OS distribution storage package
repository root path
"""
msg_src = self.__class__.__name__
# Download a package tarball
try:
cm_utl.download(
self._make_pkg_url(pkg_id=pkg_id,
dstor_root_url=dstor_root_url,
dstor_pkgrepo_path=dstor_pkgrepo_path),
str(self.tmp_dpath))
LOG.debug(f'{msg_src}: Add package: Download: {pkg_id}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}')
# Unpack a package tarball
pkgtarball_fpath = (self.tmp_dpath.joinpath(
pkg_id.pkg_id).with_suffix('.tar.xz'))
try:
cm_utl.unpack(str(pkgtarball_fpath), self.pkgrepo_dpath)
LOG.debug(f'{msg_src}: Add package: Extract: {pkg_id}')
except Exception as e:
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}')
finally:
pkgtarball_fpath.unlink()
# Create a work, runtime and log data directories for a package.
for host_dpath in self.work_dpath, self.run_dpath, self.log_dpath:
path = host_dpath.joinpath(pkg_id.pkg_name)
if not path.exists():
try:
path.mkdir(parents=True)
LOG.debug(f'{msg_src}: Add package: {pkg_id}:'
f' Create data directory: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: {type(e).__name__}: {e}') from e
elif path.is_symlink():
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: Symlink conflict')
elif path.is_reserved():
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: Reserved name conflict')
elif not path.is_dir():
# Attempt to auto-clean garbage
try:
path.unlink()
LOG.debug(
f'{msg_src}: Add package: {pkg_id}: Auto-cleanup:'
f' File: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup: File: {path}:'
f' {type(e).__name__}: {e}') from e
# Attempt to create data dir
try:
path.mkdir(parents=True)
LOG.debug(f'{msg_src}: Add package: {pkg_id}:'
f' Create data directory: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: {type(e).__name__}: {e}') from e
else:
# Leave existing directories intact
pass
# Workaround for dcos-diagnostics to be able to start
# TODO: Remove this code after correct dcos-diagnostics configuration
# is figured out and all its config files are arranged properly
# to support DC/OS installation storage FS layout.
if pkg_id.pkg_name == 'dcos-diagnostics':
# Move binary and config-files to DC/OS installation storage root
src_dpath = self.pkgrepo_dpath.joinpath(pkg_id.pkg_id, 'bin')
try:
LOG.debug(f'{msg_src}: Add package: Workaround: Copy list: '
f' {list(src_dpath.glob("*.*"))}')
for src_fpath in src_dpath.glob('*.*'):
if not self.root_dpath.joinpath(src_fpath.name).exists():
shutil.copy(str(src_fpath), str(self.root_dpath))
LOG.debug(
f'{msg_src}: Add package: Workaround: Copy file: '
f' {str(src_fpath)} -> {str(self.root_dpath)}')
# Create a folder for logs
log_dpath = self.root_dpath.joinpath('mesos-logs')
if not log_dpath.exists():
log_dpath.mkdir()
except Exception as e:
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}')
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
objects:
{
'package': {[
{'path': <str>, 'content': <str>},
...
]}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_dcoscfg_path = (self.cluster_conf.get('distribution-storage',
{}).get('dcoscfgpath', ''))
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_dcoscfg_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
dcoscfg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_path)
dcoscfg_fname = Path(dstor_dcoscfg_path).name
try:
cm_utl.download(dcoscfg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_fname}: {dcoscfg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_fname}:'
f' {dcoscfg_url}: {type(e).__name__}: {e}') from e
dcoscfg_fpath = self.inst_storage.tmp_dpath.joinpath(dcoscfg_fname)
try:
dcos_conf = cr_utl.rc_load_yaml(
dcoscfg_fpath,
emheading=f'DC/OS aggregated config: {dcoscfg_fname}',
render=True,
context=ResourceContext(
istor_nodes=self.inst_storage.istor_nodes,
cluster_conf=self.cluster_conf))
if (not isinstance(dcos_conf, dict)
or not isinstance(dcos_conf.get('package'), list)):
raise cr_exc.RCInvalidError(
f'DC/OS aggregated config: {dcos_conf}')
for element in dcos_conf.get('package'):
if (not isinstance(element, dict)
or not isinstance(element.get('path'), str)
or not isinstance(element.get('content'), str)):
raise cr_exc.RCElementError(
f'DC/OS aggregated config: {element}')
return dcos_conf
except cr_exc.RCError as e:
raise e
finally:
dcoscfg_fpath.unlink()
def test_download_should_return_destination_location(patch_smart_dl):
"""Download fake content and save it into file."""
patch_smart_dl().get_dest.return_value = os.getcwd()
path = utils.download('', '')
assert path == os.getcwd()
def get_dcos_conf(cluster_conf, tmp_dpath: Path):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if not cluster_conf or dstor_linux_pkg_index_path == 'NOP':
return {}
# Discover relative URL to the DC/OS aggregated configuration package.
dstor_dcoscfg_pkg_path = get_dstor_dcoscfgpkg_path(
dstor_root_url, dstor_linux_pkg_index_path, tmp_dpath)
# Download DC/OS aggregated configuration package ...
dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path)
try:
dcoscfg_pkg_fpath = cm_utl.download(dcoscfg_pkg_url, tmp_dpath)
LOG.debug(f'DC/OS aggregated config package:'
f' Download: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config package: {dcoscfg_pkg_url}:'
f' {type(e).__name__}: {e}') from e
# Process DC/OS aggregated configuration package.
try:
with tf.TemporaryDirectory(dir=str(tmp_dpath)) as tmp_dpath_:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath_)
LOG.debug(f'DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Extract: OK')
values_fpath = Path(tmp_dpath_).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath, emheading=f'DC/OS aggregated config: Values')
template_fpath = Path(tmp_dpath_).joinpath(template_fname)
template = load_dcos_conf_template(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config package: {dcoscfg_pkg_fpath}:'
f' {type(e).__name__}: {e}')
else:
raise
else:
LOG.debug(f'DC/OS aggregated config package:'
f' {dcoscfg_pkg_fpath}: Preprocess: OK')
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
def add_package(self, pkg_id: PackageId, dstor_root_url: str,
dstor_pkgrepo_path: str) -> None:
"""Add a package to the local package repository.
:param pkg_id: PackageId, package ID
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_pkgrepo_path: str, DC/OS distribution storage package
repository root path
"""
msg_src = self.__class__.__name__
# Download a package tarball
pkg_url = self._make_pkg_url(pkg_id=pkg_id,
dstor_root_url=dstor_root_url,
dstor_pkgrepo_path=dstor_pkgrepo_path)
try:
pkgtarball_fpath = cm_utl.download(pkg_url, self.tmp_dpath)
LOG.debug(f'{msg_src}: Add package: Download: {pkg_id}: {pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'Add package: {pkg_id}: {pkg_url}: {type(e).__name__}: {e}'
) from e
# Unpack a package tarball
try:
# Try to cleanup local package repository before trying to
# create a package installation directory there
pkg_inst_dpath = self.pkgrepo_dpath.joinpath(pkg_id.pkg_id)
try:
if pkg_inst_dpath.exists():
if pkg_inst_dpath.is_dir():
shutil.rmtree(str(pkg_inst_dpath))
elif pkg_inst_dpath.is_file and (
not pkg_inst_dpath.is_symlink()
):
pkg_inst_dpath.unlink()
else:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup'
f' package repository: Removing objects other than'
f' regular directories and files is not supported'
)
LOG.debug(f'{msg_src}: Add package: {pkg_id}: Auto-cleanup:'
f' {pkg_inst_dpath}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup: {pkg_inst_dpath}:'
f' {type(e).__name__}: {e}'
) from e
with tf.TemporaryDirectory(dir=str(self.tmp_dpath)) as temp_dpath:
cm_utl.unpack(pkgtarball_fpath, temp_dpath)
try:
# Lookup for a directory named after the package ID
src_dpath = [
path for path in Path(temp_dpath).iterdir() if (
path.name == pkg_id.pkg_id
)
][0]
if src_dpath.is_dir():
shutil.copytree(
str(src_dpath), str(pkg_inst_dpath)
)
else:
# Only a directory may be named after the package ID,
# otherwise a package structure is broken
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: Broken package structure'
)
except IndexError:
# Use the temporary directory as package's container
shutil.copytree(
temp_dpath, str(pkg_inst_dpath)
)
LOG.debug(f'{msg_src}: Add package: Extract: {pkg_id}')
except Exception as e:
if not isinstance(e, cr_exc.RCExtractError):
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}'
)
else:
raise
finally:
pkgtarball_fpath.unlink()
# Create a work, runtime and log data directories for a package.
for host_dpath in self.work_dpath, self.run_dpath, self.log_dpath:
path = host_dpath.joinpath(pkg_id.pkg_name)
if not path.exists():
try:
path.mkdir(parents=True)
LOG.debug(f'{msg_src}: Add package: {pkg_id}:'
f' Create data directory: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: {type(e).__name__}: {e}'
) from e
elif path.is_symlink():
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: Symlink conflict'
)
elif path.is_reserved():
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: Reserved name conflict'
)
elif not path.is_dir():
# Attempt to auto-clean garbage
try:
path.unlink()
LOG.debug(f'{msg_src}: Add package: {pkg_id}: Auto-cleanup:'
f' File: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Auto-cleanup: File: {path}:'
f' {type(e).__name__}: {e}'
) from e
# Attempt to create data dir
try:
path.mkdir(parents=True)
LOG.debug(f'{msg_src}: Add package: {pkg_id}:'
f' Create data directory: {path}')
except (OSError, RuntimeError) as e:
raise cr_exc.InstallationStorageError(
f'Add package: {pkg_id}: Create data directory:'
f' {path}: {type(e).__name__}: {e}'
) from e
else:
# Leave existing directories intact
pass
# Workaround for dcos-diagnostics to be able to start
# TODO: Remove this code after correct dcos-diagnostics configuration
# is figured out and all its config files are arranged properly
# to support DC/OS installation storage FS layout.
if pkg_id.pkg_name == 'dcos-diagnostics':
# Move binary and config-files to DC/OS installation storage root
src_dpath = self.pkgrepo_dpath.joinpath(pkg_id.pkg_id, 'bin')
try:
LOG.debug(
f'{msg_src}: Add package: Workaround: Copy list: '
f' {list(src_dpath.glob("*.*"))}'
)
for src_fpath in src_dpath.glob('*.*'):
if not self.root_dpath.joinpath(src_fpath.name).exists():
shutil.copy(str(src_fpath), str(self.root_dpath))
LOG.debug(
f'{msg_src}: Add package: Workaround: Copy file: '
f' {str(src_fpath)} -> {str(self.root_dpath)}'
)
# Create a folder for logs
log_dpath = self.root_dpath.joinpath('mesos-logs')
if not log_dpath.exists():
log_dpath.mkdir()
except Exception as e:
raise cr_exc.RCExtractError(
f'Add package: {pkg_id}: {type(e).__name__}: {e}'
)
def get_dstor_dcoscfgpkg_path(dstor_root_url: str, dstor_lpi_path: str,
tmp_dpath: str):
"""Retrieve the Linux Package Index (LPI) object from the DC/OS
distribution storage and discover a relative URL to the DC/OS
aggregated configuration package.
LPI is expected to be a JSON-formatted file containing descriptors for
DC/OS distribution packages:
{
"<pkg-name>":{
"filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz",
"id":"<pkg-name>--<pkg-version>"
},
...
}
:param dstor_root_url: str, DC/OS distribution storage root URL
:param dstor_lpi_path: str, URL path to the DC/OS Linux package
index object at the DC/OS distribution
storage
:return tmp_dpath: str, URL path to the DC/OS aggregated
config package at the DC/OS distribution
storage
"""
dcos_conf_pkg_name = 'dcos-config-win'
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path)
try:
lpi_fpath = cm_utl.download(lpi_url, tmp_dpath)
LOG.debug(f'DC/OS Linux package index: Download: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}: {e}'
) from e
try:
lpi = cr_utl.rc_load_json(lpi_fpath,
emheading='DC/OS Linux package index')
if not isinstance(lpi, dict):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi_url}: Invalid structure')
dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name)
if dcos_conf_pkg_desc is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor is missed:'
f' {dcos_conf_pkg_name}')
if not isinstance(dcos_conf_pkg_desc, dict):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS'
f' aggregated config package descriptor:'
f' {dcos_conf_pkg_desc}')
dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename')
if dstor_dcoscfgpkg_path is None:
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path is'
f' missed: {dcos_conf_pkg_desc}')
if not isinstance(dstor_dcoscfgpkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated'
f' config package descriptor: Distribution storage path:'
f' Invalid type: {dstor_dcoscfgpkg_path}')
finally:
lpi_fpath.unlink()
return dstor_dcoscfgpkg_path
def get_dcos_conf(self):
"""Get the DC/OS aggregated configuration object.
:return: dict, set of DC/OS shared and package specific configuration
templates coupled with 'key=value' substitution data
container:
{
'template': {
'package': [
{'path': <str>, 'content': <str>},
...
]
},
'values': {
key: value,
...
}
}
"""
dstor_root_url = (self.cluster_conf.get('distribution-storage',
{}).get('rooturl', ''))
dstor_linux_pkg_index_path = (self.cluster_conf.get(
'distribution-storage', {}).get('dcosclusterpkginfopath', ''))
dcos_conf_pkg_name = 'dcos-config-win'
template_fname = 'dcos-config-windows.yaml'
values_fname = 'expanded.config.full.json'
# Unblock irrelevant local operations
if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP':
LOG.info(f'{self.msg_src}: dcos_conf: NOP')
return {}
# Linux package index direct URL
lpi_url = posixpath.join(dstor_root_url, dstor_linux_pkg_index_path)
lpi_fname = Path(dstor_linux_pkg_index_path).name
try:
cm_utl.download(lpi_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:'
f' {lpi_fname}: {lpi_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS Linux package index: Download: {lpi_fname}:'
f' {lpi_url}: {type(e).__name__}: {e}') from e
lpi_fpath = self.inst_storage.tmp_dpath.joinpath(lpi_fname)
try:
lpi = cr_utl.rc_load_json(
lpi_fpath, emheading=f'DC/OS Linux package index: {lpi_fname}')
if (not isinstance(lpi, dict)
or not isinstance(lpi.get(dcos_conf_pkg_name), dict)):
raise cr_exc.RCInvalidError(
f'DC/OS Linux package index: {lpi}')
dstor_dcoscfg_pkg_path = lpi.get(dcos_conf_pkg_name).get(
'filename')
if not isinstance(dstor_dcoscfg_pkg_path, str):
raise cr_exc.RCElementError(
f'DC/OS Linux package index: DC/OS config package'
f' distribution storage path: {dstor_dcoscfg_pkg_path}')
except cr_exc.RCError as e:
raise e
finally:
lpi_fpath.unlink()
dcoscfg_pkg_url = posixpath.join(dstor_root_url,
dstor_dcoscfg_pkg_path)
dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name
# Download DC/OS aggregated configuration package ...
try:
cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath))
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:'
f' {dcoscfg_pkg_fname}: {dcoscfg_pkg_url}')
except Exception as e:
raise cr_exc.RCDownloadError(
f'DC/OS aggregated config: Download: {dcoscfg_pkg_fname}:'
f' {dcoscfg_pkg_url}: {type(e).__name__}: {e}') from e
dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath(
dcoscfg_pkg_fname)
try:
with tf.TemporaryDirectory(
dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath:
cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath)
LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Extract:'
f' OK')
values_fpath = Path(tmp_dpath).joinpath(values_fname)
values = cr_utl.rc_load_json(
values_fpath,
emheading=f'DC/OS aggregated config: Values: {values_fname}'
)
template_fpath = Path(tmp_dpath).joinpath(template_fname)
template = self.load_dcos_conf_templete(template_fpath)
except Exception as e:
if not isinstance(e, cr_exc.RCError):
raise cr_exc.RCExtractError(
f'DC/OS aggregated config: {type(e).__name__}: {e}')
else:
raise
else:
return {'template': template, 'values': values}
finally:
dcoscfg_pkg_fpath.unlink()
