def load(cls, fpath): """Load package manifest from a file. :param fpath: pathlib.Path, path to a JSON-formatted manifest file. :return: dict, package manifest. """ m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest') # TODO: Add content verification (jsonschema) for m_body. Raise # ValueError, if conformance was not confirmed. try: manifest = cls( pkg_id=PackageId(pkg_id=m_body.get('pkg_id')), istor_nodes=IStorNodes( **{ k: Path(v) for k, v in m_body.get('context').get( 'istor_nodes').items() }), cluster_conf=m_body.get('context').get('cluster_conf'), pkg_info=m_body.get('pkg_info'), pkg_extcfg=m_body.get('pkg_extcfg'), pkg_svccfg=m_body.get('pkg_svccfg'), ) LOG.debug(f'Package manifest: Load: {fpath}') except (ValueError, AssertionError, TypeError) as e: err_msg = (f'Package manifest: Load:' f' {fpath}: {type(e).__name__}: {e}') raise cr_exc.RCInvalidError(err_msg) from e return manifest
def rc_load_ini(fpath, emheading=None): """Load INI-formatted data from a (resource) file. :param fpath: pathlib.Path, path to a source file :param emheading: str, heading to be added to the exception's description :return: dict, configparser.ConfigParser.read_dict() compatible data. """ assert isinstance(fpath, Path) and fpath.is_absolute(), ( f'Argument: fpath: Absolute pathlib.Path is required: {fpath}') cfg_parser = cfp.ConfigParser() try: with fpath.open() as fp: cfg_parser.read_file(fp) except FileNotFoundError: err_msg = f'Load: {fpath}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCNotFoundError(err_msg) except (OSError, RuntimeError) as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCError(err_msg) except cfp.Error as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCInvalidError(err_msg) else: return {k: dict(v) for k, v in cfg_parser.items()}
def rc_load_json(fpath, emheading=None): """Load JSON-formatted data from a (resource) file. :param fpath: pathlib.Path, path to a source file. :param emheading: str, heading to be added to the exception's description :return: json obj, JSON-formatted data """ assert isinstance(fpath, Path) and fpath.is_absolute(), ( f'Argument: fpath: Absolute pathlib.Path is required: {fpath}') try: with fpath.open() as fp: j_body = json.load(fp, strict=False) except FileNotFoundError: err_msg = f'Load: {fpath}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCNotFoundError(err_msg) except (OSError, RuntimeError) as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCError(err_msg) except cr_exc.JSON_ERRORS as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCInvalidError(err_msg) else: return j_body
def rc_load_ini(fpath, emheading=None, render=False, context=None): """Load INI-formatted data from a resource file. Content of a resource file can be pre-processed by Jinja2 rendering engine before being passed to INI-parser. :param fpath: pathlib.Path, path to a source file :param emheading: str, heading to be added to the exception's description :param render: bool, perform template rendering :param context: ResourceContext, rendering context data object :return: dict, configparser.ConfigParser.read_dict() compatible data. """ assert isinstance(fpath, Path) and fpath.is_absolute(), ( f'Argument: fpath: Absolute pathlib.Path is required: {fpath}' ) if context is None: context_items = {} else: assert isinstance(context, ResourceContext), ( f'Argument: context:' f' Got {type(context).__name__} instead of ResourceContext' ) context_items = context.get_items() cfg_parser = cfp.ConfigParser() try: if render is True: jj2_env = jj2.Environment( loader=jj2.FileSystemLoader(str(fpath.parent)) ) jj2_tmpl = jj2_env.get_template(str(fpath.name)) ini_str = jj2_tmpl.render(**context_items) LOG.debug(f'rc_load_ini(): ini_str: {ini_str}') cfg_parser.read_string(ini_str, source=str(fpath)) else: with fpath.open() as fp: cfg_parser.read_file(fp) except (FileNotFoundError, jj2.TemplateNotFound) as e: err_msg = f'Load: {fpath}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCNotFoundError(err_msg) from e except (OSError, RuntimeError) as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCError(err_msg) from e except (jj2.TemplateError, cfp.Error) as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCInvalidError(err_msg) from e else: return {k: dict(v) for k, v in cfg_parser.items()}
def rc_load_json(fpath, emheading=None, render=False, context=None): """Load JSON-formatted data from a resource file. Content of a resource file can be pre-processed by Jinja2 rendering engine before being passed to JSON-parser. :param fpath: pathlib.Path, path to a source file. :param emheading: str, heading to be added to the exception's description :param render: bool, perform template rendering :param context: ResourceContext, rendering context data object :return: json obj, JSON-formatted data """ assert isinstance(fpath, Path) and fpath.is_absolute(), ( f'Argument: fpath: Absolute pathlib.Path is required: {fpath}' ) if context is None: context_items = {} else: assert isinstance(context, ResourceContext), ( f'Argument: context:' f' Got {type(context).__name__} instead of ResourceContext' ) context_items = context.get_items(json_ready=True) try: if render is True: jj2_env = jj2.Environment( loader=jj2.FileSystemLoader(str(fpath.parent)) ) jj2_tmpl = jj2_env.get_template(str(fpath.name)) json_str = jj2_tmpl.render(**context_items) LOG.debug(f'rc_load_json(): json_str: {json_str}') j_body = json.loads(json_str, strict=False) else: with fpath.open() as fp: j_body = json.load(fp, strict=False) except (FileNotFoundError, jj2.TemplateNotFound) as e: err_msg = f'Load: {fpath}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCNotFoundError(err_msg) from e except (OSError, RuntimeError) as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCError(err_msg) from e except (jj2.TemplateError,) + cr_exc.JSON_ERRORS as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCInvalidError(err_msg) from e else: return j_body
def wrapper(fpath: Path, emheading: str = None, render: bool = False, context: ResourceContext = None) -> Any: try: content = func(fpath, emheading, render, context) except (FileNotFoundError, jj2.TemplateNotFound) as e: err_msg = f'Load: {fpath}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCNotFoundError(err_msg) from e except (OSError, RuntimeError) as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCError(err_msg) from e except (jj2.TemplateError, yaml.YAMLError) + cr_exc.JSON_ERRORS as e: err_msg = f'Load: {fpath}: {type(e).__name__}: {e}' err_msg = f'{emheading}: {err_msg}' if emheading else err_msg raise cr_exc.RCInvalidError(err_msg) from e else: return content
def load(cls, fpath): """Load package manifest from a file. :param fpath: pathlib.Path, path to a JSON-formatted manifest file. :return: dict, package manifest. """ m_body = cr_utl.rc_load_json(fpath, emheading='Package manifest') try: manifest = cls( pkg_id=PackageId(pkg_id=m_body.get('pkg_id')), pkgrepo_dpath=Path(m_body.get('pkgrepo_dpath')), pkgactive_dpath=Path(m_body.get('pkgactive_dpath')), pkg_info=m_body.get('pkg_info'), pkg_ini=m_body.get('pkg_ini'), svc_conf=m_body.get('svc_conf'), ) LOG.debug(f'Package manifest: Load: {fpath}') except (ValueError, AssertionError) as e: err_msg = (f'Package manifest: Load:' f' {fpath}: {type(e).__name__}: {e}') raise cr_exc.RCInvalidError(err_msg) return manifest
def get_dstor_dcoscfgpkg_path(dstor_root_url: str, dstor_lpi_path: str, tmp_dpath: str): """Retrieve the Linux Package Index (LPI) object from the DC/OS distribution storage and discover a relative URL to the DC/OS aggregated configuration package. LPI is expected to be a JSON-formatted file containing descriptors for DC/OS distribution packages: { "<pkg-name>":{ "filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz", "id":"<pkg-name>--<pkg-version>" }, ... } :param dstor_root_url: str, DC/OS distribution storage root URL :param dstor_lpi_path: str, URL path to the DC/OS Linux package index object at the DC/OS distribution storage :return tmp_dpath: str, URL path to the DC/OS aggregated config package at the DC/OS distribution storage """ dcos_conf_pkg_name = 'dcos-config-win' # Linux package index direct URL lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path) try: lpi_fpath = cm_utl.download(lpi_url, tmp_dpath) LOG.debug(f'DC/OS Linux package index: Download: {lpi_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}: {e}' ) from e try: lpi = cr_utl.rc_load_json(lpi_fpath, emheading='DC/OS Linux package index') if not isinstance(lpi, dict): raise cr_exc.RCInvalidError( f'DC/OS Linux package index: {lpi_url}: Invalid structure') dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name) if dcos_conf_pkg_desc is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor is missed:' f' {dcos_conf_pkg_name}') if not isinstance(dcos_conf_pkg_desc, dict): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS' f' aggregated config package descriptor:' f' {dcos_conf_pkg_desc}') dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename') if dstor_dcoscfgpkg_path is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path is' f' missed: {dcos_conf_pkg_desc}') if not isinstance(dstor_dcoscfgpkg_path, str): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path:' f' Invalid type: {dstor_dcoscfgpkg_path}') finally: lpi_fpath.unlink() return dstor_dcoscfgpkg_path
def get_dstor_dcoscfgpkg_path(self, dstor_root_url: str, dstor_lpi_path: str): """Retrieve the Linux Package Index (LPI) object from the DC/OS distribution storage and discover a relative URL to the DC/OS aggregated configuration package. LPI is expected to be a JSON-formatted file containing descriptors for DC/OS distribution packages: { "<pkg-name>":{ "filename":"<base-path>/<pkg-name>--<pkg-version>.tar.xz", "id":"<pkg-name>--<pkg-version>" }, ... } :param dstor_root_url: str, DC/OS distribution storage root URL :param dstor_lpi_path: str, URL path to the DC/OS Linux package index object at the DC/OS distribution storage :return dstor_dcoscfgpkg_path: str, URL path to the DC/OS aggregated config package at the DC/OS distribution storage """ # TODO: Functionality implemented in this method needs to be reused # in other application parts (e.g. CmdConfigUpgrade) and so, it # has been arranged as a standalone function # get_dstor_dcoscfgpkg_path(). # Thus the CmdConfigSetup is to be moved to use that standalone # function instead of this method to avoid massive code # duplication. dcos_conf_pkg_name = 'dcos-config-win' # Linux package index direct URL lpi_url = posixpath.join(dstor_root_url, dstor_lpi_path) try: lpi_fpath = cm_utl.download(lpi_url, self.inst_storage.tmp_dpath) LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:' f' {lpi_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS Linux package index: {lpi_url}: {type(e).__name__}:' f' {e}') from e try: lpi = cr_utl.rc_load_json(lpi_fpath, emheading='DC/OS Linux package index') if not isinstance(lpi, dict): raise cr_exc.RCInvalidError( f'DC/OS Linux package index: {lpi_url}: Invalid structure') dcos_conf_pkg_desc = lpi.get(dcos_conf_pkg_name) if dcos_conf_pkg_desc is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor is missed:' f' {dcos_conf_pkg_name}') if not isinstance(dcos_conf_pkg_desc, dict): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: Invalid DC/OS' f' aggregated config package descriptor:' f' {dcos_conf_pkg_desc}') dstor_dcoscfgpkg_path = dcos_conf_pkg_desc.get('filename') if dstor_dcoscfgpkg_path is None: raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path is' f' missed: {dcos_conf_pkg_desc}') if not isinstance(dstor_dcoscfgpkg_path, str): raise cr_exc.RCElementError( f'DC/OS Linux package index: {lpi_url}: DC/OS aggregated' f' config package descriptor: Distribution storage path:' f' Invalid type: {dstor_dcoscfgpkg_path}') finally: lpi_fpath.unlink() return dstor_dcoscfgpkg_path
def get_dcos_conf(self): """Get the DC/OS aggregated configuration object. :return: dict, set of DC/OS shared and package specific configuration objects: { 'package': {[ {'path': <str>, 'content': <str>}, ... ]} } """ dstor_root_url = (self.cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_dcoscfg_path = (self.cluster_conf.get('distribution-storage', {}).get('dcoscfgpath', '')) # Unblock irrelevant local operations if self.cluster_conf_nop or dstor_dcoscfg_path == 'NOP': LOG.info(f'{self.msg_src}: dcos_conf: NOP') return {} dcoscfg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_path) dcoscfg_fname = Path(dstor_dcoscfg_path).name try: cm_utl.download(dcoscfg_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:' f' {dcoscfg_fname}: {dcoscfg_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS aggregated config: Download: {dcoscfg_fname}:' f' {dcoscfg_url}: {type(e).__name__}: {e}') from e dcoscfg_fpath = self.inst_storage.tmp_dpath.joinpath(dcoscfg_fname) try: dcos_conf = cr_utl.rc_load_yaml( dcoscfg_fpath, emheading=f'DC/OS aggregated config: {dcoscfg_fname}', render=True, context=ResourceContext( istor_nodes=self.inst_storage.istor_nodes, cluster_conf=self.cluster_conf)) if (not isinstance(dcos_conf, dict) or not isinstance(dcos_conf.get('package'), list)): raise cr_exc.RCInvalidError( f'DC/OS aggregated config: {dcos_conf}') for element in dcos_conf.get('package'): if (not isinstance(element, dict) or not isinstance(element.get('path'), str) or not isinstance(element.get('content'), str)): raise cr_exc.RCElementError( f'DC/OS aggregated config: {element}') return dcos_conf except cr_exc.RCError as e: raise e finally: dcoscfg_fpath.unlink()
def get_dcos_conf(self): """Get the DC/OS aggregated configuration object. :return: dict, set of DC/OS shared and package specific configuration templates coupled with 'key=value' substitution data container: { 'template': { 'package': [ {'path': <str>, 'content': <str>}, ... ] }, 'values': { key: value, ... } } """ dstor_root_url = (self.cluster_conf.get('distribution-storage', {}).get('rooturl', '')) dstor_linux_pkg_index_path = (self.cluster_conf.get( 'distribution-storage', {}).get('dcosclusterpkginfopath', '')) dcos_conf_pkg_name = 'dcos-config-win' template_fname = 'dcos-config-windows.yaml' values_fname = 'expanded.config.full.json' # Unblock irrelevant local operations if self.cluster_conf_nop or dstor_linux_pkg_index_path == 'NOP': LOG.info(f'{self.msg_src}: dcos_conf: NOP') return {} # Linux package index direct URL lpi_url = posixpath.join(dstor_root_url, dstor_linux_pkg_index_path) lpi_fname = Path(dstor_linux_pkg_index_path).name try: cm_utl.download(lpi_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: DC/OS Linux package index: Download:' f' {lpi_fname}: {lpi_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS Linux package index: Download: {lpi_fname}:' f' {lpi_url}: {type(e).__name__}: {e}') from e lpi_fpath = self.inst_storage.tmp_dpath.joinpath(lpi_fname) try: lpi = cr_utl.rc_load_json( lpi_fpath, emheading=f'DC/OS Linux package index: {lpi_fname}') if (not isinstance(lpi, dict) or not isinstance(lpi.get(dcos_conf_pkg_name), dict)): raise cr_exc.RCInvalidError( f'DC/OS Linux package index: {lpi}') dstor_dcoscfg_pkg_path = lpi.get(dcos_conf_pkg_name).get( 'filename') if not isinstance(dstor_dcoscfg_pkg_path, str): raise cr_exc.RCElementError( f'DC/OS Linux package index: DC/OS config package' f' distribution storage path: {dstor_dcoscfg_pkg_path}') except cr_exc.RCError as e: raise e finally: lpi_fpath.unlink() dcoscfg_pkg_url = posixpath.join(dstor_root_url, dstor_dcoscfg_pkg_path) dcoscfg_pkg_fname = Path(dstor_dcoscfg_pkg_path).name # Download DC/OS aggregated configuration package ... try: cm_utl.download(dcoscfg_pkg_url, str(self.inst_storage.tmp_dpath)) LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Download:' f' {dcoscfg_pkg_fname}: {dcoscfg_pkg_url}') except Exception as e: raise cr_exc.RCDownloadError( f'DC/OS aggregated config: Download: {dcoscfg_pkg_fname}:' f' {dcoscfg_pkg_url}: {type(e).__name__}: {e}') from e dcoscfg_pkg_fpath = self.inst_storage.tmp_dpath.joinpath( dcoscfg_pkg_fname) try: with tf.TemporaryDirectory( dir=str(self.inst_storage.tmp_dpath)) as tmp_dpath: cm_utl.unpack(str(dcoscfg_pkg_fpath), tmp_dpath) LOG.debug(f'{self.msg_src}: DC/OS aggregated config: Extract:' f' OK') values_fpath = Path(tmp_dpath).joinpath(values_fname) values = cr_utl.rc_load_json( values_fpath, emheading=f'DC/OS aggregated config: Values: {values_fname}' ) template_fpath = Path(tmp_dpath).joinpath(template_fname) template = self.load_dcos_conf_templete(template_fpath) except Exception as e: if not isinstance(e, cr_exc.RCError): raise cr_exc.RCExtractError( f'DC/OS aggregated config: {type(e).__name__}: {e}') else: raise else: return {'template': template, 'values': values} finally: dcoscfg_pkg_fpath.unlink()