def fetch_data(url, out_file, ca_certs=None): """ Fetch data from a given URL. If the URL starts with https://, ca_certs can be a path to PEM file with CA certificate chain to validate server certificate. :param url: URL of the data :type url: str :param out_file: path to the output file :type out_file: str :param ca_certs: path to a PEM file with CA certificate chain :type ca_certs: str :raise WrongRequestError: if a wrong combination of arguments is passed (ca_certs file path given and url starting with http://) or arguments don't have required format :raise CertificateValidationError: if server certificate validation fails :raise FetchError: if data fetching fails (usually due to I/O errors) """ # create the directory for the out_file if it doesn't exist out_dir = os.path.dirname(out_file) utils.ensure_dir_exists(out_dir) if url.startswith("http://") or url.startswith("https://") \ or url.startswith("ftp://"): _fetch_http_ftp_data(url, out_file, ca_certs) else: msg = "Cannot fetch data from '%s': unknown URL format" % url raise UnknownURLformatError(msg)
def run(self): """Run the task.""" target_content_dir = utils.join_paths(self._sysroot, self._target_directory) utils.ensure_dir_exists(target_content_dir) if self._policy_data.content_type == "scap-security-guide": pass # nothing needed elif self._policy_data.content_type == "datastream": shutil.copy2(self._content_path, target_content_dir) elif self._policy_data.content_type == "rpm": # copy the RPM to the target system shutil.copy2(self._file_path, target_content_dir) # get the path of the RPM content_name = common.get_content_name(self._policy_data) package_path = utils.join_paths(self._target_directory, content_name) # and install it with yum ret = util.execInSysroot( "yum", ["-y", "--nogpg", "install", package_path]) if ret != 0: msg = _(f"Failed to install content RPM to the target system.") terminate(msg) return else: pattern = utils.join_paths(common.INSTALLATION_CONTENT_DIR, "*") utils.universal_copy(pattern, target_content_dir) if os.path.exists(self._tailoring_path): shutil.copy2(self._tailoring_path, target_content_dir)
def _extract_rpm(rpm_path, root="/", ensure_has_files=None): """ Extract the given RPM into the directory tree given by the root argument and make sure the given file exists in the archive. :param rpm_path: path to the RPM file that should be extracted :type rpm_path: str :param root: root of the directory tree the RPM should be extracted into :type root: str :param ensure_has_files: relative paths to the files that must exist in the RPM :type ensure_has_files: iterable of strings or None :return: a list of files and directories extracted from the archive :rtype: [str] """ # run rpm2cpio and process the output with the cpioarchive module temp_fd, temp_path = tempfile.mkstemp(prefix="oscap_rpm") proc = subprocess.Popen(["rpm2cpio", rpm_path], stdout=temp_fd) proc.wait() if proc.returncode != 0: msg = "Failed to convert RPM '%s' to cpio archive" % rpm_path raise ExtractionError(msg) os.close(temp_fd) try: archive = cpioarchive.CpioArchive(temp_path) except cpioarchive.CpioError as err: raise ExtractionError(err.message) # get entries from the archive (supports only iteration over entries) entries = set(entry for entry in archive) # cpio entry names (paths) start with the dot entry_names = [entry.name.lstrip(".") for entry in entries] for fpath in ensure_has_files or (): if not fpath in entry_names: msg = "File '%s' not found in the archive '%s'" % (fpath, rpm_path) raise ExtractionError(msg) for entry in entries: dirname = os.path.dirname(entry.name.lstrip(".")) out_dir = os.path.normpath(root + dirname) utils.ensure_dir_exists(out_dir) out_fpath = os.path.normpath(root + entry.name.lstrip(".")) with open(out_fpath, "wb") as out_file: buf = entry.read(IO_BUF_SIZE) while buf: out_file.write(buf) buf = entry.read(IO_BUF_SIZE) # cleanup archive.close() os.unlink(temp_path) return [os.path.normpath(root + name) for name in entry_names]
def fetch_data(url, out_file, ca_certs=None): """ Fetch data from a given URL. If the URL starts with https://, ca_certs can be a path to PEM file with CA certificate chain to validate server certificate. :param url: URL of the data :type url: str :param out_file: path to the output file :type out_file: str :param ca_certs: path to a PEM file with CA certificate chain :type ca_certs: str :raise WrongRequestError: if a wrong combination of arguments is passed (ca_certs file path given and url starting with http://) or arguments don't have required format :raise CertificateValidationError: if server certificate validation fails :raise FetchError: if data fetching fails (usually due to I/O errors) """ # create the directory for the out_file if it doesn't exist out_dir = os.path.dirname(out_file) utils.ensure_dir_exists(out_dir) if can_fetch_from(url): _curl_fetch(url, out_file, ca_certs) else: msg = "Cannot fetch data from '%s': unknown URL format" % url raise UnknownURLformatError(msg)
def extract_data(archive, out_dir, ensure_has_files=None): """ Fuction that extracts the given archive to the given output directory. It tries to find out the archive type by the file name. :param archive: path to the archive file that should be extracted :type archive: str :param out_dir: output directory the archive should be extracted to :type out_dir: str :param ensure_has_files: relative paths to the files that must exist in the archive :type ensure_has_files: iterable of strings or None :return: a list of files and directories extracted from the archive :rtype: [str] """ # get rid of empty file paths ensure_has_files = [fpath for fpath in ensure_has_files if fpath] if archive.endswith(".zip"): # ZIP file try: zfile = zipfile.ZipFile(archive, "r") except zipfile.BadZipfile as err: raise ExtractionError(err.message) # generator for the paths of the files found in the archive (dirs end # with "/") files = set(info.filename for info in zfile.filelist if not info.filename.endswith("/")) for fpath in ensure_has_files or (): if fpath not in files: msg = "File '%s' not found in the archive '%s'" % (fpath, archive) raise ExtractionError(msg) utils.ensure_dir_exists(out_dir) zfile.extractall(path=out_dir) result = [ utils.join_paths(out_dir, info.filename) for info in zfile.filelist ] zfile.close() return result elif archive.endswith(".tar"): # plain tarball return _extract_tarball(archive, out_dir, ensure_has_files, None) elif archive.endswith(".tar.gz"): # gzipped tarball return _extract_tarball(archive, out_dir, ensure_has_files, "gz") elif archive.endswith(".tar.bz2"): # bzipped tarball return _extract_tarball(archive, out_dir, ensure_has_files, "bz2") elif archive.endswith(".rpm"): # RPM return _extract_rpm(archive, out_dir, ensure_has_files) # elif other types of archives else: raise ExtractionError("Unsuported archive type")
def test_nonexisting_dir(mock_os, monkeypatch): mock_utils_os(mock_os, monkeypatch) mock_os.path.isdir.return_value = False utils.ensure_dir_exists("/tmp/test_dir") mock_os.path.isdir.assert_called_with("/tmp/test_dir") mock_os.makedirs.assert_called_with("/tmp/test_dir")
def extract_data(archive, out_dir, ensure_has_files=None): """ Fuction that extracts the given archive to the given output directory. It tries to find out the archive type by the file name. :param archive: path to the archive file that should be extracted :type archive: str :param out_dir: output directory the archive should be extracted to :type out_dir: str :param ensure_has_files: relative paths to the files that must exist in the archive :type ensure_has_files: iterable of strings or None :return: a list of files and directories extracted from the archive :rtype: [str] """ # get rid of empty file paths ensure_has_files = [fpath for fpath in ensure_has_files if fpath] if archive.endswith(".zip"): # ZIP file try: zfile = zipfile.ZipFile(archive, "r") except zipfile.BadZipfile as err: raise ExtractionError(str(err)) # generator for the paths of the files found in the archive (dirs end # with "/") files = set(info.filename for info in zfile.filelist if not info.filename.endswith("/")) for fpath in ensure_has_files or (): if fpath not in files: msg = "File '%s' not found in the archive '%s'" % (fpath, archive) raise ExtractionError(msg) utils.ensure_dir_exists(out_dir) zfile.extractall(path=out_dir) result = [utils.join_paths(out_dir, info.filename) for info in zfile.filelist] zfile.close() return result elif archive.endswith(".tar"): # plain tarball return _extract_tarball(archive, out_dir, ensure_has_files, None) elif archive.endswith(".tar.gz"): # gzipped tarball return _extract_tarball(archive, out_dir, ensure_has_files, "gz") elif archive.endswith(".tar.bz2"): # bzipped tarball return _extract_tarball(archive, out_dir, ensure_has_files, "bz2") elif archive.endswith(".rpm"): # RPM return _extract_rpm(archive, out_dir, ensure_has_files) # elif other types of archives else: raise ExtractionError("Unsuported archive type")
def execute(self, storage, ksdata, instclass, users, payload): """ The execute method that should make changes to the installed system. It is called only once in the post-install setup phase. :see: setup :param users: information about created users :type users: pyanaconda.users.Users instance """ if self.dry_run or not self.profile_id: # nothing more to be done in the dry-run mode or if no profile is # selected return target_content_dir = utils.join_paths(getSysroot(), common.TARGET_CONTENT_DIR) utils.ensure_dir_exists(target_content_dir) if self.content_type == "datastream": shutil.copy2(self.preinst_content_path, target_content_dir) elif self.content_type == "rpm": # copy the RPM to the target system shutil.copy2(self.raw_preinst_content_path, target_content_dir) # and install it with yum ret = iutil.execInSysroot( "yum", ["-y", "--nogpg", "install", self.raw_postinst_content_path]) if ret != 0: raise common.ExtractionError("Failed to install content " "RPM to the target system") elif self.content_type == "scap-security-guide": # nothing needed pass else: utils.universal_copy( utils.join_paths(common.INSTALLATION_CONTENT_DIR, "*"), target_content_dir) if os.path.exists(self.preinst_tailoring_path): shutil.copy2(self.preinst_tailoring_path, target_content_dir) common.run_oscap_remediate(self.profile_id, self.postinst_content_path, self.datastream_id, self.xccdf_id, self.postinst_tailoring_path, chroot=getSysroot())
def _extract_tarball(archive, out_dir, ensure_has_files, alg): """ Extract the given TAR archive to the given output directory and make sure the given file exists in the archive. :see: extract_data :param alg: compression algorithm used for the tarball :type alg: str (one of "gz", "bz2") or None :return: a list of files and directories extracted from the archive :rtype: [str] """ if alg and alg not in ( "gz", "bz2", ): raise ExtractionError("Unsupported compression algorithm") mode = "r" if alg: mode += ":%s" % alg try: tfile = tarfile.TarFile.open(archive, mode) except tarfile.TarError as err: raise ExtractionError(err.message) # generator for the paths of the files found in the archive files = set(member.path for member in tfile.getmembers() if member.isfile()) for fpath in ensure_has_files or (): if fpath not in files: msg = "File '%s' not found in the archive '%s'" % (fpath, archive) raise ExtractionError(msg) utils.ensure_dir_exists(out_dir) tfile.extractall(path=out_dir) result = [ utils.join_paths(out_dir, member.path) for member in tfile.getmembers() ] tfile.close() return result
def execute(self, storage, ksdata, users, payload): """ The execute method that should make changes to the installed system. It is called only once in the post-install setup phase. :see: setup :param users: information about created users :type users: pyanaconda.users.Users instance """ if self.dry_run or not self.profile_id: # nothing more to be done in the dry-run mode or if no profile is # selected return target_content_dir = utils.join_paths(getSysroot(), common.TARGET_CONTENT_DIR) utils.ensure_dir_exists(target_content_dir) if self.content_type == "datastream": shutil.copy2(self.preinst_content_path, target_content_dir) elif self.content_type == "rpm": # copy the RPM to the target system shutil.copy2(self.raw_preinst_content_path, target_content_dir) # and install it with yum ret = util.execInSysroot("yum", ["-y", "--nogpg", "install", self.raw_postinst_content_path]) if ret != 0: raise common.ExtractionError("Failed to install content " "RPM to the target system") elif self.content_type == "scap-security-guide": # nothing needed pass else: utils.universal_copy(utils.join_paths(common.INSTALLATION_CONTENT_DIR, "*"), target_content_dir) if os.path.exists(self.preinst_tailoring_path): shutil.copy2(self.preinst_tailoring_path, target_content_dir) common.run_oscap_remediate(self.profile_id, self.postinst_content_path, self.datastream_id, self.xccdf_id, self.postinst_tailoring_path, chroot=getSysroot())
def schedule_firstboot_remediation(chroot, profile, fpath, ds_id="", xccdf_id="", tailoring=""): if not profile: return "" # make sure the directory for the results exists results_dir = os.path.dirname(RESULTS_PATH) results_dir = os.path.normpath(chroot + "/" + results_dir) utils.ensure_dir_exists(results_dir) log.info("OSCAP addon: Scheduling firstboot remediation") _schedule_firstboot_remediation(chroot, profile, fpath, RESULTS_PATH, REPORT_PATH, ds_id, xccdf_id, tailoring) return ""
def _extract_tarball(archive, out_dir, ensure_has_files, alg): """ Extract the given TAR archive to the given output directory and make sure the given file exists in the archive. :see: extract_data :param alg: compression algorithm used for the tarball :type alg: str (one of "gz", "bz2") or None :return: a list of files and directories extracted from the archive :rtype: [str] """ if alg and alg not in ("gz", "bz2",): raise ExtractionError("Unsupported compression algorithm") mode = "r" if alg: mode += ":%s" % alg try: tfile = tarfile.TarFile.open(archive, mode) except tarfile.TarError as err: raise ExtractionError(str(err)) # generator for the paths of the files found in the archive files = set(member.path for member in tfile.getmembers() if member.isfile()) for fpath in ensure_has_files or (): if fpath not in files: msg = "File '%s' not found in the archive '%s'" % (fpath, archive) raise ExtractionError(msg) utils.ensure_dir_exists(out_dir) tfile.extractall(path=out_dir) result = [utils.join_paths(out_dir, member.path) for member in tfile.getmembers()] tfile.close() return result
def test_no_dir(mock_os, monkeypatch): mock_utils_os(mock_os, monkeypatch) # shouldn't raise an exception utils.ensure_dir_exists("")
def extract_data(archive, out_dir, ensure_has_files=None): """ Fuction that extracts the given archive to the given output directory. It tries to find out the archive type by the file name. :param archive: path to the archive file that should be extracted :type archive: str :param out_dir: output directory the archive should be extracted to :type out_dir: str :param ensure_has_files: relative paths to the files that must exist in the archive :type ensure_has_files: iterable of strings or None :return: a list of files and directories extracted from the archive :rtype: [str] """ if not ensure_has_files: ensure_has_files = [] # get rid of empty file paths if not ensure_has_files: ensure_has_files = [] else: ensure_has_files = [fpath for fpath in ensure_has_files if fpath] msg = "OSCAP addon: Extracting {archive}".format(archive=archive) if ensure_has_files: msg += ", expecting to find {files} there.".format( files=tuple(ensure_has_files)) log.info(msg) result = [] if archive.endswith(".zip"): # ZIP file try: zfile = zipfile.ZipFile(archive, "r") except Exception as exc: msg = _(f"Error extracting archive as a zipfile: {exc}") raise ExtractionError(msg) # generator for the paths of the files found in the archive (dirs end # with "/") files = set(info.filename for info in zfile.filelist if not info.filename.endswith("/")) for fpath in ensure_has_files or (): if fpath not in files: msg = "File '%s' not found in the archive '%s'" % (fpath, archive) raise ExtractionError(msg) utils.ensure_dir_exists(out_dir) zfile.extractall(path=out_dir) result = [ utils.join_paths(out_dir, info.filename) for info in zfile.filelist ] zfile.close() elif archive.endswith(".tar"): # plain tarball result = _extract_tarball(archive, out_dir, ensure_has_files, None) elif archive.endswith(".tar.gz"): # gzipped tarball result = _extract_tarball(archive, out_dir, ensure_has_files, "gz") elif archive.endswith(".tar.bz2"): # bzipped tarball result = _extract_tarball(archive, out_dir, ensure_has_files, "bz2") elif archive.endswith(".rpm"): # RPM result = _extract_rpm(archive, out_dir, ensure_has_files) # elif other types of archives else: raise ExtractionError("Unsuported archive type") log.info("OSCAP addon: Extracted {files} from the supplied content".format( files=result)) return result
def run_oscap_remediate(profile, fpath, ds_id="", xccdf_id="", tailoring="", chroot=""): """ Run the evaluation and remediation with the oscap tool on a given file, doing the remediation as defined in a given profile defined in a given checklist that is a part of a given datastream. If requested, run in chroot. :param profile: id of the profile that will drive the remediation :type profile: str :param fpath: path to a file with SCAP content :type fpath: str :param ds_id: ID of the datastream that contains the checklist defining the profile :type ds_id: str :param xccdf_id: ID of the checklist that defines the profile :type xccdf_id: str :param tailoring: path to a tailoring file :type tailoring: str :param chroot: path to the root the oscap tool should be run in :type chroot: str :return: oscap tool's stdout (summary of the rules, checks and fixes) :rtype: str """ if not profile: return "" def do_chroot(): """Helper function doing the chroot if requested.""" if chroot and chroot != "/": os.chroot(chroot) os.chdir("/") # make sure the directory for the results exists results_dir = os.path.dirname(RESULTS_PATH) if chroot: results_dir = os.path.normpath(chroot + "/" + results_dir) utils.ensure_dir_exists(results_dir) args = ["oscap", "xccdf", "eval"] args.append("--remediate") args.append("--results=%s" % RESULTS_PATH) # oscap uses the default profile by default if profile.lower() != "default": args.append("--profile=%s" % profile) if ds_id: args.append("--datastream-id=%s" % ds_id) if xccdf_id: args.append("--xccdf-id=%s" % xccdf_id) if tailoring: args.append("--tailoring-file=%s" % tailoring) args.append(fpath) try: proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=do_chroot) except OSError as oserr: msg = "Failed to run the oscap tool: %s" % oserr raise OSCAPaddonError(msg) (stdout, stderr) = proc.communicate() # save stdout? # pylint thinks Popen has no attribute returncode # pylint: disable-msg=E1101 if proc.returncode not in (0, 2) or stderr: # 0 -- success; 2 -- no error, but checks/remediation failed msg = "Content evaluation and remediation with the oscap tool "\ "failed: %s" % stderr raise OSCAPaddonError(msg) return stdout
def run_oscap_remediate(profile, fpath, ds_id="", xccdf_id="", tailoring="", chroot=""): """ Run the evaluation and remediation with the oscap tool on a given file, doing the remediation as defined in a given profile defined in a given checklist that is a part of a given datastream. If requested, run in chroot. :param profile: id of the profile that will drive the remediation :type profile: str :param fpath: path to a file with SCAP content :type fpath: str :param ds_id: ID of the datastream that contains the checklist defining the profile :type ds_id: str :param xccdf_id: ID of the checklist that defines the profile :type xccdf_id: str :param tailoring: path to a tailoring file :type tailoring: str :param chroot: path to the root the oscap tool should be run in :type chroot: str :return: oscap tool's stdout (summary of the rules, checks and fixes) :rtype: str """ if not profile: return "" def do_chroot(): """Helper function doing the chroot if requested.""" if chroot and chroot != "/": os.chroot(chroot) os.chdir("/") # make sure the directory for the results exists results_dir = os.path.dirname(RESULTS_PATH) if chroot: results_dir = os.path.normpath(chroot + "/" + results_dir) utils.ensure_dir_exists(results_dir) args = ["oscap", "xccdf", "eval"] args.append("--remediate") args.append("--results=%s" % RESULTS_PATH) args.append("--report=%s" % REPORT_PATH) # oscap uses the default profile by default if profile.lower() != "default": args.append("--profile=%s" % profile) if ds_id: args.append("--datastream-id=%s" % ds_id) if xccdf_id: args.append("--xccdf-id=%s" % xccdf_id) if tailoring: args.append("--tailoring-file=%s" % tailoring) args.append(fpath) proc = SubprocessLauncher(args) proc.execute(preexec_fn=do_chroot) proc.log_messages() if proc.returncode not in (0, 2): # 0 -- success; 2 -- no error, but checks/remediation failed msg = "Content evaluation and remediation with the oscap tool "\ "failed: %s" % proc.stderr raise OSCAPaddonError(msg) return proc.stdout
def run_oscap_remediate(profile, fpath, ds_id="", xccdf_id="", tailoring="", chroot=""): """ Run the evaluation and remediation with the oscap tool on a given file, doing the remediation as defined in a given profile defined in a given checklist that is a part of a given datastream. If requested, run in chroot. :param profile: id of the profile that will drive the remediation :type profile: str :param fpath: path to a file with SCAP content :type fpath: str :param ds_id: ID of the datastream that contains the checklist defining the profile :type ds_id: str :param xccdf_id: ID of the checklist that defines the profile :type xccdf_id: str :param tailoring: path to a tailoring file :type tailoring: str :param chroot: path to the root the oscap tool should be run in :type chroot: str :return: oscap tool's stdout (summary of the rules, checks and fixes) :rtype: str """ if not profile: return "" def do_chroot(): """Helper function doing the chroot if requested.""" if chroot and chroot != "/": os.chroot(chroot) os.chdir("/") # make sure the directory for the results exists results_dir = os.path.dirname(RESULTS_PATH) if chroot: results_dir = os.path.normpath(chroot + "/" + results_dir) utils.ensure_dir_exists(results_dir) args = ["oscap", "xccdf", "eval"] args.append("--remediate") args.append("--results=%s" % RESULTS_PATH) args.append("--report=%s" % REPORT_PATH) # oscap uses the default profile by default if profile.lower() != "default": args.append("--profile=%s" % profile) if ds_id: args.append("--datastream-id=%s" % ds_id) if xccdf_id: args.append("--xccdf-id=%s" % xccdf_id) if tailoring: args.append("--tailoring-file=%s" % tailoring) args.append(fpath) try: proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=do_chroot) except OSError as oserr: msg = "Failed to run the oscap tool: %s" % oserr raise OSCAPaddonError(msg) (stdout, stderr) = proc.communicate() messages = re.findall(r'OpenSCAP Error:.*', stderr) if messages: for message in messages: log.warning("OSCAP addon: " + message) # save stdout? # pylint thinks Popen has no attribute returncode # pylint: disable-msg=E1101 if proc.returncode not in (0, 2): # 0 -- success; 2 -- no error, but checks/remediation failed msg = "Content evaluation and remediation with the oscap tool "\ "failed: %s" % stderr raise OSCAPaddonError(msg) return stdout