def _rollbackCommand(self, name):
o, e, r = exxec.processToStringsWithResult(self.mainCommand() +
["--rollback-to", name])
la.LoggingAccess().log(e, vc.Verbosity.MOCK)
la.LoggingAccess().log(o, vc.Verbosity.MOCK)
la.LoggingAccess().log(str(self.listSnapshots()), vc.Verbosity.MOCK)
return e, o, r
def out(self):
mapOfArches = self.agregate1()
mapResults = self.agregate2(mapOfArches)
for key, svalue in mapResults.items():
la.LoggingAccess().stdout(key)
for value in svalue:
la.LoggingAccess().stdout(" - " + value)
def checkRegex(self, name=None):
self._document("RPM with version 10 or higher (except itw) has following regex: " + JAVA_REGEX10 + " in case "
"of normal packages and " + JAVA_REGEX_ROLLING + " in case of rolling release.")
if name.startswith("java-latest-openjdk"):
la.LoggingAccess().log("JDK rolling release regex check.")
return CRES_JAVA_REGEXEROLLING.match(name)
else:
la.LoggingAccess().log("JDK 10 and higher regex check.")
return CRES_JAVA_REGEXE10.match(name)
def _scrubLvmCommand(self):
o, e, r = exxec.processToStringsWithResult(self.mainCommand() +
["--scrub", "all"])
if r != 0:
failed = "Build chroot is locked, please restart the testsuite."
tu.passed_or_failed(self, False, failed)
raise utils.mock.mock_execution_exception.MockExecutionException(
failed)
la.LoggingAccess().log(e, vc.Verbosity.MOCK)
la.LoggingAccess().log(o, vc.Verbosity.MOCK)
la.LoggingAccess().log(str(self.listSnapshots()), vc.Verbosity.MOCK)
def closeTestSuite(passed, failed, mtc):
la.LoggingAccess().stdout("Arch-independet mehtods counted: " + str(mtc))
la.LoggingAccess().stdout("Passed=" + str(passed))
la.LoggingAccess().stdout("Failed=" + str(failed))
la.LoggingAccess().stdout("Total=" + str(failed + passed))
la.LoggingAccess().stdout("Test suite finished.")
if failed != 0:
if config.runtime_config.RuntimeConfig().diewith:
sys.exit(config.runtime_config.RuntimeConfig().diewith)
else:
raise Exception(str(failed) + " tests failed")
def _checkDest(dir):
absOne = os.path.abspath(dir)
if not os.path.exists(absOne):
la.LoggingAccess().log("Creating: " + absOne, vc.Verbosity.TEST)
os.mkdir(absOne)
if not os.path.isdir(absOne):
raise Exception(absOne + " Must be a directory, is not")
if not os.listdir(absOne) == []:
raise Exception(absOne + " Is not empty, please fix")
la.LoggingAccess().log("Using as download target: " + absOne,
vc.Verbosity.TEST)
return absOne
def getBuild(nvr):
"This method download build from brw. The filtering of arches depends on RuntimeConfig().getArchs();"
target = _checkDest(config.runtime_config.RuntimeConfig().getPkgsDir())
command = _getCommand(nvr)
la.LoggingAccess().log("using " + command, vc.Verbosity.TEST)
packages = _getBuildInfo(command, nvr)
if len(packages) == 0:
raise Exception("No pkgs to download. Verify build or archs")
la.LoggingAccess().log("going to download " + str(len(packages)) + " rpms",
vc.Verbosity.TEST)
_downloadBrewKojiBuilds(packages, target)
return True
def __init__(self,
os="fedora",
version="rawhide",
arch="x86_64",
command="mock"):
"""
This is a base constructor for DefaultMock. Arguments should never be changed when initiating new instance,
unless you need it for some valid reasons (so far, there are NONE).
Version of mock must be currently < 27 (26 is obsoleted though). This is necessary due to rich dependencies in
fedora 28 packages, that do not work on RHEL 7 VM's. As soon as we got good RHEL 8 images, we must switch back
to rawhide and run this framework there, so we do not have to switch chroot every year or so.
"""
self.os = os
self.version = version
self.arch = arch
self.command = command
self.inited = False
self.alternatives = False
self.snapshots = dict()
la.LoggingAccess().log(
"Providing new instance of " + self.getMockName(),
vc.Verbosity.MOCK)
# comment this, set inited and alternatives to true if debug of some test needs to be done in hurry, it is
# sometimes acting strange though, so I do not recommend it (overlayfs plugin is quite fast so take the time
self._scrubLvmCommand()
def importRpm(self, rpmPath, resetBuildRoot=True):
# there is restriciton to chars and length in/of vg name
key = re.sub('[^0-9a-zA-Z]+', '_', ntpath.basename(rpmPath))
if key in self.snapshots:
la.LoggingAccess().log(
rpmPath + " already extracted in snapshot. "
"Rolling to " + key, vc.Verbosity.MOCK)
return self.getSnapshot(key)
else:
la.LoggingAccess().log(
rpmPath + " not extracted in snapshot. Creating " + key,
vc.Verbosity.MOCK)
out, serr, res = self.importRpmCommand(rpmPath, resetBuildRoot)
self.createSnapshot(key)
self.snapshots[key] = rpmPath
return out, serr, res
def cross_check_artificial_provides(self, this):
files = self.rpms.files
self._document(
"according to Jvanek every provide should be provided only once per set of rpms with exception "
+ "of javadoc-zip having common provides with javadoc")
files = [
x.replace("rpms/", "") for x in files
if tu.validate_arch_for_rpms(this.current_arch) in x
]
if files:
la.LoggingAccess().log(
" Testing VersionRelease: " + ns.get_version_full(files[0]),
vc.Verbosity.TEST)
for i in range(len(files) - 1):
actual_provides = self._get_artificial_provides(files[i])
for j in range(i + 1, len(files)):
if ns.get_arch(files[i]) == ns.get_arch(files[j]):
if ("zip" in files[i] and "javadoc" in files[j]) or (
"zip" in files[j] and "javadoc" in files[i]):
continue
compared_provides = self._get_artificial_provides(files[j])
provides_intersection = [
provide for provide in actual_provides
if provide in compared_provides
]
tu.passed_or_failed(
self, not (len(provides_intersection)),
"{} and {} have common provides: {}".format(
make_rpm_readable(files[i]),
make_rpm_readable(files[j]),
", ".join(provides_intersection)))
return
def setArchs(self, archString):
if archString == "all":
self.archs = config.global_config.getAllArchs()
else:
words = archString.split(",")
self.archs = words
la.LoggingAccess().log("archs limited/forced to " + str(self.archs), vc.Verbosity.TEST)
def getDynamicArches(self, arch):
la.LoggingAccess().log("Getting dynamic arches for: " + arch, vc.Verbosity.MOCK)
output= rpmbuild_utils.rpmbuildEval(arch)
li = output.split(" ")
for i in range(len(li)):
li[i] = li[i].strip()
return li
def copyIns(self, cwd, srcs, dest):
# unlike copyIn, this copy list of files TO destination. Because of necessary relativity of srcs,
# CWD is included
o, e, r = exxec.processToStringsWithResult(
self.mainCommand() + ["--copyin"] + srcs + [dest], True, cwd)
la.LoggingAccess().log(e, vc.Verbosity.MOCK)
return o, e, r
def checkMajorVersionSimplified(self, version=None):
self._document(
"All jdsk (except icedtea-web) have major version in name. eg: java-1.8.0-openjdk or java-9-oracle."
" For legacy JDK (like jdk 1.6.0 or 1.7.1) the major version is included as middle number."
" For modern jdks (jdk 9+) the major version is just plain number. Eg.: java-9-ibm."
" The extracted middle number *is* number")
la.LoggingAccess().log("non itw call for checkMajorVersionSimplified")
return re.compile("[0-9]+").match(version) and int(version) > 0
def closeDocSuite(documented, ignored, failed):
la.LoggingAccess().log("done - Documented: " + str(documented) + " from total: " + str(documented+ignored+failed),
vc.Verbosity.TEST)
if failed != 0:
if config.runtime_config.RuntimeConfig().diewith:
sys.exit(config.runtime_config.RuntimeConfig().diewith)
else:
raise Exception(str(failed) + " docs failed")
def getSrciplet(rpmFile, scripletId):
if not isScripletNameValid(scripletId):
la.LoggingAccess().log(
"warning! Scriplet name " + scripletId +
" is not known. It should be one of: " +
",".join(ScripletStarterFinisher.allScriplets), vc.Verbosity.TEST)
key = rpmFile + "-" + scripletId
if key in scriptlets:
la.LoggingAccess().log(key + " already cached, returning",
vc.Verbosity.MOCK)
return scriptlets[key]
la.LoggingAccess().log(key + " not yet cached, reading", vc.Verbosity.MOCK)
sf = ScripletStarterFinisher(scripletId)
script = utils.process_utils.processAsStrings(
['rpm', '-qp', '--scripts', rpmFile], sf.start, sf.stop, False)
scriptlets[key] = script
return script
def __init__(self, ddir):
self.files = utils.test_utils.get_rpms(ddir)
self.topDirs = utils.test_utils.get_top_dirs(ddir)
self.names = []
for file in self.files:
name = ntpath.basename(file)
for part in tc.IGNOREDNAMEPARTS:
name = name.replace(part, "")
self.names.append(name)
allFiles = utils.test_utils.get_files(ddir)
la.LoggingAccess().log(
"Loaded list of " + str(len(self.files)) +
" rpms from directory " + ddir, vc.Verbosity.TEST)
if len(self.files) != len(allFiles):
la.LoggingAccess().log(
"Warning, some files in " + ddir + " - " +
str(len(allFiles) - len(self.files)) +
" are NOT rpms. Ignored", vc.Verbosity.TEST)
def passed_or_failed(instance, bool, iffailed, ifpassed=""):
try:
classname = sys._getframe(1).f_locals['self'].__class__.__name__
except Exception:
classname = "NoClass"
callermethod = sys._getframe().f_back.f_code.co_name
testcase = do.Testcase(classname, callermethod)
do.Tests().add_testcase(testcase)
if bool:
instance.passed += 1
if ifpassed != "":
la.LoggingAccess().log(" " + ifpassed, vc.Verbosity.TEST)
else:
if issubclass(instance.__class__, (cs.JdkConfiguration, bx.BaseTest)):
instance.failed += 1
la.LoggingAccess().log(" " + iffailed, vc.Verbosity.TEST)
testcase.set_view_file_stub(iffailed)
return bool
def get_top_dirs(directory):
"""Walk `directory' and get a list of all top directory names in it."""
la.LoggingAccess().log("Searching in " + directory + " for: top dirs", vc.Verbosity.TEST)
resList = []
for root, dirs, files in os.walk(directory):
for d in dirs:
if os.path.isdir(directory + "/" + d):
resList.append(d)
return resList
def getTestFiles():
dir = os.path.dirname(os.path.abspath(__file__))
flist = tu.get_files(dir+"/testcases", "_test.py")
la.LoggingAccess().log("Found " + str(len(flist))+" files to run.", vc.Verbosity.TEST)
for n, file in enumerate(flist):
flist[n] = file[len(dir)+1:-3].replace("/",".")
if config.global_config.leSort:
return sorted(flist)
return flist
def _exec(args, err=False, cwd=None):
la.LoggingAccess().log("executing: " + str(args), vc.Verbosity.MOCK)
if err:
# args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None,
# close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False, cwd=None, env=None ...
proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd)
else:
proc = Popen(args, stdin=PIPE, stdout=PIPE)
return proc
def postinstall_exception_checked(self, pkg):
try:
DefaultMock().install_postscript(pkg)
except utils.mock.mock_execution_exception.MockExecutionException:
la.LoggingAccess().log(
" " + "Postinstall script not found in " +
os.path.basename(pkg), vc.Verbosity.TEST)
return False
return True
def processToStringsWithResult(args, err=True, cwd=None):
proc = _exec(args, err, cwd)
out, err = proc.communicate()
output = out.decode('utf-8').strip()
outpute = ""
if err is not None:
outpute = err.decode('utf-8').strip()
la.LoggingAccess().log("got: " + output, vc.Verbosity.MOCK)
ret = proc.wait()
return output, outpute, ret
def setFromParser(self, args):
# Order matters a lot!
# logfile must go first
if args.logfile:
self.setLogsFile(args.logfile)
if args.version:
la.LoggingAccess().stdout(VERSION_STRING)
return False
# later it does not matter as logging is already going to log file
la.LoggingAccess().log(VERSION_STRING, vc.Verbosity.TEST)
# switches should go before commands, so commands can use them
if args.dir:
self.setPkgsDir(args.dir)
if args.archs:
self.setArchs(args.archs)
if args.build:
r = utils.build_downloader.getBuild(args.build)
# failed? exit...
if not r:
return False
if args.docs:
# no setter - should not be set from outside
self.docs = True
if args.noheader:
# no setter - should not be set from outside
self.header = False
if args.verbosity:
try:
verbosity = vc.Verbosity(int(args.verbosity))
except Exception:
raise AttributeError("Invalid verbosity argument, expected 1, 2 or 3, "
"but got {}.".format(args.verbosity))
self.set_verbosity(verbosity)
if args.diewith:
try:
val = int(args.diewith)
except Exception:
raise AttributeError("diewith argument must be integer")
self.diewith = val
return True
def handle_policytool(self, args=None):
self._document(tc.POLICYTOOL + " is a binary that behaves differently than normal binaries. It has binaries in {} "
"subpackages, but slaves are in {} "
"subpackages.".format(" and ".join(self._policytool_binary_subpackages()),
" and ".join(self._policytool_slave_subpackages())))
for subpackage in self._policytool_binary_subpackages():
try:
self.installed_binaries[subpackage].remove(tc.POLICYTOOL)
tu.passed_or_failed(self, True, "")
except KeyError:
la.LoggingAccess().log(subpackage + " - subpkg not present. This should be handled by subpkg_test")
except ValueError:
tu.passed_or_failed(self, False, tc.POLICYTOOL + " binary not present in " + subpackage)
for subpkg in self._policytool_slave_subpackages():
try:
self.installed_slaves[subpkg].remove(tc.POLICYTOOL)
tu.passed_or_failed(self, True, "")
except KeyError:
la.LoggingAccess().log(subpkg + " - subpkg not present. This should be handled by subpkg_test")
except ValueError:
tu.passed_or_failed(self, False, tc.POLICYTOOL + " slave not present in " + subpkg)
return
def processAsStringsWithResult(args, starter=None, finisher=None, initialCanRead=True, log = True):
""" read process line by line. starter and finisher are methods, which returns true/false to set read. Their input
is line"""
res = []
proc = _exec(args)
canRead=initialCanRead
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
line = line.strip()
if log:
la.LoggingAccess().log("reading: "+line, vc.Verbosity.MOCK)
if canRead and finisher is not None:
canRead = finisher(line)
if not canRead:
la.LoggingAccess().log(str(finisher)+" stopped recording", vc.Verbosity.MOCK)
if canRead:
res.append(line)
if not canRead and starter is not None:
canRead = starter(line)
if canRead:
la.LoggingAccess().log(str(starter) + " started recording", vc.Verbosity.MOCK)
ret = proc.wait()
return res, ret
def _install_scriptlet(self, pkg, scriptlet):
key = re.sub('[^0-9a-zA-Z]+', '_',
ntpath.basename(pkg) + "_" + scriptlet)
if key in self.snapshots:
la.LoggingAccess().log(
pkg + " already extracted in snapshot. Rolling to " + key,
vc.Verbosity.MOCK)
self.getSnapshot(key)
return
else:
self.importRpm(pkg)
content = utils.rpmbuild_utils.getSrciplet(pkg, scriptlet)
if len(content) == 0:
raise utils.mock.mock_execution_exception.MockExecutionException(
scriptlet + " scriptlet not found in given"
" package.")
else:
o, r = self.executeScriptlet(pkg, scriptlet)
la.LoggingAccess().log(
scriptlet + "returned " + str(r) + " of " +
os.path.basename(pkg), vc.Verbosity.MOCK)
self.createSnapshot(key)
def executeShell(command):
la.LoggingAccess().log("executing shell : " + command, vc.Verbosity.MOCK)
shell = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
o, e = shell.communicate()
if o is not None:
o = o.decode('utf-8').strip()
else:
o=""
if e is not None:
e = e.decode('utf-8').strip()
else:
e = ""
r = shell.wait()
return o, e, r
def runDocks():
header()
la.LoggingAccess().log("Running documentation", vc.Verbosity.TEST)
dlist = []
ilist = []
flist = []
files = getTestFiles()
for file in files:
test_module = importlib.import_module(file)
func = getattr(test_module, "documentAll")
passed, ignored, failed = func()
dlist.append(passed)
ilist.append(ignored)
flist.append(failed)
tu.closeDocSuite(sum(dlist), sum(ilist), sum(flist))
def runTasks():
header()
la.LoggingAccess().log("Running all testsuites", vc.Verbosity.TEST)
plist=[]
flist=[]
clist = []
files = getTestFiles()
for file in files:
test_module = importlib.import_module(file)
func = getattr(test_module, "testAll")
passed, failed, methods = func()
plist.append(passed)
flist.append(failed)
clist.append(methods)
tu.closeTestSuite(sum(plist), sum(flist), sum(clist))
