def main():
SignalHandler.init()
args = parseArguments()
scriptFiles = getScriptFiles(args.start, args.scriptName)
testSuits = createTestSuits(scriptFiles, args.start)
markTestSkipedByPattern(testSuits, args.pattern)
genTestExecuter = [TestExecuter(ts, tc, args.upgradeScriptFile) for ts in testSuits for tc in ts.test_cases if not tc.is_skipped()]
if (args.jobs == None):
runner = Serial(genTestExecuter, failfast=args.failfast)
else:
runner = Parallel(genTestExecuter, failfast=args.failfast, max_workers = args.jobs)
startTime = time.time()
processStartTime = time.process_time()
res = runner.run(".", printer = Printer(args.verbose != None))
processTime = time.process_time() - processStartTime
executionTime = time.time() - startTime
if (args.failfast and res == False) or SignalHandler.failFast():
for ts in testSuits:
for tc in ts.test_cases:
if tc.elapsed_sec == None:
if (SignalHandler.failFast()):
tc.add_skipped_info("skipped due to " + SignalHandler.signalName())
else:
tc.add_skipped_info("skipped due to --failfast argument")
report = getReport(testSuits, processTime, executionTime)
Printer(args.verbose != None).print(report["text"])
if (args.reportfile):
ensure_dir(args.reportfile)
with open(args.reportfile, "w") as fw:
fw.write(TestSuite.to_xml_string(testSuits))
exit(report["error"] > 0)
def __run_powershell(self,
replCmd,
workingDir=".",
variables={},
printer=Printer()):
if self.inputfile:
printer.error("inputfile not allowed when shell = ", self.shell)
return false
inputData = str.encode(replCmd)
subProcessRes = subprocess.run(
["powershell", "-NonInteractive", "-NoLogo"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
input=inputData,
cwd=workingDir,
shell=False)
if self.outputfile:
outputfileFullPath = str(
pathlib.PurePath(workingDir).joinpath(
replaceVariables(self.outputfile, variables)))
ensure_dir(outputfileFullPath)
with open(outputfileFullPath, "wb") as fh:
fh.write(subProcessRes.stdout)
res = True
if (self.exitcode != None):
res = subProcessRes.returncode == self.exitcode
if not res:
printer.error("different exitcode received:",
subProcessRes.returncode, "!=", self.exitcode,
"for command '" + str(replCmd) + "'")
return res
def test_failfast(self):
class MockExecuter:
def __init__(self, res):
self.res = res
def run(self, workingDir=".", variables={}, printer=Printer()):
self.executed = True
return self.res
mockExecuters = [
MockExecuter(True),
MockExecuter(False),
MockExecuter(True)
]
loop = Serial(mockExecuters, failfast=True)
self.assertFalse(loop.run(workingDir=".", printer=Printer()))
self.assertTrue(hasattr(mockExecuters[0], 'executed'))
self.assertTrue(hasattr(mockExecuters[1], 'executed'))
self.assertFalse(hasattr(mockExecuters[2], 'executed'))
mockExecuters = [
MockExecuter(True),
MockExecuter(False),
MockExecuter(True)
]
loop = Serial(mockExecuters, failfast=False)
self.assertFalse(loop.run())
self.assertTrue(hasattr(mockExecuters[0], 'executed'))
self.assertTrue(hasattr(mockExecuters[1], 'executed'))
self.assertTrue(hasattr(mockExecuters[2], 'executed'))
def test_entries(self):
class MockExecuter:
def __init__(self):
self.variables = []
def run(self, workingDir, variables, printer=Printer()):
self.variables.append(variables)
self.workingDir = workingDir
self.printer = printer
return True
mockExecuter = MockExecuter()
loop = Parallel([mockExecuter],
entries=[{
"var1": "val1"
}, {
"var2": "val2"
}])
self.assertTrue(loop.run(".", printer=Printer()))
self.assertTrue(mockExecuter.variables == [{
"var1": "val1"
}, {
"var2": "val2"
}])
self.assertTrue(mockExecuter.workingDir == ".")
self.assertTrue(isinstance(mockExecuter.printer, BufferedPrinter))
def run(self, workingDir, variables = {}, printer = Printer()):
curDir = os.getcwd()
printer.print("["+"RUN".ljust(10) + "] ---", self.testSuite.name + ("/" if self.testSuite.name != "" else "") + self.testCase.name)
try:
startTime = time.time()
processStartTime = time.process_time()
data = processScriptData(self.testCase.file, saveUpgradedScript = self.upgradeScriptFile, printer = printer)
block = generateSerialBlock(data)
if block == None: return
res = block.run(workingDir = str(PurePath(self.testCase.file).parent), printer = printer)
if (not res):
self.testCase.add_failure_info(printer.errorOutput or "failed")
except Exception as e:
self.testCase.add_error_info(str(e))
printer.print(str(e))
finally:
self.testCase.elapsed_sec = time.time() - startTime
testResStr = "PASSED"
if self.testCase.is_error():
testResStr = "ERROR"
elif self.testCase.is_failure():
testResStr = "FAILED"
printStr = "["+testResStr.rjust(10) + "] --- " + self.testSuite.name + ("/" if self.testSuite.name != "" else "") + self.testCase.name
if (printer.isVerbose()):
printStr += " --- process time: "+ '%.2f' % (time.process_time() - processStartTime) + " execution time: "+ '%.2f' % (time.time() - startTime)
printer.print(printStr)
return testResStr == "PASSED"
def test_basic(self):
printer = Printer()
testPath = pathlib.Path('TestFiles/RmdirBlockTest/Output/TestDir1')
if not os.path.exists(testPath):
os.makedirs(testPath)
self.assertTrue(testPath.is_dir())
block = RmdirBlock('RmdirBlockTest/Output')
res = block.run(workingDir="TestFiles")
self.assertEqual(res, True)
self.assertFalse(pathlib.Path('TestFiles/Output').is_dir())
def run(self, workingDir="", variables={}, printer=Printer()):
printer.verbose("Diff:", self.paths)
_leftpath = replaceVariables(self.paths[0], variables, printer)
_rightpath = replaceVariables(self.paths[1], variables, printer)
_ignore = []
for ignoreLine in self.ignore:
_ignoreline = replaceVariables(ignoreLine, variables, printer)
_ignore.append(_ignoreline)
workingDirPath = pathlib.Path(workingDir)
return diff(workingDirPath.joinpath(_leftpath),
workingDirPath.joinpath(_rightpath), self.binarycompare,
self.strategy, _ignore, printer)
def processScriptData(file, saveUpgradedScript=False, printer=Printer()):
with open(file) as f:
data = yaml.load(f, Loader=SafeLineLoader)
if data == None: return data
(upgradedScript, data) = __upgradeScriptData(data, printer)
if saveUpgradedScript and upgradedScript:
dumpData = copy.deepcopy(data)
__removeLineInfo(dumpData)
with open(file, "w") as f:
yaml.dump(dumpData, f)
data.pop("version", None)
return data
def test_basicVerbose(self):
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = Printer(verbose=True)
cmdString = r"python -c \"print('Hello World')\""
cmdExe = RunBlock(cmdString)
res = cmdExe.run(printer=printer)
sys.stdout = sys.__stdout__
expectedResult = 'Run: python -c \\"print(\'Hello World\')\\"' + "\n" + "cwd: " + os.getcwd(
) + "\n" + 'call: ' + cmdString + "\n"
self.assertEqual(res, True)
self.assertEqual(expectedResult, capturedOutput.getvalue())
def rmtree(path, printer=Printer()):
try:
def remove_readonly(fn, path, excinfo):
os.chmod(path, stat.S_IWRITE)
fn(path)
if os.path.exists(path):
shutil.rmtree(path, onerror=remove_readonly, ignore_errors=False)
return True
except Exception as e:
printer.error("rmtree:", e)
return False
def test_basic(self):
class MockExecuter:
def run(self, workingDir, variables={}, printer=Printer()):
self.executed = True
self.workingDir = workingDir
self.printer = printer
return True
mockExecuter = MockExecuter()
loop = Parallel([mockExecuter], max_workers=4)
self.assertTrue(loop.run(".", printer=Printer()))
self.assertTrue(hasattr(mockExecuter, 'executed'))
self.assertTrue(mockExecuter.workingDir, '.')
self.assertTrue(isinstance(mockExecuter.printer, BufferedPrinter))
def test_unexpectedExitCodeVerbose(self):
printer = Printer(verbose=True)
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
cmdString = '"' + sys.executable + '"' + ' -c "exit(1)"'
cmdExe = RunBlock(cmdString, exitcode=0)
res = cmdExe.run(printer=printer)
sys.stdout = sys.__stdout__
expectedResult = 'Run: ' + cmdString + "\n" + "cwd: " + os.getcwd(
) + "\n" + 'call: ' + cmdString + "\n" + 'Error: different exitcode received: 1 != 0 for command \'' + cmdString + "\'\n"
self.assertEqual(res, False)
resStr = capturedOutput.getvalue()
self.assertEqual(expectedResult, capturedOutput.getvalue())
def test_exitCodeNone(self):
printer = Printer(verbose=True)
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
cmdString = '"' + sys.executable + '"' + ' -c "exit(1)"'
cmdExe = RunBlock(cmdString)
res = cmdExe.run(printer=printer)
sys.stdout = sys.__stdout__
expectedResult = 'Run: ' + cmdString + "\n" + "cwd: " + os.getcwd(
) + "\n" + 'call: ' + cmdString + "\n"
self.assertEqual(res, True)
resStr = capturedOutput.getvalue()
self.assertEqual(expectedResult, capturedOutput.getvalue())
def test_basicError(self):
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = Printer()
printer.error("Test error")
self.assertEqual("Error: Test error\n", capturedOutput.getvalue())
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = BufferedPrinter()
printer.error("Test error")
self.assertEqual("", capturedOutput.getvalue())
printer.flush()
self.assertEqual("Error: Test error\n", capturedOutput.getvalue())
sys.stdout = sys.__stdout__
def run(self, workingDir=".", variables={}, printer=Printer()):
printer.verbose("Run:", self.cmd)
replCmd = replaceVariables(self.cmd, variables)
printer.verbose("cwd:", os.getcwd())
printer.verbose("call:", replCmd)
cmdNArgs = shlex.split(replCmd)
if self.shell == "powershell":
return self.__run_powershell(replCmd, workingDir, variables,
printer)
if self.shell == "cmd":
return self.__run_cmd(replCmd, workingDir, variables, printer)
if self.shell == "sh":
return self.__run_sh(replCmd, workingDir, variables, printer)
elif self.shell != None:
printer.error("unknown shell ", self.shell)
return false
inputfileData = None
if self.inputfile:
with open(
pathlib.PurePath(workingDir).joinpath(
replaceVariables(self.inputfile, variables)),
"rb") as fh:
inputfileData = fh.read()
shell = (sys.platform == "win32")
subProcessRes = subprocess.run(cmdNArgs,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
input=inputfileData,
cwd=workingDir,
shell=shell)
if self.outputfile:
outputfileFullPath = str(
pathlib.PurePath(workingDir).joinpath(
replaceVariables(self.outputfile, variables)))
ensure_dir(outputfileFullPath)
with open(outputfileFullPath, "wb") as fh:
fh.write(subProcessRes.stdout)
res = True
if (self.exitcode != None):
res = subProcessRes.returncode == self.exitcode
if not res:
printer.error("different exitcode received:",
subProcessRes.returncode, "!=", self.exitcode,
"for command '" + str(replCmd) + "'")
return res
def run(self, workingDir="", variables={}, printer=Printer()):
res = True
if self.entries:
for loopEntry in self.entries:
if (SignalHandler.failFast()):
printer.error(SignalHandler.signalName() + " received.")
return False
callVariables = variables.copy()
callVariables.update(loopEntry)
if (not self._runEntry(workingDir, callVariables, printer)):
if (self.failfast): return False
else: res = False
else:
return self._runEntry(workingDir, variables, printer)
return res
def test_inputfileNoutputfile(self):
rmtree("Output")
printer = Printer(verbose=True)
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
cmdExe = RunBlock(
'"' + sys.executable + '"' + " RunBlock_test/stdinTostdout.py",
inputfile="RunBlock_test/stdinTostdout.py",
outputfile="RunBlock_test/Output/inputfileNoutputfile.txt")
res = cmdExe.run(workingDir="TestFiles", printer=printer)
sys.stdout = sys.__stdout__
self.assertEqual(res, True)
with open("TestFiles/RunBlock_test/stdinTostdout.py", "r") as fh1:
with open(
"TestFiles/RunBlock_test/Output/inputfileNoutputfile.txt",
"r") as fh2:
self.assertEqual(fh1.read(), fh2.read())
rmtree("TestFiles/RunBlock_test/Output")
def replaceVariables(line, dictReplVar, printer=Printer()):
global __replaceVariablesExp
if line == None: return (True, line)
searchStart = 0
while (True):
m = __replaceVariablesExp.search(line[searchStart:])
if not m:
break
foundVar = m.group()[2:-1]
replaceVal = dictReplVar.get(foundVar)
if (replaceVal):
strReplaceVal = str(replaceVal)
line = line[:searchStart +
m.start()] + strReplaceVal + line[searchStart +
m.end():]
searchStart += m.start() + len(strReplaceVal)
else:
raise GeneratorException("Variable " + str(m.group()) +
" does not exist")
line = line.replace(r'\$', "$")
return line
def run(self, workingDir="", variables={}, printer=Printer()):
self.listFutures = []
with ThreadPoolExecutor(max_workers=self.max_workers) as poolExecutor:
if self.entries:
for loopEntry in self.entries:
callVariables = variables.copy()
callVariables.update(loopEntry)
self._runEntry(poolExecutor,
workingDir=workingDir,
variables=callVariables,
printer=printer)
else:
self._runEntry(poolExecutor,
workingDir=workingDir,
variables=variables,
printer=printer)
if threading.current_thread().name == 'MainThread':
while (poolExecutor._work_queue.qsize()):
time.sleep(0.05)
for future in self.listFutures:
if future.cancelled() or future.result() == False: return False
return True
def run(self, workingDir="", variables={}, printer=Printer()):
printer.verbose("Rmdir:", self.dir)
_dir = replaceVariables(self.dir, variables, printer)
return rmtree(str(pathlib.PurePath(workingDir).joinpath(_dir)),
printer)
def test_printSeparator(self):
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = Printer(verbose=True)
printer.print("Hello", "World", sep=",")
printer.verbose("How", "do", "you", "do?", sep=",")
self.assertEqual("Hello,World\nHow,do,you,do?\n",
capturedOutput.getvalue())
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = printer.getBufferedPrinter()
printer.print("Hello", "World", sep=",")
printer.verbose("How", "do", "you", "do?", sep=",")
printer.flush()
self.assertEqual("Hello,World\nHow,do,you,do?\n",
capturedOutput.getvalue())
sys.stdout = sys.__stdout__
def test_basic(self):
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = Printer()
printer.print("Hello World")
printer.verbose("How do you do?")
self.assertEqual("Hello World\n", capturedOutput.getvalue())
capturedOutput = io.StringIO()
sys.stdout = capturedOutput
printer = Printer(verbose=True)
printer.print("Hello World")
printer.verbose("How do you do?")
self.assertEqual("Hello World\nHow do you do?\n",
capturedOutput.getvalue())
sys.stdout = sys.__stdout__
def run(self, workingDir, variables, printer=Printer()):
self.variables = variables
self.workingDir = workingDir
return True
def run(self, workingDir, variables, printer=Printer()):
self.variables.append(variables)
self.workingDir = workingDir
self.printer = printer
return True
def run(self, workingDir=".", variables={}, printer=Printer()):
self.executed = True
return self.res
def run(self, workingDir="", variables={}, printer=Printer()):
printer.verbose("Makedirs:", self.dir)
_dir = replaceVariables(self.dir, variables, printer)
_dir = str(pathlib.PurePath(workingDir).joinpath(_dir)) + "/"
ensure_dir(_dir)
return True
def diff(pathL,
pathR,
binaryCompare=False,
diffStrategy=diffStrategy.All,
ignore=[],
printer=Printer(verbose=True)):
rootM = pathlib.Path(pathL)
rootS = pathlib.Path(pathR)
compiledignore = list(map(lambda x: re.compile(x), ignore))
if (rootM.is_file() and rootS.is_file()):
if (not diffFiles(rootM,
rootS,
compiledignore=compiledignore,
binaryCompare=binaryCompare)):
printer.error(f"diff failed: '{rootM}' != '{rootS}'")
return False
return True
for file in [rootM, rootS]:
if not file.exists():
printer.error(f"diff failed: '{file}' doesn't exist")
return False
filesM = set(__listAllFilesWithoutRoot(pathL))
filesS = set(__listAllFilesWithoutRoot(pathR))
swapFiles = False
if diffStrategy == diffStrategy.IgnoreLeftOrphans:
swapFiles = True
filesM, filesS = filesS, filesM
rootM, rootS = rootS, rootM
if diffStrategy != diffStrategy.IgnoreOrphans:
for file in (filesM - filesS):
printer.error(
f"diff failed: '{str(rootM/file)}' not in '{rootS / file.parent}'"
)
return False
if diffStrategy == diffStrategy.All:
for file in (filesS - filesM):
printer.error(
f"diff failed: '{str(rootS/file)}' not in '{rootM / file.parent}'"
)
return False
comPaths = [(rootM / path, rootS / path) for path in (filesM & filesS)]
for pathM, pathS in comPaths:
if (pathM.is_file() != pathS.is_file()):
(pathL_, pathR_) = (pathM, pathS) if not swapFiles else (pathS,
pathM)
printer.error(f"diff failed: '{pathL_}' != '{pathR_}'")
return False
comFiles = [(pathM, pathS) for pathM, pathS in comPaths if pathM.is_file()]
for fileM, fileS in comFiles:
if (not diffFiles(fileM,
fileS,
binaryCompare=binaryCompare,
compiledignore=compiledignore)):
(fileL, fileR) = (fileM, fileS) if not swapFiles else (fileS,
fileM)
printer.error(f"diff failed: '{fileL}' != '{fileR}'")
return False
return True
def run(self, workingDir, variables={}, printer=Printer()):
self.executed = True
self.workingDir = workingDir
self.printer = printer
return True