def execute(self, outputFolder):
success = os.path.exists(os.path.join(outputFolder, self.pathName))
if (not success):
messager.printError("assertion pathExists " + self.pathName +
" failed. Cannot locate path " +
os.path.join(outputFolder, self.pathName))
return success
def test_printErrorShouldPrintInRed(styleMock, echoMock):
styleMock.return_value = formattedMessage
printError(message)
echoMock.assert_called_once_with(formattedMessage)
styleMock.assert_called_once_with(message, fg="red")
def execute(self, outputFolder):
success = not rules_util.path_exists_case_sensitive(
self.pathName, parent_path=outputFolder)
if (not success):
messager.printError("assertion pathNotExists " + self.pathName +
" failed. path " +
os.path.join(outputFolder, self.pathName) +
" exists")
return success
def execute(self, outputFolder):
workingDir = str(os.path.join(outputFolder, self.runFolder))
scriptprocess = subprocess.Popen(self.script,
cwd=workingDir,
shell=True)
scriptprocess.wait()
success = scriptprocess.returncode == 0
if (not success):
errorMessage = "assertion runScript {} {} failed. with non-zero return code [{}]".format(
self.runFolder, self.script, scriptprocess.returncode)
messager.printError(errorMessage)
return success
def execute(self, outputFolder):
if (not path_exists_case_sensitive(self.fileName,
parent_path=outputFolder)):
messager.printError(
"assertion fileHasMatchingLine {0} {1} failed. {0} does not exist in {2}."
.format(self.fileName, self.regex, outputFolder))
return False
fileLines = readLinesFromFile(self.fileName, folder=outputFolder)
filtered = filter(lambda line: re.search(self.regex, line), fileLines)
success = len(list(filtered)) > 0
if not success:
messager.printError(
"assertion fileHasMatchingLine {0} {1} failed. Matching line not found in {2}/{0}."
.format(self.fileName, self.regex, outputFolder))
return success
def execute(self, outputFolder):
if (not path_exists_case_sensitive(self.fileName,
parent_path=outputFolder)):
messager.printError(
"assertion fileContainsLine {0} {1} failed. {0} does not exist in {2}."
.format(self.fileName, self.line, outputFolder))
return False
fileLines = readLinesFromFile(self.fileName, folder=outputFolder)
success = self.line in fileLines
if not success:
messager.printError(
"assertion fileContainsLine {0} {1} failed. Matching line not found in {2}/{0}."
.format(self.fileName, self.line, outputFolder))
return success
def runAllTests(templatefolder, visible_whitespace):
"""Runs all test folders in the test directory"""
success = True
cli_options = create_cli_options(visible_whitespace)
try :
printMessage('Running all tests in %s' % templatefolder)
success = test_coordinator.runAllTestsInAllFolders(templatefolder, cli_options)
except:
printError(traceback.format_exc())
sys.exit(-1)
if (success):
printSuccess('All tests passed')
else : # non-zero exit code to indicate failure
printError('There were failing tests')
sys.exit(1)
def printDifferences(self, outputFile, fixtureFile):
try:
outputLines = rules_util.readLinesFromFile(outputFile,
removeNewline=False)
fixtureLines = rules_util.readLinesFromFile(fixtureFile,
removeNewline=False)
i = 0
for diffLine in difflib.unified_diff(outputLines,
fixtureLines,
fromfile=outputFile,
tofile=fixtureFile):
if (i > 2):
diffLine = self.getVisibleWhitespace(diffLine)
styledLine = diffLine
if (diffLine.startswith("+")):
styledLine = click.style(diffLine, fg='blue')
elif (diffLine.startswith("-")):
styledLine = click.style(diffLine, fg="yellow")
click.echo(styledLine)
i = i + 1
except UnicodeDecodeError:
messager.printError(
"One or both files are binary, unable to print differences")
def runAllTestsInAllFolders(projectRootFolder, cli_options):
rootTestFolder = projectRootFolder + '/test'
if (not os.path.isdir(rootTestFolder)):
messager.printError(
f'No test folder found, expecting to find one at {rootTestFolder}')
return False
testFolders = folder_scanner.findAllTestFolders(rootTestFolder)
if (len(testFolders) == 0):
messager.printError(
f'No test cases found. Expecting to find one or more subdirectories in {rootTestFolder} with valid config.yaml and assertions.yaml files'
)
return False
allTestsPass = True
for folder in testFolders:
folderSuccess = test_folder_executor.executeAllTestsInFolder(
projectRootFolder, folder, cli_options)
allTestsPass = allTestsPass and folderSuccess
stale_test_folder_cleanup.delete_stale_test_folders(rootTestFolder)
return allTestsPass
def execute(self, outputFolder):
if (not path_exists_case_sensitive(self.fileName,
parent_path=outputFolder)):
messager.printError(
"assertion fileContainsSnippet {0} {1} failed. {0} does not exist in {2}."
.format(self.fileName, self.snippetFile, outputFolder))
return False
if (not path_exists_case_sensitive(self.snippetFile,
parent_path=self.testFolder)):
messager.printError(
"assertion fileContainsSnippet {0} {1} failed. {1} does not exist in {2}."
.format(self.fileName, self.snippetFile, self.testFolder))
return False
fileLines = readLinesFromFile(self.fileName, folder=outputFolder)
snippetLines = readLinesFromFile(self.snippetFile,
folder=self.testFolder)
success = snippetExistsInFile(snippetLines, fileLines)
if not success:
messager.printError(
"assertion fileContainsSnippet {0} {1} failed. Matching lines from {1} not found in {2}/{0}."
.format(self.fileName, self.snippetFile, outputFolder))
return success
def execute(self, outputFolder):
fixtureFile = os.path.join(self.testFolder, self.fixturePath)
outputFile = os.path.join(outputFolder, self.fileName)
if (not rules_util.path_exists_case_sensitive(fixtureFile)):
messager.printError(
"assertion fileMatches " + self.fileName + " " +
self.fixturePath + " failed. " + os.path.abspath(
os.path.join(self.testFolder, self.fixturePath)) +
" does not exist")
return False
if (not rules_util.path_exists_case_sensitive(outputFile)):
messager.printError(
"assertion fileMatches " + self.fileName + " " +
self.fixturePath + " failed. " +
os.path.abspath(os.path.join(outputFolder, self.fileName)) +
" does not exist")
return False
success = filecmp.cmp(outputFile, fixtureFile, shallow=False)
if (not success):
messager.printError("assertion fileMatches " + self.fileName +
" " + self.fixturePath +
" failed. Files differ")
self.printDifferences(outputFile, fixtureFile)
return success
def parseAssertionFile(assertionFile, testFolder, cli_options):
rules = []
with open(assertionFile, 'r') as yamlAssertionFile:
assertionData = yaml.load(yamlAssertionFile, Loader=yaml.FullLoader)
options = {}
if ('options' in assertionData):
options = assertionData['options']
options.update(cli_options)
for ruleString in assertionData["assertions"]:
tokens = ruleString.split()
if (tokens[0] == "pathExists"):
rule = PathExistsRule(options, testFolder, tokens[1])
rules.append(rule)
elif (tokens[0] == "fileMatches"):
rule = FileMatchesRule(options, testFolder, tokens[1],
tokens[2])
rules.append(rule)
elif (tokens[0] == "pathNotExists"):
rule = PathNotExistsRule(options, testFolder, tokens[1])
rules.append(rule)
elif (tokens[0] == "runScript"):
script = getRestOfLineWithSpacesStartingWithToken(
2, tokens, ruleString)
rule = RunScriptRule(options, testFolder, tokens[1], script)
rules.append(rule)
elif (tokens[0] == "fileContainsLine"):
line = getRestOfLineWithSpacesStartingWithToken(
2, tokens, ruleString)
rule = FileContainsLineRule(options, testFolder, tokens[1],
line)
rules.append(rule)
elif (tokens[0] == "fileDoesNotContainLine"):
line = getRestOfLineWithSpacesStartingWithToken(
2, tokens, ruleString)
rule = FileDoesNotContainLineRule(options, testFolder,
tokens[1], line)
rules.append(rule)
elif (tokens[0] == "fileHasMatchingLine"):
regex = getRestOfLineWithSpacesStartingWithToken(
2, tokens, ruleString)
rule = FileHasRegexMatchLineRule(options, testFolder,
tokens[1], regex)
rules.append(rule)
elif (tokens[0] == "fileDoesNotHaveMatchingLine"):
regex = getRestOfLineWithSpacesStartingWithToken(
2, tokens, ruleString)
rule = FileDoesNotRegexMatchRule(options, testFolder,
tokens[1], regex)
rules.append(rule)
elif (tokens[0] == "fileContainsSnippet"):
rule = FileContainsSnippetRule(options, testFolder, tokens[1],
tokens[2])
rules.append(rule)
elif (tokens[0] == "fileDoesNotContainSnippet"):
rule = FileDoesNotContainSnippetRule(options, testFolder,
tokens[1], tokens[2])
rules.append(rule)
else:
messager.printError(
"Unable to parse assertion file {}. Error on assertion \"{}\""
.format(assertionFile, ruleString))
return []
return rules