def execute(self):
self.log.info(u'Temp dir = %s', tempfile.gettempdir())
assert os.path.exists(tempfile.gettempdir())
env = self.getDefaultEnvirons()
with openfile(self.output + '/env.txt', 'w', encoding='utf-8') as f:
for k in sorted(env.keys()):
f.write(u'%s=%s\n' % (k, env[k]))
env = self.getDefaultEnvirons(command=sys.executable)
with openfile(self.output + '/env-python.txt', 'w',
encoding='utf-8') as f:
for k in sorted(env.keys()):
f.write(u'%s=%s\n' % (k, env[k]))
self.startProcess(command=sys.executable,
arguments=[self.input + '/test.py'],
stdout='python.out',
stderr='python.err') #, ignoreExitStatus=True)
if os.path.exists(self.output +
'/mytemp'): # prevent it getting purged
with openfile(self.output + '/mytemp' + '/tmpfile.txt',
'w',
encoding='ascii') as f:
f.write(u'xxx')
def createProjectConfig(targetdir, templatepath=None):
"""Create a new project configuration file in the specified targetdir.
"""
if not templatepath: templatepath = getProjectConfigTemplates()['default']
mkdir(targetdir)
# using ascii ensures we don't unintentionally add weird characters to the default (utf-8) file
with openfile(templatepath, encoding='ascii') as src:
with openfile(os.path.abspath(targetdir+'/'+DEFAULT_PROJECTFILE[0]), 'w', encoding='ascii') as target:
for l in src:
l = l.replace('@PYTHON_VERSION@', '%s.%s.%s'%sys.version_info[0:3])
l = l.replace('@PYSYS_VERSION@', '.'.join(__version__.split('.')[0:3]))
target.write(l)
def execute(self):
self.log.info('This test is executing fine')
self.log.info('self.input = <%s>', self.input)
self.log.info('self.output = <%s>', self.output)
self.mkdir('somedirectory')
self.deletedir('somedirectory')
openfile(toLongPathSafe(self.output + '/purgablefile.txt'),
'w').close() # should get purged, if purging is working
with openfile(toLongPathSafe(self.output + '/somefile.txt'),
'w',
encoding='ascii') as f:
f.write(u'Hello world - logFileContents')
self.logFileContents(self.output + '/somefile.txt')
self.addOutcome(PASSED)
def loadProperties(path, encoding='utf-8-sig'):
"""
Reads keys and values from the specified ``.properties`` file.
Support ``#`` and ``!`` comments but does not perform any special handling of backslash ``\\`` characters
(either for escaping or for line continuation). Leading and trailing whitespace around keys and values
is stripped. If you need handling of ``\\`` escapes and/or expansion of ``${...}`` placeholders this should be
performed manually on the returned values.
:param str path: The absolute path to the properties file.
:param str encoding: The encoding to use (unless running under Python 2 in which case byte strings are always returned).
The default is UTF-8 (with optional Byte Order Mark).
:return dict[str:str]: An ordered dictionary containing the keys and values from the file.
"""
assert os.path.isabs(path), 'Cannot use relative path: "%s"' % path
result = collections.OrderedDict()
with openfile(path,
mode='r',
encoding=None if PY2 else encoding,
errors='strict') as fp:
for line in fp:
line = line.lstrip()
if len(line) == 0 or line.startswith(('#', '!')): continue
line = line.split('=', 1)
if len(line) != 2: continue
result[line[0].strip()] = line[1].strip()
return result
def getmatches(file, regexpr, ignores=None, encoding=None):
"""Look for matches on a regular expression in an input file, return a sequence of the matches.
@param file: The full path to the input file
@param regexpr: The regular expression used to search for matches
@param ignores: A list of regexes which will cause matches to be discarded
@param encoding: Specifies the encoding to be used for opening the file, or None for default.
@return: A list of the match objects
@rtype: list
@raises FileNotFoundException: Raised if the input file does not exist
"""
matches = []
rexp = re.compile(regexpr)
log.debug("Looking for expression \"%s\" in input file %s" %(regexpr, file))
if not os.path.exists(file):
raise FileNotFoundException("unable to find file %s" % (os.path.basename(file)))
else:
with openfile(file, 'r', encoding=encoding) as f:
for l in f:
match = rexp.search(l)
if match is not None:
shouldignore = False
if ignores:
for i in ignores:
if re.search(i, l):
shouldignore = True
break
if shouldignore: continue
log.debug(("Found match for line: %s" % l).rstrip())
matches.append(match)
return matches
def lastgrep(file, expr, ignore=[], include=[], encoding=None, returnMatch=False, flags=0):
"""Search for matches to a regular expression in the last line of an input file, returning true if a match occurs.
:param file: The full path to the input file
:param expr: The regular expression (uncompiled) to search for in the last line of the input file
:return: success (True / False)
:param ignore: A list of regular expressions which remove entries in the input file contents before making the grep
:param include: A list of regular expressions used to select lines from the input file contents to use in the grep
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:rtype: integer
:raises FileNotFoundException: Raised if the input file does not exist
"""
if not pathexists(file):
raise FileNotFoundException("unable to find file \"%s\"" % (file))
else:
with openfile(file, 'r', encoding=encoding) as f:
contents = f.readlines()
contents = trimContents(contents, ignore, exclude=True, flags=flags)
contents = trimContents(contents, include, exclude=False, flags=flags)
logContents("Contents of %s after pre-processing;" % os.path.basename(file), contents)
if len(contents) > 0:
line = contents[len(contents)-1]
regexpr = re.compile(expr, flags=flags)
result = regexpr.search(line)
if result is not None: return result if returnMatch else True
return None if returnMatch else False
def _runTestFile(self, testFile):
globals = {}
locals = {}
displayName = 'PyUnit'
instance = self.getInstanceCount(displayName)
dstdout = os.path.join(self.output, 'pyunit.out')
dstderr = os.path.join(self.output, 'pyunit.err')
if instance: dstdout = "%s.%d" % (dstdout, instance)
if instance: dstderr = "%s.%d" % (dstderr, instance)
arguments = [__file__, testFile]
environ = os.environ.copy()
environ['PYTHONPATH'] = os.pathsep.join(self.getPythonPath() +
sys.path)
process = self.startPython(arguments,
environs=environ,
workingDir=self.output,
state=FOREGROUND,
timeout=DEFAULT_TIMEOUT,
stdout=dstdout,
stderr=dstderr,
displayName=displayName,
ignoreExitStatus=True,
quiet=True)
if process.exitStatus:
self.addOutcome(FAILED,
'Non-zero exit code from %s' %
os.path.basename(testFile),
printReason=False)
else:
self.addOutcome(PASSED)
with openfile(dstdout,
encoding=self.getDefaultFileEncoding(dstdout)) as f:
for l in f:
self.log.info(l.rstrip())
def setup(self, **kwargs):
# Creates the file handle to the logfile and logs initial details of the date,
# platform and test host.
self.logfile = os.path.join(
self.outputDir or kwargs['runner'].output + '/..', self.logfile)
self.fp = flushfile(
openfile(self.logfile, "w", encoding=None if PY2 else 'utf-8'))
self.fp.write('id, title, cycle, startTime, duration, outcome\n')
def getTestClasses(testFile):
globals = {}
# Use globals dictionary for locals as well because we
# want to treat this like it is being run in a global
# (file scope) context)
exec(compile(openfile(testFile).read(), testFile, 'exec'), globals,
globals)
testClasses = []
for k, v in list(globals.items()):
if isinstance(v, type(unittest.TestCase)):
testClasses.append(v)
return (testClasses, globals)
def makeTest(self, input=None, output=None, reference=None, descriptor=None, testclass=None, module=None,
group="", constantsImport=None, basetestImport=None, basetest=None, teststring=None):
"""
Makes a new test on disk.
"""
if input==None: input = DEFAULT_INPUT
if output==None: output = DEFAULT_OUTPUT
if reference==None: reference = DEFAULT_REFERENCE
if descriptor==None: descriptor = DEFAULT_DESCRIPTOR[0]
if testclass==None: testclass = DEFAULT_TESTCLASS
if module==None: module = DEFAULT_MODULE
if constantsImport ==None: constantsImport = "from pysys.constants import *"
if basetestImport == None: basetestImport = "from pysys.basetest import BaseTest"
if basetest == None: basetest = "BaseTest"
log.info("Creating testcase %s ..." % self.testId)
try:
os.makedirs(os.path.join(self.testdir, self.testId))
log.info("Created directory %s" % os.path.join(self.testdir, self.testId))
except OSError:
log.info("Error creating testcase " + os.path.join(self.testdir, self.testId) + " - directory already exists")
return
else:
os.makedirs(os.path.join(self.testdir, self.testId, input))
log.info("Created directory %s " % os.path.join(self.testdir, self.testId, input))
os.makedirs(os.path.join(self.testdir, self.testId, output))
log.info("Created directory %s " % os.path.join(self.testdir, self.testId, output))
os.makedirs(os.path.join(self.testdir, self.testId, reference))
log.info("Created directory %s " % os.path.join(self.testdir, self.testId, reference))
descriptor_fp = openfile(os.path.join(self.testdir, self.testId, descriptor), "w", encoding=None if PY2 else 'utf-8')
descriptor_fp.write(self.DESCRIPTOR_TEMPLATE %(self.type, group, testclass, module))
descriptor_fp.close()
log.info("Created descriptor %s " % os.path.join(self.testdir, self.testId, descriptor))
testclass_fp = openfile(os.path.join(self.testdir, self.testId, "%s.py" % module), "w")
if teststring == None:
testclass_fp.write(self.TEST_TEMPLATE % (constantsImport, basetestImport, testclass, basetest))
else:
testclass_fp.write(teststring)
testclass_fp.close()
log.info("Created test class module %s " % os.path.join(self.testdir, self.testId, "%s.py" % module))
def getmatches(file,
regexpr,
ignores=None,
encoding=None,
flags=0,
mappers=[],
returnFirstOnly=False):
"""Look for matches on a regular expression in an input file, return a sequence of the matches
(or if returnFirstOnly=True, just the first).
:param file: The full path to the input file
:param regexpr: The regular expression used to search for matches
:param mappers: A list of lambdas or generator functions used to pre-process the file's lines before looking for matches.
:param ignores: A list of regexes which will cause matches to be discarded. These are applied *after* any mappers.
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:param returnFirstOnly: If True, stops reading the file as soon as the first match is found and returns it.
:return: A list of the match objects, or the match object or None if returnFirstOnly is True
:rtype: list
:raises FileNotFoundException: Raised if the input file does not exist
"""
matches = []
rexp = re.compile(regexpr, flags=flags)
log.debug("Looking for expression \"%s\" in input file %s" %
(regexpr, file))
if isinstance(ignores, str):
ignores = [
ignores
] # it's easy to pass in a str by mistake and we definitely don't want to be ignoring lines containing any letter from that string!
ignores = [re.compile(i, flags=flags) for i in (ignores or [])]
if not pathexists(file):
raise FileNotFoundException("unable to find file \"%s\"" % (file))
else:
with openfile(file, 'r', encoding=encoding) as f:
for l in applyMappers(f, mappers):
match = rexp.search(l)
if match is not None:
shouldignore = False
for i in ignores:
if i.search(l):
shouldignore = True
break
if shouldignore: continue
log.debug(("Found match for line: %s" % l).rstrip())
if returnFirstOnly is True: return match
matches.append(match)
if returnFirstOnly is True: return None
return matches
def replace(input, output, dict={}, marker='', encoding=None):
"""Read an input file, and write to output tailoring the file to replace set keywords with values.
:deprecated: It is recommended to use `pysys.basetest.BaseTest.copy()` instead of this method.
The replace method reads in the contents of the input file line by line, checks for matches in
each line to the keys in the dictionary dict parameter, and replaces each match with the value
of the dictionary for that key, writing the line to the output file. The marker parameter is a
character which can be used to denoted keywords in the file to be replaced. For instance, with
dict of the form C{{'CATSEAT':'mat', 'DOGSEAT':'hat'}}, marker set to '$', and an input file::
The cat sat on the $CATSEAT$
The dog sat on the $DOGSEAT$
the ouptut file produced would have the contents::
The cat sat on the mat
The dog sat on the hat
:param input: The full path to the input file
:param output: The full path to the output file with the keywords replaced
:param dict: A dictionary of key/value pairs to use in the replacement
:param marker: The character used to mark key words to be replaced (may be the empty string
if no characters are used)
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:raises FileNotFoundException: Raised if the input file does not exist
"""
if not os.path.exists(input):
raise FileNotFoundException("unable to find file %s" % (input))
else:
with openfile(input, 'r', encoding=encoding) as fi, openfile(
output, 'w', encoding=encoding) as fo:
for line in fi.readlines():
for key in list(dict.keys()):
line = line.replace('%s%s%s' % (marker, key, marker),
"%s" % (dict[key]))
fo.write(line)
def loadJSON(path, **kwargs):
"""
Reads JSON from the specified path.
This is a small wrapper around Python's ``json.load()`` function.
:param str path: The absolute path to the JSON file, which must be encoded using UTF-8 (with optional Byte Order Mark).
:param kwargs: Keyword arguments will be passed to ``json.load()``.
:return obj: A dict, list, or other Python object representing the contents of the JSON file.
"""
assert os.path.isabs(path), 'Cannot use relative path: "%s"' % path
with openfile(path, mode='r', encoding='utf-8-sig', errors='strict') as fp:
return json.load(fp, **kwargs)
def orderedgrep(file, exprList, encoding=None, flags=0):
"""Seach for ordered matches to a set of regular expressions in an input file, returning None
on success, and a string indicating the missing expression something is missing.
The ordered grep method will only pass if matches to the set of regular expression in the expression
list occur in the input file in the order they appear in the expression list. Matches to the regular expressions
do not have to be across sequential lines in the input file, only in the correct order. For example, for a file
with contents ::
A is for apple
B is for book
C is for cat
D is for dog
an expression list of ["^A.*$", "^C.*$", "^D.*$"] will return true, whilst an expression list of
["^A.*$", "^C.$", "^B.$"] will return false.
:param file: The full path to the input file
:param exprList: A list of regular expressions (uncompiled) to search for in the input file
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:return: None on success, or on failure the string expression that was not found (with an indicator of its index in the array).
:rtype: string
:raises FileNotFoundException: Raised if the input file does not exist
"""
list = copy.deepcopy(exprList)
list.reverse()
expr, exprIndex = list.pop(), 1
regexpr = re.compile(expr, flags=flags)
if not pathexists(file):
raise FileNotFoundException('unable to find file "%s"' %
(file)) # pragma: no cover
with openfile(file, 'r', encoding=encoding) as f:
for line in f:
if regexpr.search(line) is not None:
if len(list) == 0:
return None # success - found them all
expr, exprIndex = list.pop(), exprIndex + 1
regexpr = re.compile(expr, flags=flags)
return '#%d: %s' % (exprIndex, expr
) # the expression we were trying to match
def filegrep(file, expr, ignores=None, returnMatch=False, encoding=None, flags=0, mappers=[]):
"""Search for matches to a regular expression in an input file, returning true if a match occurs.
:param file: The full path to the input file
:param expr: The regular expression (uncompiled) to search for in the input file
:param ignores: Optional list of regular expression strings to ignore when searching file.
:param returnMatch: return the regex match object instead of a simple boolean
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:param mappers: Mappers to pre-process the file.
:return: success (True / False), unless returnMatch=True in which case it returns the regex match
object (or None if not matched)
:rtype: integer
:raises FileNotFoundException: Raised if the input file does not exist
"""
if not pathexists(file):
raise FileNotFoundException("unable to find file \"%s\"" % (file))
else:
with openfile(file, 'r', encoding=encoding) as f:
if log.isEnabledFor(logging.DEBUG):
contents = f.readlines()
logContents("Contents of %s;" % os.path.basename(file), contents)
else:
contents = f
ignores = [re.compile(i, flags=flags) for i in (ignores or [])]
regexpr = re.compile(expr, flags=flags)
if None in mappers: mappers = [m for m in mappers if m]
for line in contents:
# pre-process
for mapper in mappers:
line = mapper(line)
if line is None: break
if line is None: continue
m = regexpr.search(line)
if m is not None:
if not any([i.search(line) for i in ignores]):
if returnMatch: return m
return True
if returnMatch: return None
return False
def getmatches(file, regexpr, ignores=None, encoding=None, flags=0, mappers=[]):
"""Look for matches on a regular expression in an input file, return a sequence of the matches.
:param file: The full path to the input file
:param regexpr: The regular expression used to search for matches
:param ignores: A list of regexes which will cause matches to be discarded
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:return: A list of the match objects
:rtype: list
:raises FileNotFoundException: Raised if the input file does not exist
"""
matches = []
rexp = re.compile(regexpr, flags=0)
log.debug("Looking for expression \"%s\" in input file %s" %(regexpr, file))
ignores = [re.compile(i, flags=flags) for i in (ignores or [])]
if None in mappers: mappers = [m for m in mappers if m]
if not pathexists(file):
raise FileNotFoundException("unable to find file \"%s\"" % (file))
else:
with openfile(file, 'r', encoding=encoding) as f:
for l in f:
# pre-process
for mapper in mappers:
l = mapper(l)
if l is None: break
if l is None: continue
match = rexp.search(l)
if match is not None:
shouldignore = False
for i in ignores:
if i.search(l):
shouldignore = True
break
if shouldignore: continue
log.debug(("Found match for line: %s" % l).rstrip())
matches.append(match)
return matches
def setup(self, **kwargs):
# Creates the file handle to the logfile and logs initial details of the date,
# platform and test host.
self.logfile = os.path.join(
self.outputDir or kwargs['runner'].output + '/..', self.logfile)
self.fp = flushfile(
openfile(self.logfile, "w", encoding=None if PY2 else 'utf-8'))
self.fp.write('DATE: %s\n' % (time.strftime(
'%Y-%m-%d %H:%M:%S (%Z)', time.localtime(time.time()))))
self.fp.write('PLATFORM: %s\n' % (PLATFORM))
self.fp.write('TEST HOST: %s\n' % (HOSTNAME))
self.fp.write('\n')
for k, v in kwargs['runner'].runDetails.items():
if k in {'startTime', 'hostname'}:
continue # don't duplicate the above
self.fp.write("%-12s%s\n" % (k + ': ', v))
def orderedgrep(file, exprList, encoding=None):
"""Seach for ordered matches to a set of regular expressions in an input file, returning true if the matches occur in the correct order.
The ordered grep method will only return true if matches to the set of regular expression in the expression
list occur in the input file in the order they appear in the expression list. Matches to the regular expressions
do not have to be across sequential lines in the input file, only in the correct order. For example, for a file
with contents ::
A is for apple
B is for book
C is for cat
D is for dog
an expression list of ["^A.*$", "^C.*$", "^D.*$"] will return true, whilst an expression list of
["^A.*$", "^C.$", "^B.$"] will return false.
@param file: The full path to the input file
@param exprList: A list of regular expressions (uncompiled) to search for in the input file
@param encoding: Specifies the encoding to be used for opening the file, or None for default.
@returns: success (True / False)
@rtype: integer
@raises FileNotFoundException: Raised if the input file does not exist
"""
list = copy.deepcopy(exprList)
list.reverse();
expr = list.pop();
if not os.path.exists(file):
raise FileNotFoundException("unable to find file %s" % (os.path.basename(file)))
else:
with openfile(file, 'r', encoding=encoding) as f:
contents = f.readlines()
for i in range(len(contents)):
regexpr = re.compile(expr)
if regexpr.search(r"%s"%contents[i]) is not None:
try:
expr = list.pop();
except Exception:
return None
return expr
def filegrep(file, expr, ignores=None, returnMatch=False, encoding=None):
"""Search for matches to a regular expression in an input file, returning true if a match occurs.
@param file: The full path to the input file
@param expr: The regular expression (uncompiled) to search for in the input file
@param ignores: Optional list of regular expression strings to ignore when searching file.
@param returnMatch: return the regex match object instead of a simple boolean
@param encoding: Specifies the encoding to be used for opening the file, or None for default.
@returns: success (True / False), unless returnMatch=True in which case it returns the regex match
object (or None if not matched)
@rtype: integer
@raises FileNotFoundException: Raised if the input file does not exist
"""
if not os.path.exists(file):
raise FileNotFoundException("unable to find file %s" % (os.path.basename(file)))
else:
f = openfile(file, 'r', encoding=encoding)
try:
if log.isEnabledFor(logging.DEBUG):
contents = f.readlines()
logContents("Contents of %s;" % os.path.basename(file), contents)
else:
contents = f
ignores = [re.compile(i) for i in (ignores or [])]
regexpr = re.compile(expr)
for line in contents:
m = regexpr.search(line)
if m is not None:
if not any([i.search(line) for i in ignores]):
if returnMatch: return m
return True
if returnMatch: return None
return False
finally:
f.close()
def cleanup(self, **kwargs):
if self.archiveAtEndOfRun:
for _, id, outputDir in sorted(
self.queuedInstructions
): # sort by hash of testId so make order deterministic
self._archiveTestOutputDir(id, outputDir)
if self.skippedTests:
# if we hit a limit, at least record the names of the tests we missed
mkdir(self.destDir)
with openfile(self.destDir + os.sep + 'skipped_artifacts.txt',
'w',
encoding='utf-8') as f:
f.write('\n'.join(
os.path.normpath(t) for t in self.skippedTests))
(log.info if self.archivesCreated else log.debug)(
'%s created %d test output archive artifacts in: %s',
self.__class__.__name__, self.archivesCreated, self.destDir)
if self.archivesCreated:
self.runner.publishArtifact(self.destDir, 'TestOutputArchiveDir')
def _runTestFile(self, testFile):
globals = {}
locals = {}
command = sys.executable
displayName = 'PyUnit'
instance = self.getInstanceCount(displayName)
dstdout = os.path.join(self.output, 'pyunit.out')
dstderr = os.path.join(self.output, 'pyunit.err')
if instance: dstdout = "%s.%d" % (dstdout, instance)
if instance: dstderr = "%s.%d" % (dstderr, instance)
arguments = [__file__, testFile]
filter = ThreadFilter()
self.log.addFilter(filter)
environ = os.environ.copy()
environ['PYTHONPATH'] = os.pathsep.join(self.getPythonPath() +
sys.path)
process = self.startProcess(command,
arguments,
environ,
self.output,
FOREGROUND,
DEFAULT_TIMEOUT,
dstdout,
dstderr,
displayName,
ignoreExitStatus=True)
self.log.removeFilter(filter)
if process.exitStatus:
self.addOutcome(FAILED,
'Non-zero exit code from %s' %
os.path.basename(testFile),
printReason=False)
else:
self.addOutcome(PASSED)
with openfile(dstdout,
encoding=self.getDefaultFileEncoding(dstdout)) as f:
for l in f:
self.log.info(l.rstrip())
def writeXML(self):
'''Write a test descriptor template to file.'''
fp = openfile(self.file, 'w')
fp.writelines(DESCRIPTOR_TEMPLATE %
(self.type, self.group, self.testclass, self.module))
fp.close
def filediff(file1,
file2,
ignore=[],
sort=True,
replacementList=[],
include=[],
unifiedDiffOutput=None,
encoding=None):
"""Perform a file comparison between two (preprocessed) input files, returning true if the files are equivalent.
The method reads in the files and loads the contents of each as a list of strings. The two files are
said to be equal if the two lists are equal. The method allows for preprocessing of the string lists
to trim down their contents prior to the comparison being performed. Preprocessing is either to remove
entries from the lists which match any entry in a set of regular expressions, include only lines which
match any entry in a set of regular expressions, replace certain keywords in the string values of each list
with a set value (e.g. to replace time stamps etc), or to sort the lists before the comparison (e.g. where
determinism may not exist). Verbose logging of the method occurs at DEBUG level showing the contents of the
processed lists prior to the comparison being performed.
@param file1: The full path to the first file to use in the comparison
@param file2: The full path to the second file to use in the comparison, typically a reference file
@param ignore: A list of regular expressions which remove entries in the input file contents before making the comparison
@param sort: Boolean to sort the input file contents before making the comparison
@param replacementList: A list of tuples (key, value) where matches to key are replaced with value in the input file contents before making the comparison
@param include: A list of regular expressions used to select lines from the input file contents to use in the comparison
@param unifiedDiffOutput: If specified, indicates the full path of a file to which unified diff output will be written,
if the diff fails.
@param encoding: Specifies the encoding to be used for opening the file, or None for default.
@return: success (True / False)
@rtype: boolean
@raises FileNotFoundException: Raised if either of the files do not exist
"""
for file in file1, file2:
if not os.path.exists(file):
raise FileNotFoundException("unable to find file %s" %
(os.path.basename(file)))
else:
list1 = []
list2 = []
with openfile(file1, 'r', encoding=encoding) as f:
for i in f:
list1.append(i.strip())
with openfile(file2, 'r', encoding=encoding) as f:
for i in f:
list2.append(i.strip())
list1 = trimContents(list1, ignore, exclude=True)
list2 = trimContents(list2, ignore, exclude=True)
list1 = trimContents(list1, include, exclude=False)
list2 = trimContents(list2, include, exclude=False)
list1 = replace(list1, replacementList)
list2 = replace(list2, replacementList)
if sort:
list1.sort()
list2.sort()
logContents(
"Contents of %s after pre-processing;" % os.path.basename(file1),
list1)
logContents(
"Contents of %s after pre-processing;" % os.path.basename(file2),
list2)
if not list1 and not list2:
# maybe this should be an exception... it's probably not what was intended
log.warn(
'File comparison pre-processing has filtered out all lines from the files to be diffed, please check if this is intended: %s, %s',
os.path.basename(file1), os.path.basename(file2))
if list1 != list2:
log.debug("Unified diff between pre-processed input files;")
l1 = []
l2 = []
for i in list1:
l1.append("%s\n" % i)
for i in list2:
l2.append("%s\n" % i)
# nb: have to switch 1 and 2 around to get the right diff for a typical output,ref file pair
diff = ''.join(
difflib.unified_diff(
l2,
l1,
fromfile='%s (%d lines)' %
(os.path.basename(file2), len(l2)),
tofile='%s (%d lines)' %
(os.path.basename(file1), len(l1)),
))
if unifiedDiffOutput:
with openfile(unifiedDiffOutput, 'w', encoding=encoding) as f:
f.write(diff)
for line in diff.split('\n'):
log.debug(" %s", line)
if list1 == list2: return True
return False
def filediff(file1,
file2,
ignore=[],
sort=True,
replacementList=[],
include=[],
unifiedDiffOutput=None,
encoding=None,
stripWhitespace=True,
flags=0):
"""Perform a file comparison between two (preprocessed) input files, returning true if the files are equivalent.
The method reads in the files and loads the contents of each as a list of strings. The two files are
said to be equal if the two lists are equal. The method allows for preprocessing of the string lists
to trim down their contents prior to the comparison being performed. Preprocessing is either to remove
entries from the lists which match any entry in a set of regular expressions, include only lines which
match any entry in a set of regular expressions, replace certain keywords in the string values of each list
with a set value (e.g. to replace time stamps etc), or to sort the lists before the comparison (e.g. where
determinism may not exist). Verbose logging of the method occurs at DEBUG level showing the contents of the
processed lists prior to the comparison being performed.
:param file1: The full path to the first file to use in the comparison
:param file2: The full path to the second file to use in the comparison, typically a reference file
:param ignore: A list of regular expressions which remove entries in the input file contents before making the comparison
:param sort: Boolean to sort the input file contents before making the comparison
:param replacementList: A list of tuples (key, value) where matches to key are replaced with value in the input file contents before making the comparison
:param stripWhitespace: If True, every line has leading and trailing whitespace stripped before comparison,
which means indentation differences and whether the file ends with a blank line do not affect the outcome.
If False, only newline characters are stripped.
:param include: A list of regular expressions used to select lines from the input file contents to use in the comparison
:param unifiedDiffOutput: If specified, indicates the full path of a file to which unified diff output will be written,
if the diff fails.
:param encoding: Specifies the encoding to be used for opening the file, or None for default.
:return: success (True / False)
:rtype: boolean
:raises FileNotFoundException: Raised if either of the files do not exist
"""
for file in file1, file2:
if not pathexists(file):
raise FileNotFoundException("unable to find file \"%s\"" % file)
else:
stripchars = None if stripWhitespace else '\r\n' # None means all whitespace
with openfile(file1, 'r', encoding=encoding) as f:
list1 = [i.strip(stripchars) for i in f]
with openfile(file2, 'r', encoding=encoding) as f:
list2 = [i.strip(stripchars) for i in f]
list1 = trimContents(list1, ignore, exclude=True, flags=flags)
list2 = trimContents(list2, ignore, exclude=True, flags=flags)
list1 = trimContents(list1, include, exclude=False, flags=flags)
list2 = trimContents(list2, include, exclude=False, flags=flags)
list1 = replace(list1, replacementList, flags=flags)
list2 = replace(list2, replacementList, flags=flags)
if sort:
list1.sort()
list2.sort()
logContents(
"Contents of %s after pre-processing;" % os.path.basename(file1),
list1)
logContents(
"Contents of %s after pre-processing;" % os.path.basename(file2),
list2)
if not list1 and not list2:
# maybe this should be an exception... it's probably not what was intended
log.warning(
'File comparison pre-processing has filtered out all lines from the files to be diffed, please check if this is intended: %s, %s',
os.path.basename(file1), os.path.basename(file2))
if list1 != list2:
log.debug("Unified diff between pre-processed input files;")
l1 = []
l2 = []
for i in list1:
l1.append("%s\n" % i)
for i in list2:
l2.append("%s\n" % i)
file1display = file1
file2display = file2
try:
commonprefix = os.path.commonprefix(
[file1display, file2display])
except ValueError:
pass
else:
if commonprefix:
# heuristic to give a longer prefix than just basename (to distinguish reference+output files with same basename)
file1display = file1display[len(commonprefix):]
file2display = file2display[len(commonprefix):]
# nb: have to switch 1 and 2 around to get the right diff for a typical output,ref file pair
diff = ''.join(
difflib.unified_diff(
l2,
l1,
fromfile='%s (%d lines)' % (file2display, len(l2)),
tofile='%s (%d lines)' % (file1display, len(l1)),
))
if unifiedDiffOutput:
with openfile(unifiedDiffOutput, 'w', encoding=encoding) as f:
f.write(diff)
for line in diff.split('\n'):
log.debug(" %s", line)
if list1 == list2: return True
return False
def execute(self):
p = self.startProcess(command=sys.executable,
arguments=[self.input + '/spinner.py'],
stdout="%s/test.out" % self.output,
stderr="%s/test.err" % self.output,
state=BACKGROUND)
childtest = runPySys(self,
'pysys', [
'run', '-o', self.output + '/myoutdir', '-X',
'pidToMonitor=%d' % p.pid
],
state=BACKGROUND,
workingDir=self.input + '/nestedtests')
# should be a no-op, just check it doesn't throw
ProcessMonitorTextFileHandler.setDefaults(
ProcessMonitorTextFileHandler.DEFAULT_COLUMNS)
pm = self.startProcessMonitor(p,
interval=0.1,
file='monitor-default.tsv')
pm2 = self.startProcessMonitor(p,
interval=0.1,
file=self.output +
'/monitor-numproc.tsv',
numProcessors='10')
# test all supported stats, and also use of stream rather than filename
filehandle = openfile(self.output + '/monitor-all.csv',
'w',
encoding='utf-8')
self.addCleanupFunction(lambda: filehandle.close())
pm_all = self.startProcessMonitor(
p,
interval=0.1,
handlers=[
ProcessMonitorTextFileHandler(
file=filehandle,
columns=[
ProcessMonitorKey.DATE_TIME,
ProcessMonitorKey.SAMPLE,
ProcessMonitorKey.CPU_CORE_UTILIZATION,
ProcessMonitorKey.CPU_TOTAL_UTILIZATION,
ProcessMonitorKey.MEMORY_RESIDENT_KB,
ProcessMonitorKey.MEMORY_VIRTUAL_KB,
],
delimiter=',')
])
pidmonitor = self.startProcessMonitor(p.pid,
interval=0.1,
file=self.output +
'/monitor-pid.tsv')
assert pm.running(), 'monitor is still running'
self.waitForGrep('monitor-default.tsv',
expr='.',
condition='>=5',
ignores=['#.*'])
self.waitForGrep('monitor-numproc.tsv',
expr='.',
condition='>=5',
ignores=['#.*'])
self.waitForGrep('monitor-pid.tsv',
expr='.',
condition='>=5',
ignores=['#.*'])
self.waitForGrep('monitor-all.csv',
expr='.',
condition='>=5',
ignores=['#.*'])
assert pm.running(), 'monitor is still running'
assert pidmonitor.running(), 'pid monitor is still running'
self.stopProcessMonitor(pidmonitor)
self.waitProcess(childtest, timeout=60)
self.assertTrue(childtest.exitStatus == 0,
assertMessage='nested pysys test passed')
self.stopProcessMonitor(pm)
pm.stop() # should silently do nothing
self.stopProcessMonitor(pm) # should silently do nothing
assert p.exitStatus == None, 'spinner process terminated unexpectedly'
p.stop()
self.wait(
1
) # keep process monitor running after it to check it doesn't cause an error
self.stopProcessMonitor(pm2)
