def run_replay(testfile, inputfiles, reporterrors, settings):
for filename in [testfile] + inputfiles:
if not os.path.isfile(filename):
print('Error:', filename, 'does not exist!')
return
t = Test(testfile, settings)
if settings['verbose']:
print 'Running test ' + testfile
t.reset()
t.replay(inputfiles)
try:
result = t.run(reporterrors)
if result == True:
return True
else:
return False
except ToolInputError as e:
print 'Result: ' + next(x for x in e.value.split('\n') if 'error' in x)
except UnusedToolsError as e:
print 'Result: UnusedToolsError'
except ToolCrashedError as e:
print 'Result: ToolCrashedError'
except MemoryExceededError as e:
print 'Result: Memory Exceeded'
except TimeExceededError as e:
print 'Result: Time Exceeded'
def parse_old_test_description(line, autotest_dir, debug=0):
fields = re.split(r' *\t *', line.strip())
assert len(fields) >= 3
(label, time_limit, command) = fields[0:3]
time_limit = int(time_limit)
if len(fields) < 4:
compare_command = 'diff'
else:
compare_command = fields[3]
if len(fields) <= 5:
prediff_filter = ''
description = command
elif len(fields) == 6:
prediff_filter = ''
description = fields[5]
else:
prediff_filter = fields[5]
description = '\t'.join(fields[6:])
if debug:
print('from_test_txt_line: (label="%s", time_limit="%s", command="%s", compare_command="%s", prediff_filter="%s", description="%s")' % (label, time_limit, command, compare_command, prediff_filter, description))
kwargs = {'max_cpu' : time_limit, 'debug' : debug}
if description:
kwargs['description'] = description
expected_stdout_file = 'sampleout' + label
if os.path.exists(os.path.join(autotest_dir, expected_stdout_file)):
kwargs['expected_stdout_file'] = expected_stdout_file
expected_stderr_file = 'sampleerr' + label
if os.path.exists(os.path.join(autotest_dir, expected_stderr_file)):
kwargs['expected_stderr_file'] = expected_stderr_file
stdin_file = 'test' + label
if os.path.exists(os.path.join(autotest_dir, stdin_file)):
kwargs['stdin_file'] = stdin_file
command = command.strip()
m = re.search(r'(^|[\s;\&\|\.])\.\/([\w\-\.]+)', command)
if m:
program = m.group(2)
else:
program = re.sub(r'\s.*', '', command.strip())
if debug:
print('from_test_txt_line: program="%s"' % program)
if re.search(r'diff.*-\w*i', compare_command):
kwargs['ignore_case'] = True
if re.search(r'diff.*-\w*w', compare_command):
kwargs['ignore_white_space'] = True
if re.search(r'diff.*-\w*B', compare_command):
kwargs['ignore_blank_lines'] = True
kwargs['prediff_filter'] = prediff_filter
if debug:
print('kwargs:', kwargs)
command = make_list(command)
return Test(autotest_dir, label, program, command, **kwargs)
def parse_test_line(autotest_dir, filename, line_number, line, debug, global_variables):
line_variables = {}
line_words = []
current_variable = None
# crude hack to add back quotes
m = re.match(r'^(\w+)=(`.*`)\s*$', line)
if m:
global_variables[m.group(1)] = m.group(2)
return
try:
for token in shlex.split(line, posix=True, comments=True):
# https://bugs.python.org/issue21331
token = codecs.getdecoder("unicode_escape")(token.encode('latin1',errors='backslashreplace'))[0]
m = re.match(r'^(\w+)=(.*)', token, flags=re.S)
if m:
line_variables[m.group(1)] = m.group(2)
current_variable = m.group(1)
elif current_variable:
line_variables[current_variable] = make_list(line_variables[current_variable])
line_variables[current_variable].append(token)
else:
line_words.append(token)
except ValueError as e:
print("{}:{} bad test description ({}): {}".format(filename, line_number, e, line))
return
if len(line_words) == 1:
line_variables['label'] = line_words[0]
elif len(line_words) == 2:
(line_variables['label'], line_variables['command']) = line_words
elif len(line_words) == 3:
(line_variables['label'], line_variables['max_cpu'], line_variables['command']) = line_words
elif len(line_words) > 3:
print("%s:%d bad test description line: %s" % (filename, line_number, line), file=sys.stderr)
return
# line is just setting variables
if 'label' not in line_variables:
global_variables.update(line_variables)
return
test_variables = global_variables.copy()
test_variables.update(line_variables)
label = test_variables.pop('label')
try:
command = test_variables.pop('command')
except KeyError:
# if command not specified infer from command
command = re.sub('[_0-9]*$', '', label)
command = make_list(command)
if 'program' in test_variables:
program = test_variables.pop('program')
else:
# if program not specified infer from command
m = re.search(r'(^|[\s;\&\|\.])\.\/([\w\-\.]+)', command[0])
if m:
program = m.group(2)
else:
program = re.sub(r'\s.*', '', command[0].strip())
if 'files' in test_variables:
test_variables['files'] = make_list(test_variables['files'])
else:
# if files not specified infer from program
if '.' in program:
test_variables['files'] = [program]
elif program:
test_variables['files'] = [program + '.c']
else:
test_variables['files'] = []
if 'files' in test_variables:
test_variables['files'] = make_list(test_variables['files'])
if 'optional_files' in test_variables:
test_variables['optional_files'] = make_list(test_variables['optional_files'])
expected_files = []
for parameter in test_variables:
if not test_variables[parameter]:
continue
m = re.match(r'^expected_(file.*)_name', parameter)
if m:
expected_files.append(m.group(1))
test_variables['expected_files'] = expected_files
for stream in ['stdout', 'stderr'] + expected_files:
if 'expected_' + stream in test_variables:
continue
if 'expected_' + stream + '_file' in test_variables:
continue
for potential_filename in [label + '.' + 'expected_' + stream, 'sample' + stream[3:6] + label]:
if os.path.exists(os.path.join(autotest_dir, potential_filename)):
if debug > 1:
print('test_variables[%s]=%s' %('expected_' + stream + '_file', potential_filename), file=sys.stderr)
test_variables['expected_' + stream + '_file'] = potential_filename
break
for stdin_file in [label + '.stdin', 'test' + label]:
if os.path.exists(os.path.join(autotest_dir, stdin_file)):
test_variables['stdin_file'] = stdin_file
if debug > 1:
print("%s:%d" % (filename, line_number), label, program, command, test_variables, file=sys.stderr)
return Test(autotest_dir, label, program, command, **test_variables)
parser.add_argument('I', type=int, help='the number of tests that is performed')
parser.add_argument('N', type=int, help='the length of the randomly generated strings')
parser.add_argument('-r' '--report', action='store_true', dest='reporterrors', help='capture error messages in node values')
parser.add_argument('-t' '--toolpath', dest='toolpath', help='the tool path')
options = parser.parse_args()
termlength = options.N
testfile = options.T
numtests = options.I
reporterrors = not options.reporterrors
settings = dict()
settings['toolpath'] = options.toolpath
settings['verbose'] = options.verbose
t = Test(testfile, settings)
grammarfiles = t.options['grammar']
constraintsfiles = t.options['constraints']
constraints = yaml.load(''.join([open(f).read() for f in constraintsfiles]))
grammar = ''.join([open(f).read() for f in grammarfiles])
generator = randgen.RandGen(grammar, constraints)
if not t.validTermLengths(generator, termlength):
print 'invalid term length: ' + str(termlength)
else:
for i in range(numtests):
if settings['verbose']:
print 'Running test ' + testfile
t.reset()