Beispiel #1
0
def run_replay(testfile, inputfiles, reporterrors, settings):
    for filename in [testfile] + inputfiles:
        if not os.path.isfile(filename):
            print('Error:', filename, 'does not exist!')
            return

    t = Test(testfile, settings)

    if settings['verbose']:
        print 'Running test ' + testfile
    t.reset()
    t.replay(inputfiles)

    try:
        result = t.run(reporterrors)
        if result == True:
            return True
        else:
            return False
    except ToolInputError as e:
        print 'Result: ' + next(x for x in e.value.split('\n') if 'error' in x)
    except UnusedToolsError as e:
        print 'Result: UnusedToolsError'
    except ToolCrashedError as e:
        print 'Result: ToolCrashedError'
    except MemoryExceededError as e:
        print 'Result: Memory Exceeded'
    except TimeExceededError as e:
        print 'Result: Time Exceeded'
def parse_old_test_description(line, autotest_dir, debug=0):
	fields = re.split(r' *\t *', line.strip())
	assert len(fields) >= 3
	(label, time_limit, command) = fields[0:3]
	time_limit = int(time_limit)
	if len(fields) < 4:
		compare_command = 'diff'
	else:
		compare_command = fields[3]
	if len(fields) <= 5:
		prediff_filter = ''
		description = command
	elif len(fields) == 6:
		prediff_filter = ''
		description = fields[5]
	else:
		prediff_filter = fields[5]
		description = '\t'.join(fields[6:])
	if debug:
		print('from_test_txt_line: (label="%s", time_limit="%s", command="%s", compare_command="%s", prediff_filter="%s", description="%s")' % (label, time_limit, command, compare_command, prediff_filter, description))
	kwargs = {'max_cpu' : time_limit, 'debug' : debug}
	if description:
		kwargs['description'] = description
	expected_stdout_file = 'sampleout' + label
	if os.path.exists(os.path.join(autotest_dir, expected_stdout_file)):
		kwargs['expected_stdout_file'] = expected_stdout_file
	expected_stderr_file = 'sampleerr' + label
	if os.path.exists(os.path.join(autotest_dir, expected_stderr_file)):
		kwargs['expected_stderr_file'] = expected_stderr_file
	stdin_file = 'test' + label
	if os.path.exists(os.path.join(autotest_dir, stdin_file)):
		kwargs['stdin_file'] = stdin_file
	command = command.strip()
	m = re.search(r'(^|[\s;\&\|\.])\.\/([\w\-\.]+)', command)
	if m:
		program = m.group(2)
	else:
		program = re.sub(r'\s.*', '', command.strip())
	if debug:
		print('from_test_txt_line: program="%s"' % program)
	if re.search(r'diff.*-\w*i', compare_command):
		kwargs['ignore_case'] = True
	if re.search(r'diff.*-\w*w', compare_command):
		kwargs['ignore_white_space'] = True
	if re.search(r'diff.*-\w*B', compare_command):
		kwargs['ignore_blank_lines'] = True
	kwargs['prediff_filter'] = prediff_filter
	if debug:
		print('kwargs:', kwargs)
	command = make_list(command)
	return Test(autotest_dir, label, program, command, **kwargs)
def parse_test_line(autotest_dir, filename, line_number, line, debug, global_variables):
	line_variables = {}
	line_words = []
	current_variable = None
	# crude hack to add back quotes
	m = re.match(r'^(\w+)=(`.*`)\s*$', line)
	if m:
		global_variables[m.group(1)] = m.group(2)
		return
	try:
		for token in shlex.split(line, posix=True, comments=True):
			# https://bugs.python.org/issue21331
			token = codecs.getdecoder("unicode_escape")(token.encode('latin1',errors='backslashreplace'))[0]
			m = re.match(r'^(\w+)=(.*)', token, flags=re.S)
			if m:
				line_variables[m.group(1)] = m.group(2)
				current_variable = m.group(1)
			elif current_variable:
				line_variables[current_variable] = make_list(line_variables[current_variable])
				line_variables[current_variable].append(token)
			else:
				line_words.append(token)
	except ValueError as e:
		print("{}:{} bad test description ({}): {}".format(filename, line_number, e, line))
		return
	if len(line_words) == 1:
		line_variables['label'] = line_words[0]
	elif len(line_words) == 2:
		(line_variables['label'], line_variables['command']) = line_words
	elif len(line_words) == 3:
		(line_variables['label'], line_variables['max_cpu'], line_variables['command']) = line_words
	elif len(line_words) > 3:
		print("%s:%d bad test description line: %s" % (filename, line_number, line), file=sys.stderr)
		return

	# line is just setting variables
	if 'label' not in line_variables:
		global_variables.update(line_variables)
		return

	test_variables = global_variables.copy()
	test_variables.update(line_variables)
	label = test_variables.pop('label')
	try:
		command = test_variables.pop('command')
	except KeyError:
		# if command not specified infer from command
		command = re.sub('[_0-9]*$', '', label)

	command = make_list(command)
	if 'program' in test_variables:
		program = test_variables.pop('program')
	else:
		# if program not specified infer from command
		m = re.search(r'(^|[\s;\&\|\.])\.\/([\w\-\.]+)', command[0])
		if m:
			program = m.group(2)
		else:
			program = re.sub(r'\s.*', '', command[0].strip())

	if 'files' in test_variables:
		test_variables['files'] = make_list(test_variables['files'])
	else:
		# if files not specified infer from program
		if '.' in program:
			test_variables['files'] = [program]
		elif program:
			test_variables['files'] = [program + '.c']
		else:
			test_variables['files'] = []

	if 'files' in test_variables:
		test_variables['files'] = make_list(test_variables['files'])
	if 'optional_files' in test_variables:
		test_variables['optional_files'] = make_list(test_variables['optional_files'])
	expected_files = []
	for parameter in test_variables:
		if not test_variables[parameter]:
			continue
		m = re.match(r'^expected_(file.*)_name', parameter)
		if m:
			expected_files.append(m.group(1))
	test_variables['expected_files'] = expected_files
	for stream in ['stdout', 'stderr'] + expected_files:
		if 'expected_' + stream in test_variables:
			continue
		if 'expected_' + stream + '_file' in test_variables:
			continue
		for potential_filename in [label + '.' + 'expected_' + stream, 'sample' + stream[3:6] + label]:
			if os.path.exists(os.path.join(autotest_dir, potential_filename)):
				if debug > 1:
					print('test_variables[%s]=%s' %('expected_' + stream + '_file',	 potential_filename), file=sys.stderr)
				test_variables['expected_' + stream + '_file'] =  potential_filename
				break
	for stdin_file in [label + '.stdin',  'test' + label]:
		if os.path.exists(os.path.join(autotest_dir, stdin_file)):
			test_variables['stdin_file'] = stdin_file
	if debug > 1:
		 print("%s:%d" % (filename, line_number), label, program, command, test_variables, file=sys.stderr)
	return Test(autotest_dir, label, program, command, **test_variables)
Beispiel #4
0
parser.add_argument('I', type=int, help='the number of tests that is performed')
parser.add_argument('N', type=int, help='the length of the randomly generated strings')
parser.add_argument('-r' '--report', action='store_true', dest='reporterrors', help='capture error messages in node values')
parser.add_argument('-t' '--toolpath', dest='toolpath', help='the tool path')
options = parser.parse_args()

termlength = options.N
testfile = options.T
numtests = options.I
reporterrors = not options.reporterrors

settings = dict()
settings['toolpath'] = options.toolpath
settings['verbose'] = options.verbose

t = Test(testfile, settings)

grammarfiles = t.options['grammar']
constraintsfiles = t.options['constraints']

constraints = yaml.load(''.join([open(f).read() for f in constraintsfiles]))
grammar = ''.join([open(f).read() for f in grammarfiles])
generator = randgen.RandGen(grammar, constraints)

if not t.validTermLengths(generator, termlength):
    print 'invalid term length: ' + str(termlength)
else:
    for i in range(numtests):
        if settings['verbose']:
            print 'Running test ' + testfile
        t.reset()