def main():
options = parse_args()
suppress = v8_suppressions.get_suppression(options.skip_suppressions)
# Static bailout based on test case content or metadata.
kwargs = {}
if PYTHON3:
kwargs['encoding'] = 'utf-8'
with open(options.testcase, 'r', **kwargs) as f:
content = f.read()
content_bailout(get_meta_data(content), suppress.ignore_by_metadata)
content_bailout(content, suppress.ignore_by_content)
# Prepare the baseline, default and a secondary configuration to compare to.
# The baseline (turbofan) takes precedence as many of the secondary configs
# are based on the turbofan config with additional parameters.
execution_configs = [
ExecutionConfig(options, 'first'),
ExecutionConfig(options, 'default'),
ExecutionConfig(options, 'second'),
]
# First, run some fixed smoke tests in all configs to ensure nothing
# is fundamentally wrong, in order to prevent bug flooding.
if not options.skip_sanity_checks:
run_comparisons(
suppress,
execution_configs,
test_case=SANITY_CHECKS,
timeout=SANITY_CHECK_TIMEOUT_SEC,
verbose=False,
# Don't accept crashes during sanity checks. A crash would hint at
# a flag that might be incompatible or a broken test file.
ignore_crashes=False,
# Special source key for sanity checks so that clusterfuzz dedupes all
# cases on this in case it's hit.
source_key='sanity check failed',
)
# Second, run all configs against the fuzz test case.
run_comparisons(
suppress,
execution_configs,
test_case=options.testcase,
timeout=TEST_TIMEOUT_SEC,
)
# TODO(machenbach): Figure out if we could also return a bug in case
# there's no difference, but one of the line suppressions has matched -
# and without the match there would be a difference.
print('# V8 correctness - pass')
return RETURN_PASS
def main():
options = parse_args()
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch,
options.first_config,
options.second_arch,
options.second_config,
)
# Static bailout based on test case content or metadata.
with open(options.testcase) as f:
content = f.read()
if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
return RETURN_FAIL
if content_bailout(content, suppress.ignore_by_content):
return RETURN_FAIL
# Set up runtime arguments.
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
def run_d8(d8, config_flags):
preamble = PREAMBLE[:]
if options.first_arch != options.second_arch:
preamble.append(ARCH_MOCKS)
args = [d8] + config_flags + preamble + [options.testcase]
print " ".join(args)
if d8.endswith('.py'):
# Wrap with python in tests.
args = [sys.executable] + args
return v8_commands.Execute(
args,
cwd=os.path.dirname(options.testcase),
timeout=TIMEOUT,
)
first_config_output = run_d8(options.first_d8, first_config_flags)
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
second_config_output = run_d8(options.second_d8, second_config_flags)
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
difference, source = suppress.diff(first_config_output.stdout,
second_config_output.stdout)
if source:
source_key = hashlib.sha1(
source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
else:
source = ORIGINAL_SOURCE_DEFAULT
source_key = ORIGINAL_SOURCE_DEFAULT
if difference:
# Only bail out due to suppressed output if there was a difference. If a
# suppression doesn't show up anymore in the statistics, we might want to
# remove it.
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
# The first three entries will be parsed by clusterfuzz. Format changes
# will require changes on the clusterfuzz side.
first_config_label = '%s,%s' % (options.first_arch,
options.first_config)
second_config_label = '%s,%s' % (options.second_arch,
options.second_config)
print(
FAILURE_TEMPLATE % dict(
configs='%s:%s' % (first_config_label, second_config_label),
source_key=source_key,
suppression='', # We can't tie bugs to differences.
first_config_label=first_config_label,
second_config_label=second_config_label,
first_config_flags=' '.join(first_config_flags),
second_config_flags=' '.join(second_config_flags),
first_config_output=first_config_output.stdout.decode(
'utf-8', 'replace'),
second_config_output=second_config_output.stdout.decode(
'utf-8', 'replace'),
source=source,
difference=difference.decode('utf-8', 'replace'),
)).encode('utf-8', 'replace')
return RETURN_FAIL
# TODO (machenbach): Figure out if we could also return a bug in case there's id:3989
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print '# V8 correctness - pass'
return RETURN_PASS
def testDiff(self):
# TODO(machenbach): Mock out suppression configuration.
suppress = v8_suppressions.get_suppression('x64', 'ignition', 'x64',
'ignition_turbo')
one = ''
two = ''
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
# Ignore line before caret, caret position and error message.
one = """
undefined
weird stuff
^
somefile.js: TypeError: undefined is not a function
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: baz is not a function
undefined
"""
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
self.assertEquals(diff, suppress.diff(one, two))
def main():
options = parse_args()
rng = random.Random(options.random_seed)
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch, options.first_config,
options.second_arch, options.second_config,
)
# Static bailout based on test case content or metadata.
with open(options.testcase) as f:
content = f.read()
if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
return RETURN_FAIL
if content_bailout(content, suppress.ignore_by_content):
return RETURN_FAIL
# Set up runtime arguments.
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
# Add additional flags to second config based on experiment percentages.
for p, flag in ADDITIONAL_FLAGS:
if rng.random() < p:
second_config_flags.append(flag)
def run_d8(d8, config_flags):
preamble = PREAMBLE[:]
if options.first_arch != options.second_arch:
preamble.append(ARCH_MOCKS)
args = [d8] + config_flags + preamble + [options.testcase]
print " ".join(args)
if d8.endswith('.py'):
# Wrap with python in tests.
args = [sys.executable] + args
return v8_commands.Execute(
args,
cwd=os.path.dirname(os.path.abspath(options.testcase)),
timeout=TIMEOUT,
)
first_config_output = run_d8(options.first_d8, first_config_flags)
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
second_config_output = run_d8(options.second_d8, second_config_flags)
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
difference, source = suppress.diff(
first_config_output.stdout, second_config_output.stdout)
if source:
source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
else:
source = ORIGINAL_SOURCE_DEFAULT
source_key = ORIGINAL_SOURCE_DEFAULT
if difference:
# Only bail out due to suppressed output if there was a difference. If a
# suppression doesn't show up anymore in the statistics, we might want to
# remove it.
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
# The first three entries will be parsed by clusterfuzz. Format changes
# will require changes on the clusterfuzz side.
first_config_label = '%s,%s' % (options.first_arch, options.first_config)
second_config_label = '%s,%s' % (options.second_arch, options.second_config)
print (FAILURE_TEMPLATE % dict(
configs='%s:%s' % (first_config_label, second_config_label),
source_key=source_key,
suppression='', # We can't tie bugs to differences.
first_config_label=first_config_label,
second_config_label=second_config_label,
first_config_flags=' '.join(first_config_flags),
second_config_flags=' '.join(second_config_flags),
first_config_output=
first_config_output.stdout.decode('utf-8', 'replace'),
second_config_output=
second_config_output.stdout.decode('utf-8', 'replace'),
source=source,
difference=difference.decode('utf-8', 'replace'),
)).encode('utf-8', 'replace')
return RETURN_FAIL
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print '# V8 correctness - pass'
return RETURN_PASS
def main():
options = parse_args()
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first.arch,
options.first.config,
options.second.arch,
options.second.config,
)
# Static bailout based on test case content or metadata.
with open(options.testcase) as f:
content = f.read()
if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
return RETURN_FAIL
if content_bailout(content, suppress.ignore_by_content):
return RETURN_FAIL
first_cmd = v8_commands.Command(options, 'first', options.first.d8,
options.first.flags)
second_cmd = v8_commands.Command(options, 'second', options.second.d8,
options.second.flags)
# Sanity checks. Run both configurations with the sanity-checks file only and
# bail out early if different.
if not options.skip_sanity_checks:
first_config_output = first_cmd.run(SANITY_CHECKS)
second_config_output = second_cmd.run(SANITY_CHECKS)
difference, _ = suppress.diff(first_config_output.stdout,
second_config_output.stdout)
if difference:
# Special source key for sanity checks so that clusterfuzz dedupes all
# cases on this in case it's hit.
source_key = 'sanity check failed'
print_difference(options, source_key, first_cmd, second_cmd,
first_config_output, second_config_output,
difference)
return RETURN_FAIL
first_config_output = first_cmd.run(options.testcase, verbose=True)
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
second_config_output = second_cmd.run(options.testcase, verbose=True)
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
difference, source = suppress.diff(first_config_output.stdout,
second_config_output.stdout)
if source:
source_key = hashlib.sha1(
source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
else:
source_key = ORIGINAL_SOURCE_DEFAULT
if difference:
# Only bail out due to suppressed output if there was a difference. If a
# suppression doesn't show up anymore in the statistics, we might want to
# remove it.
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
print_difference(options, source_key, first_cmd, second_cmd,
first_config_output, second_config_output, difference,
source)
return RETURN_FAIL
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print('# V8 correctness - pass')
return RETURN_PASS
def diff_fun(one, two, skip=False):
suppress = v8_suppressions.get_suppression(
'x64', 'ignition', 'x64', 'ignition_turbo', skip)
return suppress.diff_lines(one.splitlines(), two.splitlines())
def testDiff(self):
# TODO(machenbach): Mock out suppression configuration.
suppress = v8_suppressions.get_suppression(
'x64', 'ignition', 'x64', 'ignition_turbo')
one = ''
two = ''
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
# Ignore line before caret, caret position and error message.
one = """
undefined
weird stuff
^
somefile.js: TypeError: undefined is not a function
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: baz is not a function
undefined
"""
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
self.assertEquals(diff, suppress.diff(one, two))
def main():
options = parse_args()
rng = random.Random(options.random_seed)
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch,
options.first_config,
options.second_arch,
options.second_config,
)
# Static bailout based on test case content or metadata.
with open(options.testcase) as f:
content = f.read()
if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
return RETURN_FAIL
if content_bailout(content, suppress.ignore_by_content):
return RETURN_FAIL
# Set up runtime arguments.
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
# Add additional flags to second config based on experiment percentages.
for p, flag in ADDITIONAL_FLAGS:
if rng.random() < p:
second_config_flags.append(flag)
def run_d8(d8, config_flags, config_label=None, testcase=options.testcase):
preamble = PREAMBLE[:]
if options.first_arch != options.second_arch:
preamble.append(ARCH_MOCKS)
args = [d8] + config_flags + preamble + [testcase]
if config_label:
print('# Command line for %s comparison:' % config_label)
print(' '.join(args))
if d8.endswith('.py'):
# Wrap with python in tests.
args = [sys.executable] + args
return v8_commands.Execute(
args,
cwd=os.path.dirname(os.path.abspath(testcase)),
timeout=TIMEOUT,
)
# Sanity checks. Run both configurations with the sanity-checks file only and
# bail out early if different.
if not options.skip_sanity_checks:
first_config_output = run_d8(options.first_d8,
first_config_flags,
testcase=SANITY_CHECKS)
second_config_output = run_d8(options.second_d8,
second_config_flags,
testcase=SANITY_CHECKS)
difference, _ = suppress.diff(first_config_output.stdout,
second_config_output.stdout)
if difference:
# Special source key for sanity checks so that clusterfuzz dedupes all
# cases on this in case it's hit.
source_key = 'sanity check failed'
print_difference(options, source_key, first_config_flags,
second_config_flags, first_config_output,
second_config_output, difference)
return RETURN_FAIL
first_config_output = run_d8(options.first_d8, first_config_flags, 'first')
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
second_config_output = run_d8(options.second_d8, second_config_flags,
'second')
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
difference, source = suppress.diff(first_config_output.stdout,
second_config_output.stdout)
if source:
source_key = hashlib.sha1(
source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
else:
source_key = ORIGINAL_SOURCE_DEFAULT
if difference:
# Only bail out due to suppressed output if there was a difference. If a
# suppression doesn't show up anymore in the statistics, we might want to
# remove it.
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
print_difference(options, source_key, first_config_flags,
second_config_flags, first_config_output,
second_config_output, difference, source)
return RETURN_FAIL
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print('# V8 correctness - pass')
return RETURN_PASS
def main():
options = parse_args()
rng = random.Random(options.random_seed)
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch, options.first_config,
options.second_arch, options.second_config,
)
# Static bailout based on test case content or metadata.
with open(options.testcase) as f:
content = f.read()
if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
return RETURN_FAIL
if content_bailout(content, suppress.ignore_by_content):
return RETURN_FAIL
# Set up runtime arguments.
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
# Add additional flags to second config based on experiment percentages.
for p, flag in ADDITIONAL_FLAGS:
if rng.random() < p:
second_config_flags.append(flag)
def run_d8(d8, config_flags, config_label=None, testcase=options.testcase):
preamble = PREAMBLE[:]
if options.first_arch != options.second_arch:
preamble.append(ARCH_MOCKS)
args = [d8] + config_flags + preamble + [testcase]
if config_label:
print('# Command line for %s comparison:' % config_label)
print(' '.join(args))
if d8.endswith('.py'):
# Wrap with python in tests.
args = [sys.executable] + args
return v8_commands.Execute(
args,
cwd=os.path.dirname(os.path.abspath(testcase)),
timeout=TIMEOUT,
)
# Sanity checks. Run both configurations with the sanity-checks file only and
# bail out early if different.
if not options.skip_sanity_checks:
first_config_output = run_d8(
options.first_d8, first_config_flags, testcase=SANITY_CHECKS)
second_config_output = run_d8(
options.second_d8, second_config_flags, testcase=SANITY_CHECKS)
difference, _ = suppress.diff(
first_config_output.stdout, second_config_output.stdout)
if difference:
# Special source key for sanity checks so that clusterfuzz dedupes all
# cases on this in case it's hit.
source_key = 'sanity check failed'
print_difference(
options, source_key, first_config_flags, second_config_flags,
first_config_output, second_config_output, difference)
return RETURN_FAIL
first_config_output = run_d8(options.first_d8, first_config_flags, 'first')
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
second_config_output = run_d8(
options.second_d8, second_config_flags, 'second')
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
difference, source = suppress.diff(
first_config_output.stdout, second_config_output.stdout)
if source:
source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
else:
source_key = ORIGINAL_SOURCE_DEFAULT
if difference:
# Only bail out due to suppressed output if there was a difference. If a
# suppression doesn't show up anymore in the statistics, we might want to
# remove it.
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
print_difference(
options, source_key, first_config_flags, second_config_flags,
first_config_output, second_config_output, difference, source)
return RETURN_FAIL
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print('# V8 correctness - pass')
return RETURN_PASS
def testDiff(self):
# TODO(machenbach): Mock out suppression configuration.
suppress = v8_suppressions.get_suppression(
'x64', 'fullcode', 'x64', 'default')
one = ''
two = ''
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
# Ignore line before caret, caret position, stack trace char numbers
# error message and validator output.
one = """
undefined
weird stuff
^
Validation of asm.js module failed: foo bar
somefile.js: TypeError: undefined is not a function
stack line :15: foo
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: baz is not a function
stack line :2: foo
Validation of asm.js module failed: baz
undefined
"""
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
self.assertEquals(diff, suppress.diff(one, two))
def testDiff(self):
# TODO(machenbach): Mock out suppression configuration.
suppress = v8_suppressions.get_suppression('x64', 'fullcode', 'x64',
'default')
one = ''
two = ''
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
# Ignore line before caret, caret position, stack trace char numbers
# error message and validator output.
one = """
undefined
weird stuff
^
Validation of asm.js module failed: foo bar
somefile.js: TypeError: undefined is not a function
stack line :15: foo
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: baz is not a function
stack line :2: foo
Validation of asm.js module failed: baz
undefined
"""
diff = None, None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line', None
self.assertEquals(diff, suppress.diff(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
self.assertEquals(diff, suppress.diff(one, two))
def main():
options = parse_args()
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first.arch,
options.first.config,
options.second.arch,
options.second.config,
options.skip_suppressions,
)
# Static bailout based on test case content or metadata.
kwargs = {}
if PYTHON3:
kwargs['encoding'] = 'utf-8'
with open(options.testcase, 'r', **kwargs) as f:
content = f.read()
if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
return RETURN_FAIL
if content_bailout(content, suppress.ignore_by_content):
return RETURN_FAIL
first_cmd = v8_commands.Command(options, 'first', options.first.d8,
options.first.flags)
second_cmd = v8_commands.Command(options, 'second', options.second.d8,
options.second.flags)
# Sanity checks. Run both configurations with the sanity-checks file only and
# bail out early if different.
if not options.skip_sanity_checks:
first_config_output = first_cmd.run(SANITY_CHECKS)
second_config_output = second_cmd.run(SANITY_CHECKS)
difference, _ = suppress.diff(first_config_output,
second_config_output)
if difference:
# Special source key for sanity checks so that clusterfuzz dedupes all
# cases on this in case it's hit.
source_key = 'sanity check failed'
print_difference(options, source_key, first_cmd, second_cmd,
first_config_output, second_config_output,
difference)
return RETURN_FAIL
first_config_output = first_cmd.run(options.testcase, verbose=True)
# Early bailout if first run was a timeout.
if timeout_bailout(first_config_output, 1):
return RETURN_PASS
second_config_output = second_cmd.run(options.testcase, verbose=True)
# Bailout if second run was a timeout.
if timeout_bailout(second_config_output, 2):
return RETURN_PASS
difference, source = suppress.diff(first_config_output,
second_config_output)
if difference:
# Only bail out due to suppressed output if there was a difference. If a
# suppression doesn't show up anymore in the statistics, we might want to
# remove it.
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
source_key = cluster_failures(source)
print_difference(options, source_key, first_cmd, second_cmd,
first_config_output, second_config_output, difference,
source)
return RETURN_FAIL
# Show if a crash has happened in one of the runs and no difference was
# detected.
if first_config_output.HasCrashed():
print('# V8 correctness - C-R-A-S-H 1')
elif second_config_output.HasCrashed():
print('# V8 correctness - C-R-A-S-H 2')
else:
# TODO(machenbach): Figure out if we could also return a bug in case
# there's no difference, but one of the line suppressions has matched -
# and without the match there would be a difference.
print('# V8 correctness - pass')
return RETURN_PASS
def main():
options = parse_args()
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch,
options.first_config,
options.second_arch,
options.second_config,
)
if test_pattern_bailout(options.testcase, suppress.ignore):
return RETURN_FAIL
# Get metadata.
with open(options.meta_data_path) as f:
metadata = json.load(f)
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
def run_d8(d8, config_flags):
args = [d8] + config_flags + PREAMBLE + [options.testcase]
if d8.endswith('.py'):
# Wrap with python in tests.
args = [sys.executable] + args
return v8_commands.Execute(
args,
cwd=os.path.dirname(options.testcase),
timeout=TIMEOUT,
)
first_config_output = run_d8(options.first_d8, first_config_flags)
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
second_config_output = run_d8(options.second_d8, second_config_flags)
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
difference = suppress.diff(first_config_output.stdout,
second_config_output.stdout)
if difference:
# The first three entries will be parsed by clusterfuzz. Format changes
# will require changes on the clusterfuzz side.
first_config_label = '%s,%s' % (options.first_arch,
options.first_config)
second_config_label = '%s,%s' % (options.second_arch,
options.second_config)
hsh = lambda x: hashlib.sha1(x).hexdigest()[:8]
print FAILURE_TEMPLATE % dict(
configs='%s:%s' % (first_config_label, second_config_label),
sources=','.join(map(hsh, metadata['sources'])),
suppression='', # We can't tie bugs to differences.
first_config_label=first_config_label,
second_config_label=second_config_label,
first_config_flags=' '.join(first_config_flags),
second_config_flags=' '.join(second_config_flags),
first_config_output=first_config_output.stdout,
second_config_output=second_config_output.stdout,
difference=difference,
)
return RETURN_FAIL
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print '# V8 correctness - pass'
return RETURN_PASS
def diff_fun(one, two, skip=False):
suppress = v8_suppressions.get_suppression(skip)
return suppress.diff_lines(one.splitlines(), two.splitlines())
def main():
options = parse_args()
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch, options.first_config,
options.second_arch, options.second_config,
)
if test_pattern_bailout(options.testcase, suppress.ignore):
return RETURN_FAIL
# Get metadata.
with open(options.meta_data_path) as f:
metadata = json.load(f)
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
def run_d8(d8, config_flags):
args = [d8] + config_flags + PREAMBLE + [options.testcase]
if d8.endswith('.py'):
# Wrap with python in tests.
args = [sys.executable] + args
return v8_commands.Execute(
args,
cwd=os.path.dirname(options.testcase),
timeout=TIMEOUT,
)
first_config_output = run_d8(options.first_d8, first_config_flags)
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAIL
second_config_output = run_d8(options.second_d8, second_config_flags)
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAIL
difference = suppress.diff(
first_config_output.stdout, second_config_output.stdout)
if difference:
# The first three entries will be parsed by clusterfuzz. Format changes
# will require changes on the clusterfuzz side.
first_config_label = '%s,%s' % (options.first_arch, options.first_config)
second_config_label = '%s,%s' % (options.second_arch, options.second_config)
hsh = lambda x: hashlib.sha1(x).hexdigest()[:8]
print FAILURE_TEMPLATE % dict(
configs='%s:%s' % (first_config_label, second_config_label),
sources=','.join(map(hsh, metadata['sources'])),
suppression='', # We can't tie bugs to differences.
first_config_label=first_config_label,
second_config_label=second_config_label,
first_config_flags=' '.join(first_config_flags),
second_config_flags=' '.join(second_config_flags),
first_config_output=first_config_output.stdout,
second_config_output=second_config_output.stdout,
difference=difference,
)
return RETURN_FAIL
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print '# V8 correctness - pass'
return RETURN_PASS