def main(): options = parse_args() suppress = v8_suppressions.get_suppression(options.skip_suppressions) # Static bailout based on test case content or metadata. kwargs = {} if PYTHON3: kwargs['encoding'] = 'utf-8' with open(options.testcase, 'r', **kwargs) as f: content = f.read() content_bailout(get_meta_data(content), suppress.ignore_by_metadata) content_bailout(content, suppress.ignore_by_content) # Prepare the baseline, default and a secondary configuration to compare to. # The baseline (turbofan) takes precedence as many of the secondary configs # are based on the turbofan config with additional parameters. execution_configs = [ ExecutionConfig(options, 'first'), ExecutionConfig(options, 'default'), ExecutionConfig(options, 'second'), ] # First, run some fixed smoke tests in all configs to ensure nothing # is fundamentally wrong, in order to prevent bug flooding. if not options.skip_sanity_checks: run_comparisons( suppress, execution_configs, test_case=SANITY_CHECKS, timeout=SANITY_CHECK_TIMEOUT_SEC, verbose=False, # Don't accept crashes during sanity checks. A crash would hint at # a flag that might be incompatible or a broken test file. ignore_crashes=False, # Special source key for sanity checks so that clusterfuzz dedupes all # cases on this in case it's hit. source_key='sanity check failed', ) # Second, run all configs against the fuzz test case. run_comparisons( suppress, execution_configs, test_case=options.testcase, timeout=TEST_TIMEOUT_SEC, ) # TODO(machenbach): Figure out if we could also return a bug in case # there's no difference, but one of the line suppressions has matched - # and without the match there would be a difference. print('# V8 correctness - pass') return RETURN_PASS
def main(): options = parse_args() # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first_arch, options.first_config, options.second_arch, options.second_config, ) # Static bailout based on test case content or metadata. with open(options.testcase) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL if content_bailout(content, suppress.ignore_by_content): return RETURN_FAIL # Set up runtime arguments. common_flags = FLAGS + ['--random-seed', str(options.random_seed)] first_config_flags = common_flags + CONFIGS[options.first_config] second_config_flags = common_flags + CONFIGS[options.second_config] def run_d8(d8, config_flags): preamble = PREAMBLE[:] if options.first_arch != options.second_arch: preamble.append(ARCH_MOCKS) args = [d8] + config_flags + preamble + [options.testcase] print " ".join(args) if d8.endswith('.py'): # Wrap with python in tests. args = [sys.executable] + args return v8_commands.Execute( args, cwd=os.path.dirname(options.testcase), timeout=TIMEOUT, ) first_config_output = run_d8(options.first_d8, first_config_flags) # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS second_config_output = run_d8(options.second_d8, second_config_flags) # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS difference, source = suppress.diff(first_config_output.stdout, second_config_output.stdout) if source: source_key = hashlib.sha1( source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] else: source = ORIGINAL_SOURCE_DEFAULT source_key = ORIGINAL_SOURCE_DEFAULT if difference: # Only bail out due to suppressed output if there was a difference. If a # suppression doesn't show up anymore in the statistics, we might want to # remove it. if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL # The first three entries will be parsed by clusterfuzz. Format changes # will require changes on the clusterfuzz side. first_config_label = '%s,%s' % (options.first_arch, options.first_config) second_config_label = '%s,%s' % (options.second_arch, options.second_config) print( FAILURE_TEMPLATE % dict( configs='%s:%s' % (first_config_label, second_config_label), source_key=source_key, suppression='', # We can't tie bugs to differences. first_config_label=first_config_label, second_config_label=second_config_label, first_config_flags=' '.join(first_config_flags), second_config_flags=' '.join(second_config_flags), first_config_output=first_config_output.stdout.decode( 'utf-8', 'replace'), second_config_output=second_config_output.stdout.decode( 'utf-8', 'replace'), source=source, difference=difference.decode('utf-8', 'replace'), )).encode('utf-8', 'replace') return RETURN_FAIL # TODO (machenbach): Figure out if we could also return a bug in case there's id:3989 # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print '# V8 correctness - pass' return RETURN_PASS
def testDiff(self): # TODO(machenbach): Mock out suppression configuration. suppress = v8_suppressions.get_suppression('x64', 'ignition', 'x64', 'ignition_turbo') one = '' two = '' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = 'a \n b\nc();' two = 'a \n b\nc();' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) # Ignore line before caret, caret position and error message. one = """ undefined weird stuff ^ somefile.js: TypeError: undefined is not a function undefined """ two = """ undefined other weird stuff ^ somefile.js: TypeError: baz is not a function undefined """ diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal Extra line """ two = """ Still equal """ diff = '- Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal """ two = """ Still equal Extra line """ diff = '+ Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ undefined somefile.js: TypeError: undefined is not a constructor """ two = """ undefined otherfile.js: TypeError: undefined is not a constructor """ diff = """- somefile.js: TypeError: undefined is not a constructor + otherfile.js: TypeError: undefined is not a constructor""", None self.assertEquals(diff, suppress.diff(one, two))
def main(): options = parse_args() rng = random.Random(options.random_seed) # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first_arch, options.first_config, options.second_arch, options.second_config, ) # Static bailout based on test case content or metadata. with open(options.testcase) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL if content_bailout(content, suppress.ignore_by_content): return RETURN_FAIL # Set up runtime arguments. common_flags = FLAGS + ['--random-seed', str(options.random_seed)] first_config_flags = common_flags + CONFIGS[options.first_config] second_config_flags = common_flags + CONFIGS[options.second_config] # Add additional flags to second config based on experiment percentages. for p, flag in ADDITIONAL_FLAGS: if rng.random() < p: second_config_flags.append(flag) def run_d8(d8, config_flags): preamble = PREAMBLE[:] if options.first_arch != options.second_arch: preamble.append(ARCH_MOCKS) args = [d8] + config_flags + preamble + [options.testcase] print " ".join(args) if d8.endswith('.py'): # Wrap with python in tests. args = [sys.executable] + args return v8_commands.Execute( args, cwd=os.path.dirname(os.path.abspath(options.testcase)), timeout=TIMEOUT, ) first_config_output = run_d8(options.first_d8, first_config_flags) # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS second_config_output = run_d8(options.second_d8, second_config_flags) # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS difference, source = suppress.diff( first_config_output.stdout, second_config_output.stdout) if source: source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] else: source = ORIGINAL_SOURCE_DEFAULT source_key = ORIGINAL_SOURCE_DEFAULT if difference: # Only bail out due to suppressed output if there was a difference. If a # suppression doesn't show up anymore in the statistics, we might want to # remove it. if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL # The first three entries will be parsed by clusterfuzz. Format changes # will require changes on the clusterfuzz side. first_config_label = '%s,%s' % (options.first_arch, options.first_config) second_config_label = '%s,%s' % (options.second_arch, options.second_config) print (FAILURE_TEMPLATE % dict( configs='%s:%s' % (first_config_label, second_config_label), source_key=source_key, suppression='', # We can't tie bugs to differences. first_config_label=first_config_label, second_config_label=second_config_label, first_config_flags=' '.join(first_config_flags), second_config_flags=' '.join(second_config_flags), first_config_output= first_config_output.stdout.decode('utf-8', 'replace'), second_config_output= second_config_output.stdout.decode('utf-8', 'replace'), source=source, difference=difference.decode('utf-8', 'replace'), )).encode('utf-8', 'replace') return RETURN_FAIL # TODO(machenbach): Figure out if we could also return a bug in case there's # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print '# V8 correctness - pass' return RETURN_PASS
def main(): options = parse_args() # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first.arch, options.first.config, options.second.arch, options.second.config, ) # Static bailout based on test case content or metadata. with open(options.testcase) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL if content_bailout(content, suppress.ignore_by_content): return RETURN_FAIL first_cmd = v8_commands.Command(options, 'first', options.first.d8, options.first.flags) second_cmd = v8_commands.Command(options, 'second', options.second.d8, options.second.flags) # Sanity checks. Run both configurations with the sanity-checks file only and # bail out early if different. if not options.skip_sanity_checks: first_config_output = first_cmd.run(SANITY_CHECKS) second_config_output = second_cmd.run(SANITY_CHECKS) difference, _ = suppress.diff(first_config_output.stdout, second_config_output.stdout) if difference: # Special source key for sanity checks so that clusterfuzz dedupes all # cases on this in case it's hit. source_key = 'sanity check failed' print_difference(options, source_key, first_cmd, second_cmd, first_config_output, second_config_output, difference) return RETURN_FAIL first_config_output = first_cmd.run(options.testcase, verbose=True) # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS second_config_output = second_cmd.run(options.testcase, verbose=True) # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS difference, source = suppress.diff(first_config_output.stdout, second_config_output.stdout) if source: source_key = hashlib.sha1( source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] else: source_key = ORIGINAL_SOURCE_DEFAULT if difference: # Only bail out due to suppressed output if there was a difference. If a # suppression doesn't show up anymore in the statistics, we might want to # remove it. if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL print_difference(options, source_key, first_cmd, second_cmd, first_config_output, second_config_output, difference, source) return RETURN_FAIL # TODO(machenbach): Figure out if we could also return a bug in case there's # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print('# V8 correctness - pass') return RETURN_PASS
def diff_fun(one, two, skip=False): suppress = v8_suppressions.get_suppression( 'x64', 'ignition', 'x64', 'ignition_turbo', skip) return suppress.diff_lines(one.splitlines(), two.splitlines())
def testDiff(self): # TODO(machenbach): Mock out suppression configuration. suppress = v8_suppressions.get_suppression( 'x64', 'ignition', 'x64', 'ignition_turbo') one = '' two = '' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = 'a \n b\nc();' two = 'a \n b\nc();' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) # Ignore line before caret, caret position and error message. one = """ undefined weird stuff ^ somefile.js: TypeError: undefined is not a function undefined """ two = """ undefined other weird stuff ^ somefile.js: TypeError: baz is not a function undefined """ diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal Extra line """ two = """ Still equal """ diff = '- Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal """ two = """ Still equal Extra line """ diff = '+ Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ undefined somefile.js: TypeError: undefined is not a constructor """ two = """ undefined otherfile.js: TypeError: undefined is not a constructor """ diff = """- somefile.js: TypeError: undefined is not a constructor + otherfile.js: TypeError: undefined is not a constructor""", None self.assertEquals(diff, suppress.diff(one, two))
def main(): options = parse_args() rng = random.Random(options.random_seed) # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first_arch, options.first_config, options.second_arch, options.second_config, ) # Static bailout based on test case content or metadata. with open(options.testcase) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL if content_bailout(content, suppress.ignore_by_content): return RETURN_FAIL # Set up runtime arguments. common_flags = FLAGS + ['--random-seed', str(options.random_seed)] first_config_flags = common_flags + CONFIGS[options.first_config] second_config_flags = common_flags + CONFIGS[options.second_config] # Add additional flags to second config based on experiment percentages. for p, flag in ADDITIONAL_FLAGS: if rng.random() < p: second_config_flags.append(flag) def run_d8(d8, config_flags, config_label=None, testcase=options.testcase): preamble = PREAMBLE[:] if options.first_arch != options.second_arch: preamble.append(ARCH_MOCKS) args = [d8] + config_flags + preamble + [testcase] if config_label: print('# Command line for %s comparison:' % config_label) print(' '.join(args)) if d8.endswith('.py'): # Wrap with python in tests. args = [sys.executable] + args return v8_commands.Execute( args, cwd=os.path.dirname(os.path.abspath(testcase)), timeout=TIMEOUT, ) # Sanity checks. Run both configurations with the sanity-checks file only and # bail out early if different. if not options.skip_sanity_checks: first_config_output = run_d8(options.first_d8, first_config_flags, testcase=SANITY_CHECKS) second_config_output = run_d8(options.second_d8, second_config_flags, testcase=SANITY_CHECKS) difference, _ = suppress.diff(first_config_output.stdout, second_config_output.stdout) if difference: # Special source key for sanity checks so that clusterfuzz dedupes all # cases on this in case it's hit. source_key = 'sanity check failed' print_difference(options, source_key, first_config_flags, second_config_flags, first_config_output, second_config_output, difference) return RETURN_FAIL first_config_output = run_d8(options.first_d8, first_config_flags, 'first') # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS second_config_output = run_d8(options.second_d8, second_config_flags, 'second') # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS difference, source = suppress.diff(first_config_output.stdout, second_config_output.stdout) if source: source_key = hashlib.sha1( source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] else: source_key = ORIGINAL_SOURCE_DEFAULT if difference: # Only bail out due to suppressed output if there was a difference. If a # suppression doesn't show up anymore in the statistics, we might want to # remove it. if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL print_difference(options, source_key, first_config_flags, second_config_flags, first_config_output, second_config_output, difference, source) return RETURN_FAIL # TODO(machenbach): Figure out if we could also return a bug in case there's # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print('# V8 correctness - pass') return RETURN_PASS
def main(): options = parse_args() rng = random.Random(options.random_seed) # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first_arch, options.first_config, options.second_arch, options.second_config, ) # Static bailout based on test case content or metadata. with open(options.testcase) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL if content_bailout(content, suppress.ignore_by_content): return RETURN_FAIL # Set up runtime arguments. common_flags = FLAGS + ['--random-seed', str(options.random_seed)] first_config_flags = common_flags + CONFIGS[options.first_config] second_config_flags = common_flags + CONFIGS[options.second_config] # Add additional flags to second config based on experiment percentages. for p, flag in ADDITIONAL_FLAGS: if rng.random() < p: second_config_flags.append(flag) def run_d8(d8, config_flags, config_label=None, testcase=options.testcase): preamble = PREAMBLE[:] if options.first_arch != options.second_arch: preamble.append(ARCH_MOCKS) args = [d8] + config_flags + preamble + [testcase] if config_label: print('# Command line for %s comparison:' % config_label) print(' '.join(args)) if d8.endswith('.py'): # Wrap with python in tests. args = [sys.executable] + args return v8_commands.Execute( args, cwd=os.path.dirname(os.path.abspath(testcase)), timeout=TIMEOUT, ) # Sanity checks. Run both configurations with the sanity-checks file only and # bail out early if different. if not options.skip_sanity_checks: first_config_output = run_d8( options.first_d8, first_config_flags, testcase=SANITY_CHECKS) second_config_output = run_d8( options.second_d8, second_config_flags, testcase=SANITY_CHECKS) difference, _ = suppress.diff( first_config_output.stdout, second_config_output.stdout) if difference: # Special source key for sanity checks so that clusterfuzz dedupes all # cases on this in case it's hit. source_key = 'sanity check failed' print_difference( options, source_key, first_config_flags, second_config_flags, first_config_output, second_config_output, difference) return RETURN_FAIL first_config_output = run_d8(options.first_d8, first_config_flags, 'first') # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS second_config_output = run_d8( options.second_d8, second_config_flags, 'second') # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS difference, source = suppress.diff( first_config_output.stdout, second_config_output.stdout) if source: source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] else: source_key = ORIGINAL_SOURCE_DEFAULT if difference: # Only bail out due to suppressed output if there was a difference. If a # suppression doesn't show up anymore in the statistics, we might want to # remove it. if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL print_difference( options, source_key, first_config_flags, second_config_flags, first_config_output, second_config_output, difference, source) return RETURN_FAIL # TODO(machenbach): Figure out if we could also return a bug in case there's # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print('# V8 correctness - pass') return RETURN_PASS
def testDiff(self): # TODO(machenbach): Mock out suppression configuration. suppress = v8_suppressions.get_suppression( 'x64', 'fullcode', 'x64', 'default') one = '' two = '' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = 'a \n b\nc();' two = 'a \n b\nc();' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) # Ignore line before caret, caret position, stack trace char numbers # error message and validator output. one = """ undefined weird stuff ^ Validation of asm.js module failed: foo bar somefile.js: TypeError: undefined is not a function stack line :15: foo undefined """ two = """ undefined other weird stuff ^ somefile.js: TypeError: baz is not a function stack line :2: foo Validation of asm.js module failed: baz undefined """ diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal Extra line """ two = """ Still equal """ diff = '- Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal """ two = """ Still equal Extra line """ diff = '+ Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ undefined somefile.js: TypeError: undefined is not a constructor """ two = """ undefined otherfile.js: TypeError: undefined is not a constructor """ diff = """- somefile.js: TypeError: undefined is not a constructor + otherfile.js: TypeError: undefined is not a constructor""", None self.assertEquals(diff, suppress.diff(one, two))
def testDiff(self): # TODO(machenbach): Mock out suppression configuration. suppress = v8_suppressions.get_suppression('x64', 'fullcode', 'x64', 'default') one = '' two = '' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = 'a \n b\nc();' two = 'a \n b\nc();' diff = None, None self.assertEquals(diff, suppress.diff(one, two)) # Ignore line before caret, caret position, stack trace char numbers # error message and validator output. one = """ undefined weird stuff ^ Validation of asm.js module failed: foo bar somefile.js: TypeError: undefined is not a function stack line :15: foo undefined """ two = """ undefined other weird stuff ^ somefile.js: TypeError: baz is not a function stack line :2: foo Validation of asm.js module failed: baz undefined """ diff = None, None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal Extra line """ two = """ Still equal """ diff = '- Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ Still equal """ two = """ Still equal Extra line """ diff = '+ Extra line', None self.assertEquals(diff, suppress.diff(one, two)) one = """ undefined somefile.js: TypeError: undefined is not a constructor """ two = """ undefined otherfile.js: TypeError: undefined is not a constructor """ diff = """- somefile.js: TypeError: undefined is not a constructor + otherfile.js: TypeError: undefined is not a constructor""", None self.assertEquals(diff, suppress.diff(one, two))
def main(): options = parse_args() # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first.arch, options.first.config, options.second.arch, options.second.config, options.skip_suppressions, ) # Static bailout based on test case content or metadata. kwargs = {} if PYTHON3: kwargs['encoding'] = 'utf-8' with open(options.testcase, 'r', **kwargs) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL if content_bailout(content, suppress.ignore_by_content): return RETURN_FAIL first_cmd = v8_commands.Command(options, 'first', options.first.d8, options.first.flags) second_cmd = v8_commands.Command(options, 'second', options.second.d8, options.second.flags) # Sanity checks. Run both configurations with the sanity-checks file only and # bail out early if different. if not options.skip_sanity_checks: first_config_output = first_cmd.run(SANITY_CHECKS) second_config_output = second_cmd.run(SANITY_CHECKS) difference, _ = suppress.diff(first_config_output, second_config_output) if difference: # Special source key for sanity checks so that clusterfuzz dedupes all # cases on this in case it's hit. source_key = 'sanity check failed' print_difference(options, source_key, first_cmd, second_cmd, first_config_output, second_config_output, difference) return RETURN_FAIL first_config_output = first_cmd.run(options.testcase, verbose=True) # Early bailout if first run was a timeout. if timeout_bailout(first_config_output, 1): return RETURN_PASS second_config_output = second_cmd.run(options.testcase, verbose=True) # Bailout if second run was a timeout. if timeout_bailout(second_config_output, 2): return RETURN_PASS difference, source = suppress.diff(first_config_output, second_config_output) if difference: # Only bail out due to suppressed output if there was a difference. If a # suppression doesn't show up anymore in the statistics, we might want to # remove it. if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL source_key = cluster_failures(source) print_difference(options, source_key, first_cmd, second_cmd, first_config_output, second_config_output, difference, source) return RETURN_FAIL # Show if a crash has happened in one of the runs and no difference was # detected. if first_config_output.HasCrashed(): print('# V8 correctness - C-R-A-S-H 1') elif second_config_output.HasCrashed(): print('# V8 correctness - C-R-A-S-H 2') else: # TODO(machenbach): Figure out if we could also return a bug in case # there's no difference, but one of the line suppressions has matched - # and without the match there would be a difference. print('# V8 correctness - pass') return RETURN_PASS
def main(): options = parse_args() # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first_arch, options.first_config, options.second_arch, options.second_config, ) if test_pattern_bailout(options.testcase, suppress.ignore): return RETURN_FAIL # Get metadata. with open(options.meta_data_path) as f: metadata = json.load(f) common_flags = FLAGS + ['--random-seed', str(options.random_seed)] first_config_flags = common_flags + CONFIGS[options.first_config] second_config_flags = common_flags + CONFIGS[options.second_config] def run_d8(d8, config_flags): args = [d8] + config_flags + PREAMBLE + [options.testcase] if d8.endswith('.py'): # Wrap with python in tests. args = [sys.executable] + args return v8_commands.Execute( args, cwd=os.path.dirname(options.testcase), timeout=TIMEOUT, ) first_config_output = run_d8(options.first_d8, first_config_flags) # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL second_config_output = run_d8(options.second_d8, second_config_flags) # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL difference = suppress.diff(first_config_output.stdout, second_config_output.stdout) if difference: # The first three entries will be parsed by clusterfuzz. Format changes # will require changes on the clusterfuzz side. first_config_label = '%s,%s' % (options.first_arch, options.first_config) second_config_label = '%s,%s' % (options.second_arch, options.second_config) hsh = lambda x: hashlib.sha1(x).hexdigest()[:8] print FAILURE_TEMPLATE % dict( configs='%s:%s' % (first_config_label, second_config_label), sources=','.join(map(hsh, metadata['sources'])), suppression='', # We can't tie bugs to differences. first_config_label=first_config_label, second_config_label=second_config_label, first_config_flags=' '.join(first_config_flags), second_config_flags=' '.join(second_config_flags), first_config_output=first_config_output.stdout, second_config_output=second_config_output.stdout, difference=difference, ) return RETURN_FAIL # TODO(machenbach): Figure out if we could also return a bug in case there's # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print '# V8 correctness - pass' return RETURN_PASS
def diff_fun(one, two, skip=False): suppress = v8_suppressions.get_suppression(skip) return suppress.diff_lines(one.splitlines(), two.splitlines())
def main(): options = parse_args() # Suppressions are architecture and configuration specific. suppress = v8_suppressions.get_suppression( options.first_arch, options.first_config, options.second_arch, options.second_config, ) if test_pattern_bailout(options.testcase, suppress.ignore): return RETURN_FAIL # Get metadata. with open(options.meta_data_path) as f: metadata = json.load(f) common_flags = FLAGS + ['--random-seed', str(options.random_seed)] first_config_flags = common_flags + CONFIGS[options.first_config] second_config_flags = common_flags + CONFIGS[options.second_config] def run_d8(d8, config_flags): args = [d8] + config_flags + PREAMBLE + [options.testcase] if d8.endswith('.py'): # Wrap with python in tests. args = [sys.executable] + args return v8_commands.Execute( args, cwd=os.path.dirname(options.testcase), timeout=TIMEOUT, ) first_config_output = run_d8(options.first_d8, first_config_flags) # Early bailout based on first run's output. if pass_bailout(first_config_output, 1): return RETURN_PASS if fail_bailout(first_config_output, suppress.ignore_by_output1): return RETURN_FAIL second_config_output = run_d8(options.second_d8, second_config_flags) # Bailout based on second run's output. if pass_bailout(second_config_output, 2): return RETURN_PASS if fail_bailout(second_config_output, suppress.ignore_by_output2): return RETURN_FAIL difference = suppress.diff( first_config_output.stdout, second_config_output.stdout) if difference: # The first three entries will be parsed by clusterfuzz. Format changes # will require changes on the clusterfuzz side. first_config_label = '%s,%s' % (options.first_arch, options.first_config) second_config_label = '%s,%s' % (options.second_arch, options.second_config) hsh = lambda x: hashlib.sha1(x).hexdigest()[:8] print FAILURE_TEMPLATE % dict( configs='%s:%s' % (first_config_label, second_config_label), sources=','.join(map(hsh, metadata['sources'])), suppression='', # We can't tie bugs to differences. first_config_label=first_config_label, second_config_label=second_config_label, first_config_flags=' '.join(first_config_flags), second_config_flags=' '.join(second_config_flags), first_config_output=first_config_output.stdout, second_config_output=second_config_output.stdout, difference=difference, ) return RETURN_FAIL # TODO(machenbach): Figure out if we could also return a bug in case there's # no difference, but one of the line suppressions has matched - and without # the match there would be a difference. print '# V8 correctness - pass' return RETURN_PASS