def _get_project_results_for_jobs(jobs): """Return projects for jobs.""" projects = {} for job in sorted(jobs, key=lambda j: j.name): project_name = job.get_environment().get('PROJECT_NAME', job.name) if project_name not in projects: projects[project_name] = {'name': project_name, 'jobs': []} if utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')): projects[project_name]['coverage_job'] = job.name engine_display_name, engine_name = _get_engine_names(job.name) projects[project_name]['jobs'].append({ 'engine_display_name': engine_display_name, 'engine_name': engine_name, 'sanitizer_string': environment.get_memory_tool_display_string(job.name), 'name': job.name, 'single_target': get_single_fuzz_target_or_none(project_name, engine_name), 'has_stats': True }) projects = projects.values() projects.sort(key=_sort_by_name) for project in projects: project['jobs'].sort(key=_sort_by_name) return projects
def get_issue_description(testcase, reporter=None, show_reporter=False, hide_crash_state=False): """Returns testcase as string.""" # Get issue tracker configuration parameters. config = db_config.get() domain = get_domain() testcase_id = testcase.key.id() fuzzer_name = testcase.actual_fuzzer_name() download_url = TESTCASE_DOWNLOAD_URL.format( domain=domain, testcase_id=testcase_id) report_url = TESTCASE_REPORT_URL.format( domain=domain, testcase_id=testcase_id) regressed_revision_range_url = TESTCASE_REVISION_RANGE_URL.format( domain=domain, job_type=testcase.job_type, revision_range=testcase.regression) fixed_revision_range_url = TESTCASE_REVISION_RANGE_URL.format( domain=domain, job_type=testcase.job_type, revision_range=testcase.fixed) if testcase.status == 'Unreproducible': return ('Testcase {testcase_id} failed to reproduce the crash. ' 'Please inspect the program output at {report_url}.'.format( testcase_id=testcase_id, report_url=report_url)) # Now create the content string. content_string = 'Detailed report: %s\n\n' % report_url project_name = get_project_name(testcase.job_type) if project_name and project_name != utils.default_project_name(): content_string += 'Project: %s\n' % project_name if fuzzer_name: content_string += 'Fuzzer: %s\n' % fuzzer_name binary_name = testcase.get_metadata('fuzzer_binary_name') if binary_name: content_string += 'Fuzz target binary: %s\n' % binary_name content_string += 'Job Type: %s\n' % testcase.job_type # Add platform id if other than default ones. Only applicable to Android. # e.g. android:shamu_asan if testcase.platform_id: content_string += 'Platform Id: %s\n\n' % testcase.platform_id content_string += 'Crash Type: %s\n' % get_crash_type_string(testcase) content_string += 'Crash Address: %s\n' % testcase.crash_address if hide_crash_state: crash_state = '...see report...' else: crash_state = testcase.crash_state content_string += 'Crash State:\n%s\n' % ( utils.indent_string(crash_state + '\n', 2)) content_string += '%s\n\n' % environment.get_memory_tool_display_string( testcase.job_type) if data_types.SecuritySeverity.is_valid(testcase.security_severity): content_string += ( 'Recommended Security Severity: %s\n\n' % severity_analyzer.severity_to_string(testcase.security_severity)) if (testcase.regression and testcase.regression != 'NA' and not testcase.regression.startswith('0:') and not testcase.regression.endswith('!')): content_string += 'Regressed: %s\n' % regressed_revision_range_url if (testcase.fixed and testcase.fixed != 'NA' and testcase.fixed != 'Yes' and not testcase.fixed.endswith('!')): content_string += 'Fixed: %s\n' % fixed_revision_range_url if not content_string.endswith('\n\n'): content_string += '\n' content_string += 'Reproducer Testcase: %s\n\n' % download_url second_crash_stacktrace = get_stacktrace( testcase, stack_attribute='second_crash_stacktrace') if testcase.one_time_crasher_flag and second_crash_stacktrace: content_string += (second_crash_stacktrace.split('\n'))[0] + '\n\n' if testcase.gestures: content_string += 'Additional requirements: Requires Gestures\n\n' if testcase.http_flag: content_string += 'Additional requirements: Requires HTTP\n\n' if show_reporter: if reporter: content_string += ( 'Issue manually filed by: %s\n\n' % reporter.split('@')[0]) else: content_string += 'Issue filed automatically.\n\n' # Jobs can override the help url. content_string += 'See %s for instructions to reproduce this bug locally.' % ( get_reproduction_help_url(testcase, config)) # Unreproducible crash text is only applicable when we are consistently seeing # it happening, and hence the reason for auto-filing it. Otherwise, someone # filed it manually, so skip the text in that case. if not reporter and testcase.one_time_crasher_flag: content_string += '\n\n' + FILE_UNREPRODUCIBLE_TESTCASE_TEXT return content_string
def get_testcase_detail(testcase): """Get testcase detail for rendering the testcase detail page.""" config = db_config.get() crash_address = testcase.crash_address crash_state = testcase.crash_state crash_state_lines = crash_state.strip().splitlines() crash_type = data_handler.get_crash_type_string(testcase) reproduction_help_url = data_handler.get_reproduction_help_url( testcase, config) external_user = not access.has_access(job_type=testcase.job_type) issue_url = issue_tracker_utils.get_issue_url(testcase) metadata = testcase.get_metadata() original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys) minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys) has_issue_tracker = bool(data_handler.get_issue_tracker_name()) if not testcase.regression: regression = 'Pending' elif testcase.regression == 'NA': regression = 'NA' else: regression = _get_revision_range_html_from_string(testcase.job_type, testcase.regression) fixed_full = None if 'progression_pending' in metadata: fixed = 'Pending' elif not testcase.fixed: fixed = 'NO' elif testcase.fixed == 'NA': fixed = 'NA' elif testcase.fixed == 'Yes': fixed = 'YES' else: fixed = 'YES' fixed_full = _get_revision_range_html_from_string(testcase.job_type, testcase.fixed) last_tested = None last_tested_revision = ( metadata.get('last_tested_revision') or testcase.crash_revision) if last_tested_revision: last_tested = _get_revision_range_html(testcase.job_type, last_tested_revision) crash_revision = testcase.crash_revision crash_revisions_dict = revisions.get_component_revisions_dict( crash_revision, testcase.job_type) crash_stacktrace = data_handler.get_stacktrace(testcase) crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type, crash_revisions_dict) crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines, crash_type) crash_stacktrace_preview_lines = _preview_stacktrace(crash_stacktrace) second_crash_stacktrace_revision = metadata.get( 'second_crash_stacktrace_revision') second_crash_stacktrace_revisions_dict = ( revisions.get_component_revisions_dict(second_crash_stacktrace_revision, testcase.job_type)) second_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute='second_crash_stacktrace') second_crash_stacktrace = filter_stacktrace( second_crash_stacktrace, testcase.crash_type, second_crash_stacktrace_revisions_dict) second_crash_stacktrace = convert_to_lines(second_crash_stacktrace, crash_state_lines, crash_type) second_crash_stacktrace_preview_lines = _preview_stacktrace( second_crash_stacktrace) last_tested_crash_revision = metadata.get('last_tested_crash_revision') last_tested_crash_revisions_dict = revisions.get_component_revisions_dict( last_tested_crash_revision, testcase.job_type) last_tested_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute='last_tested_crash_stacktrace') last_tested_crash_stacktrace = filter_stacktrace( last_tested_crash_stacktrace, testcase.crash_type, last_tested_crash_revisions_dict) last_tested_crash_stacktrace = convert_to_lines(last_tested_crash_stacktrace, crash_state_lines, crash_type) last_tested_crash_stacktrace_preview_lines = _preview_stacktrace( last_tested_crash_stacktrace) privileged_user = access.has_access(need_privileged_access=True) # Fix build url link. |storage.cloud.google.com| takes care of using the # right set of authentication credentials needed to access the link. if 'build_url' in metadata: metadata['build_url'] = metadata['build_url'].replace( 'gs://', 'https://storage.cloud.google.com/') pending_blame_task = ( testcase.has_blame() and 'blame_pending' in metadata and metadata['blame_pending']) pending_impact_task = ( testcase.has_impacts() and not testcase.is_impact_set_flag) pending_minimize_task = not testcase.minimized_keys pending_progression_task = ('progression_pending' in metadata and metadata['progression_pending']) pending_regression_task = not testcase.regression pending_stack_task = testcase.last_tested_crash_stacktrace == 'Pending' needs_refresh = ( testcase.status == 'Pending' or ((testcase.status == 'Processed' or testcase.status == 'Duplicate') and (pending_blame_task or pending_impact_task or pending_minimize_task or pending_progression_task or pending_regression_task or pending_stack_task))) if data_types.SecuritySeverity.is_valid(testcase.security_severity): security_severity = severity_analyzer.severity_to_string( testcase.security_severity) else: security_severity = None auto_delete_timestamp = None auto_close_timestamp = None if testcase.one_time_crasher_flag: last_crash_time = ( crash_stats.get_last_crash_time(testcase) or testcase.timestamp) # Set auto-delete timestamp for unreproducible testcases with # no associated bug. if not testcase.bug_information: auto_delete_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE)) # Set auto-close timestamp for unreproducible testcases with # an associated bug. if testcase.open and testcase.bug_information: auto_close_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)) memory_tool_display_string = environment.get_memory_tool_display_string( testcase.job_type) memory_tool_display_label = memory_tool_display_string.split(':')[0] memory_tool_display_value = memory_tool_display_string.split(':')[1].strip() helpers.log('Testcase %s' % testcase.key.id(), helpers.VIEW_OPERATION) return { 'id': testcase.key.id(), 'crash_type': crash_type, 'crash_address': crash_address, 'crash_state': crash_state, # Used by reproduce tool. 'crash_state_lines': crash_state_lines, 'crash_revision': testcase.crash_revision, 'csrf_token': form.generate_csrf_token(), 'external_user': external_user, 'footer': testcase.comments, 'fixed': fixed, 'fixed_full': fixed_full, 'issue_url': issue_url, 'is_admin': auth.is_current_user_admin(), 'metadata': metadata, 'minimized_testcase_size': minimized_testcase_size, 'needs_refresh': needs_refresh, 'original_testcase_size': original_testcase_size, 'privileged_user': privileged_user, 'regression': regression, 'crash_stacktrace': { 'lines': crash_stacktrace, 'preview_lines': crash_stacktrace_preview_lines, 'revision': revisions.get_real_revision( crash_revision, testcase.job_type, display=True) }, 'second_crash_stacktrace': { 'lines': second_crash_stacktrace, 'preview_lines': second_crash_stacktrace_preview_lines, 'revision': revisions.get_real_revision( second_crash_stacktrace_revision, testcase.job_type, display=True) }, 'last_tested_crash_stacktrace': { 'lines': last_tested_crash_stacktrace, 'preview_lines': last_tested_crash_stacktrace_preview_lines, 'revision': revisions.get_real_revision( last_tested_crash_revision, testcase.job_type, display=True) }, 'security_severity': security_severity, 'security_severities': data_types.SecuritySeverity.list(), 'stats': { 'min_hour': crash_stats.get_min_hour(), 'max_hour': crash_stats.get_max_hour(), }, 'suspected_cls': _parse_suspected_cls(metadata.get('predator_result')), 'testcase': testcase, 'timestamp': utils.utc_datetime_to_timestamp(testcase.timestamp), 'show_blame': testcase.has_blame(), 'show_impact': testcase.has_impacts(), 'impacts_production': testcase.impacts_production(), 'find_similar_issues_options': FIND_SIMILAR_ISSUES_OPTIONS, 'auto_delete_timestamp': auto_delete_timestamp, 'auto_close_timestamp': auto_close_timestamp, 'memory_tool_display_label': memory_tool_display_label, 'memory_tool_display_value': memory_tool_display_value, 'last_tested': last_tested, 'is_admin_or_not_oss_fuzz': is_admin_or_not_oss_fuzz(), 'has_issue_tracker': has_issue_tracker, 'reproduction_help_url': reproduction_help_url, 'is_local_development': environment.is_running_on_app_engine_development(), }
def get_testcase_detail(testcase): """Get testcase detail for rendering the testcase detail page.""" config = db_config.get() crash_address = testcase.crash_address crash_state = testcase.crash_state crash_state_lines = crash_state.strip().splitlines() crash_type = data_handler.get_crash_type_string(testcase) external_user = not access.has_access(job_type=testcase.job_type) issue_url = issue_tracker_utils.get_issue_url(testcase) metadata = testcase.get_metadata() original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys) minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys) has_issue_tracker = bool(data_handler.get_issue_tracker_name()) fuzzer_display = data_handler.get_fuzzer_display(testcase) formatted_reproduction_help = _format_reproduction_help( data_handler.get_formatted_reproduction_help(testcase)) # When we have a HELP_TEMPLATE, ignore any default values set for HELP_URL. if not formatted_reproduction_help: reproduction_help_url = data_handler.get_reproduction_help_url( testcase, config) else: reproduction_help_url = None if not testcase.regression: regression = "Pending" elif testcase.regression == "NA": regression = "NA" else: regression = _get_revision_range_html_from_string( testcase.job_type, testcase.regression) fixed_full = None if "progression_pending" in metadata: fixed = "Pending" elif not testcase.fixed: fixed = "NO" elif testcase.fixed == "NA": fixed = "NA" elif testcase.fixed == "Yes": fixed = "YES" else: fixed = "YES" fixed_full = _get_revision_range_html_from_string( testcase.job_type, testcase.fixed) last_tested = None last_tested_revision = (metadata.get("last_tested_revision") or testcase.crash_revision) if last_tested_revision: last_tested = _get_revision_range_html(testcase.job_type, last_tested_revision) crash_revision = testcase.crash_revision crash_revisions_dict = revisions.get_component_revisions_dict( crash_revision, testcase.job_type) crash_stacktrace = data_handler.get_stacktrace(testcase) crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type, crash_revisions_dict) crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines, crash_type) crash_stacktrace_preview_lines = _preview_stacktrace(crash_stacktrace) last_tested_crash_revision = metadata.get("last_tested_crash_revision") last_tested_crash_revisions_dict = revisions.get_component_revisions_dict( last_tested_crash_revision, testcase.job_type) last_tested_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute="last_tested_crash_stacktrace") last_tested_crash_stacktrace = filter_stacktrace( last_tested_crash_stacktrace, testcase.crash_type, last_tested_crash_revisions_dict, ) last_tested_crash_stacktrace = convert_to_lines( last_tested_crash_stacktrace, crash_state_lines, crash_type) last_tested_crash_stacktrace_preview_lines = _preview_stacktrace( last_tested_crash_stacktrace) privileged_user = access.has_access(need_privileged_access=True) # Fix build url link. |storage.cloud.google.com| takes care of using the # right set of authentication credentials needed to access the link. if "build_url" in metadata: metadata["build_url"] = metadata["build_url"].replace( "gs://", "https://storage.cloud.google.com/") pending_blame_task = (testcase.has_blame() and "blame_pending" in metadata and metadata["blame_pending"]) pending_impact_task = testcase.has_impacts( ) and not testcase.is_impact_set_flag pending_minimize_task = not testcase.minimized_keys pending_progression_task = ("progression_pending" in metadata and metadata["progression_pending"]) pending_regression_task = not testcase.regression pending_stack_task = testcase.last_tested_crash_stacktrace == "Pending" needs_refresh = testcase.status == "Pending" or ( (testcase.status == "Processed" or testcase.status == "Duplicate") and (pending_blame_task or pending_impact_task or pending_minimize_task or pending_progression_task or pending_regression_task or pending_stack_task)) if data_types.SecuritySeverity.is_valid(testcase.security_severity): security_severity = severity_analyzer.severity_to_string( testcase.security_severity) else: security_severity = None auto_delete_timestamp = None auto_close_timestamp = None if testcase.one_time_crasher_flag: last_crash_time = (crash_stats.get_last_crash_time(testcase) or testcase.timestamp) # Set auto-delete timestamp for unreproducible testcases with # no associated bug. if not testcase.bug_information: auto_delete_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE)) # Set auto-close timestamp for unreproducible testcases with # an associated bug. if testcase.open and testcase.bug_information: auto_close_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)) memory_tool_display_string = environment.get_memory_tool_display_string( testcase.job_type) memory_tool_display_label = memory_tool_display_string.split(":")[0] memory_tool_display_value = memory_tool_display_string.split( ":")[1].strip() helpers.log("Testcase %s" % testcase.key.id(), helpers.VIEW_OPERATION) return { "id": testcase.key.id(), "crash_type": crash_type, "crash_address": crash_address, "crash_state": crash_state, # Used by reproduce tool. "crash_state_lines": crash_state_lines, "crash_revision": testcase.crash_revision, "csrf_token": form.generate_csrf_token(), "external_user": external_user, "footer": testcase.comments, "formatted_reproduction_help": formatted_reproduction_help, "fixed": fixed, "fixed_full": fixed_full, "issue_url": issue_url, "is_admin": auth.is_current_user_admin(), "metadata": metadata, "minimized_testcase_size": minimized_testcase_size, "needs_refresh": needs_refresh, "original_testcase_size": original_testcase_size, "privileged_user": privileged_user, "regression": regression, "crash_stacktrace": { "lines": crash_stacktrace, "preview_lines": crash_stacktrace_preview_lines, "revision": revisions.get_real_revision(crash_revision, testcase.job_type, display=True), }, "last_tested_crash_stacktrace": { "lines": last_tested_crash_stacktrace, "preview_lines": last_tested_crash_stacktrace_preview_lines, "revision": revisions.get_real_revision(last_tested_crash_revision, testcase.job_type, display=True), }, "security_severity": security_severity, "security_severities": data_types.SecuritySeverity.list(), "stats": { "min_hour": crash_stats.get_min_hour(), "max_hour": crash_stats.get_max_hour(), }, "suspected_cls": _parse_suspected_cls(metadata.get("predator_result")), "testcase": testcase, "timestamp": utils.utc_datetime_to_timestamp(testcase.timestamp), "show_blame": testcase.has_blame(), "show_impact": testcase.has_impacts(), "impacts_production": testcase.impacts_production(), "find_similar_issues_options": FIND_SIMILAR_ISSUES_OPTIONS, "auto_delete_timestamp": auto_delete_timestamp, "auto_close_timestamp": auto_close_timestamp, "memory_tool_display_label": memory_tool_display_label, "memory_tool_display_value": memory_tool_display_value, "last_tested": last_tested, "is_admin_or_not_oss_fuzz": is_admin_or_not_oss_fuzz(), "has_issue_tracker": has_issue_tracker, "reproduction_help_url": reproduction_help_url, "is_local_development": environment.is_running_on_app_engine_development(), "fuzzer_display": vars(fuzzer_display), }